diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index be3cceeb25..7eca7e55d6 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -242,6 +242,40 @@ pub enum ChaosScenarioArg { S12, /// Rolling executor restarts under load. S13, + /// Executor pod kill while scheduled actions are between claim and fire. + S10, + /// Executor pod kill while agents are suspended on promises being completed. + S11, + /// Executor cut off from worker-service while it keeps its shards. + S3, + /// Executor pod kill while agents are having their state reverted. + S7, + /// Executor pod kill while agents are being deleted. + S6, + /// Executor pod kill while a component rollback is in flight. + S9, + /// Executors cut off from the key-value PostgreSQL cluster for about as + /// long as an AWS storage failover takes. + S16, + /// The same cut, held for longer than the key-value retry budget. + S22, + /// Executors cut off from the indexed-oplog PostgreSQL cluster for the + /// length of a writer failover. + S14, + /// Executors cut off from the Redis cache in front of the key-value layer, + /// for longer than a caller is willing to wait. + S18, + /// The same Redis cache slowed rather than removed. + S17, + /// The key-value PostgreSQL cluster slowed rather than removed. S17's + /// mirror on the other half of the split key-value layer. + S15, + /// S15 driving `ephemeral` alone: the control for the elimination series. + S15A, + /// S15A plus `durable`. + S15B, + /// S15B plus `promise`. + S15C, } /// Density subcommand action. diff --git a/golem-test-framework/src/benchmark/results.rs b/golem-test-framework/src/benchmark/results.rs index 15785319ae..9ccd9685a7 100644 --- a/golem-test-framework/src/benchmark/results.rs +++ b/golem-test-framework/src/benchmark/results.rs @@ -525,6 +525,17 @@ pub struct RunMetadata { /// Container image tag of the deployed `worker-service`. #[serde(skip_serializing_if = "Option::is_none", default)] pub worker_service_image_tag: Option, + /// Image digest of the deployed `worker-executor`, when the manifest pins + /// one. A tag can be moved; the digest is what actually identifies the + /// build a run tested, which is the whole point of recording it. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub worker_executor_image_digest: Option, + /// Image digest of the deployed `registry-service`. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub registry_service_image_digest: Option, + /// Image digest of the deployed `worker-service`. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub worker_service_image_digest: Option, /// Aurora ACU capacity for the main (`golem_dev`) cluster at run start. #[serde(skip_serializing_if = "Option::is_none", default)] pub aurora_acu_main: Option, @@ -575,6 +586,9 @@ impl RunMetadata { worker_executor_image_tag: env_str("GOLEM_BENCH_WORKER_EXECUTOR_IMAGE_TAG"), registry_service_image_tag: env_str("GOLEM_BENCH_REGISTRY_SERVICE_IMAGE_TAG"), worker_service_image_tag: env_str("GOLEM_BENCH_WORKER_SERVICE_IMAGE_TAG"), + worker_executor_image_digest: env_str("GOLEM_BENCH_WORKER_EXECUTOR_IMAGE_DIGEST"), + registry_service_image_digest: env_str("GOLEM_BENCH_REGISTRY_SERVICE_IMAGE_DIGEST"), + worker_service_image_digest: env_str("GOLEM_BENCH_WORKER_SERVICE_IMAGE_DIGEST"), aurora_acu_main: env_f64("GOLEM_BENCH_AURORA_ACU_MAIN"), aurora_acu_indexed: env_f64("GOLEM_BENCH_AURORA_ACU_INDEXED"), aurora_acu_keyvalue: env_f64("GOLEM_BENCH_AURORA_ACU_KEYVALUE"), diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 00ed926cfc..758728331b 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -175,7 +175,11 @@ scenarios: # executors that stay connected keep getting told the truth, and the # disagreement between the two groups is the whole experiment. targetCount: 1 - durationSecs: 120 + # Must equal phases.faultSecs; see the note on S16. The joiner partition + # this scenario applies mid-fault is already sized from faultSecs, so a + # shorter value here also made the two partitions lapse at different + # times. + durationSecs: 180 phases: baselineSecs: 300 @@ -318,6 +322,151 @@ scenarios: signalTimeoutSecs: 1800 + # S10 — executor pod kill while scheduled actions are pending (GOL-378). + # + # The only scenario whose workload the driver is not holding a connection to. + # Every target registers an action ten seconds ahead, over and over, so a few + # hundred actions are always accepted and not yet run. Killing the executor + # that owns their shards asks whether the platform still runs each of them, + # exactly once, after the shards move. + # + # Like S8 the driver names the pod: it picks the executor owning the largest + # share of targets and reports its address in `baseline-ready.json`. Unlike + # S8, the targets on the *other* executor keep registering too — they are the + # control group, and the report splits the two. + - code: S10 + name: executor-crash-during-scheduled-fire + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, exactly as in S8. + mode: one + durationSecs: 60 + + phases: + # Eighteen leads' worth of registrations, so the baseline measures a + # cluster that has been firing scheduled actions steadily rather than one + # that has just started. + baselineSecs: 180 + # Covers the kill, the reschedule, and the shard reassignment that has to + # happen before anything claims the dead executor's due actions. + faultSecs: 120 + # Registrations continue throughout. Long enough that the actions + # registered during the fault also fall due, fire and are accounted for + # inside the run rather than after it. + recoverySecs: 300 + + scheduled: + # 100 targets, each with its own emitter. Also the resolution of the + # report: a lost action localises to one target out of a hundred. + targets: 100 + # 50 registrations a second overall, which is half what the mixed workload + # offers. The scheduled path is the whole experiment here rather than one + # stream of four, so the rate is set by what keeps the pending population + # large rather than by a share of a budget. + intervalMillis: 2000 + # 500 actions accepted and not yet run at any instant. It also has to + # comfortably exceed the workflow's inject-and-verify path — signal poll + # (5s) plus `kubectl apply` plus waiting for `AllInjected` — or everything + # registered before the kill would already have fired by the time the pod + # died. + leadSecs: 10 + # What recovering a pending action may cost, and the number the fire-delay + # percentiles are reported against. Derived rather than picked: the + # executor's scheduler holds a 30s lease and refreshes every 2s, and a + # shard reassignment has to complete before the surviving executor claims + # anything at all. 60s covers both without being so generous that a + # regression would sit inside it. + # + # Recorded, not asserted. A p99 past it is an attention line and a number + # in the result, because how much a reassignment may cost is a judgement. + leaseBudgetSecs: 60 + + retryPolicy: + # Identical to the others and load-bearing for the same reason, with one + # extra consequence here: the retry goes out under the original key, and + # that key is what the scheduled action carries back to the target. So a + # retried registration that registered twice shows up as one token firing + # twice, not as an arithmetic argument about totals. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S11 — executor pod kill while agents are suspended on promises (GOL-377). + # + # The only scenario whose agents are asleep when the fault lands. Each waiter + # creates a promise, parks inside an invocation awaiting it, and is resolved + # from outside a few seconds later. Killing the executor that owns them asks + # whether a completion the platform accepted still reaches the agent it was + # meant to wake, exactly once, after the shards move. + # + # Like S10 the driver names the pod and keeps driving the waiters on the other + # executor as a control group. Unlike every other scenario, its agents live in + # the promise component rather than the counters one. + - code: S11 + name: executor-crash-during-promise-completion + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, exactly as in S8 and S10. + mode: one + durationSecs: 60 + + phases: + # Long enough for cold starts and route warm-up to settle, so the kill + # lands on a population that has been cycling steadily rather than one + # still arriving. + baselineSecs: 180 + # Covers the kill, the reschedule, and the shard reassignment that has to + # happen before anything can resume a waiter the dead executor owned. + faultSecs: 120 + # Rounds continue throughout. Long enough that waiters disturbed by the + # kill get several whole rounds afterwards, so a waiter that recovered + # late is distinguishable from one that never did. + recoverySecs: 300 + + promise: + # 200 waiters, each parked on at most one promise, so 200 agents are + # suspended at any instant. Also the resolution of the report: a lost + # wakeup localises to one waiter out of two hundred. + waiters: 200 + # 5s parked before the completion goes out, so ~40 completions a second + # across the pool. It has to comfortably exceed the workflow's + # inject-and-verify path — signal poll (5s) plus `kubectl apply` plus + # waiting for `AllInjected` — or every promise armed before the kill would + # already have been completed by the time the pod died. + dwellMillis: 5000 + # What resuming a suspended waiter may cost, and the number the wakeup + # delay percentiles are reported against. Derived rather than picked: a + # shard reassignment has to complete before anything owns the waiter, and + # then the worker has to be recovered and its oplog replayed to the point + # it was parked at. 60s covers both without being so generous that a + # regression would sit inside it. + # + # Recorded, not asserted. A p99 past it is an attention line and a number + # in the result, because how much a reassignment may cost is a judgement. + wakeupBudgetSecs: 60 + + retryPolicy: + # Identical to the others. One extra consequence here, spelled out in + # integration-tests/src/chaos/waiters.rs: a completion retry can repair a + # wakeup that was lost, but only for a completion that had *not* already + # been accepted — so the question this scenario asks, about completions the + # platform confirmed, is untouched by it. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + # S13 — rolling executor restarts under load (GOL-367). # # One executor killed every 60 seconds for five minutes while the mixed @@ -384,3 +533,1359 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # S3 — executor / worker-service network partition (GOL-370). + # + # The only fault in the suite the platform never learns about. S1 cuts the + # shard-manager off from an executor and the cluster reacts: health checks are + # missed and the shards move. S3 cuts the link worker-service uses, leaves the + # shard-manager link alone, and so the executor keeps its shards and stays in + # the routing table. worker-service is told, correctly, that this pod owns + # those agents, and keeps trying to reach a pod it cannot reach. There is no + # route around it, because as far as the platform is concerned nothing is + # wrong. + # + # Like S8, S10 and S11 the driver names the pod rather than letting Chaos Mesh + # pick one, and keeps driving the agents on the other executor as the control + # group. That control group is where the interesting finding would be: a + # partition it is not part of should cost it nothing. + - code: S3 + name: executor-worker-service-partition + enabled: true + + fault: + kind: network-partition + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, exactly as in S8, S10 and S11. + # + # Only the executor side is pinned. The other side of the cut is every + # worker-service replica, selected by label — a partition from two of the + # three would leave the agents reachable through the third and the run + # would measure nothing. + mode: one + durationSecs: 180 + + phases: + # Long enough for cold starts and route warm-up to settle, so the cut + # lands on a population that has been cycling steadily. + baselineSecs: 300 + # Has to comfortably exceed the caller's 120s attempt timeout, or no + # operation would ever reach the timeout inside the window and the + # pending-then-timeout behaviour the scenario exists to show would fall + # outside it. 180s leaves room for the first attempt to time out, the + # one permitted retry to go out, and that retry to still be waiting when + # the link comes back. + faultSecs: 180 + # Every isolated agent has to get several whole operations after the heal, + # so one that recovered slowly is distinguishable from one that never did. + recoverySecs: 420 + + isolation: + # 200 agents, split by shard ownership across the two executors, so ~100 + # end up isolated and ~100 are the control group. Also the resolution of + # the report: a lost agent localises to one out of two hundred. + agents: 200 + # One operation per agent per second, so ~200 ops/s offered while + # everything is reachable. Higher than the other scenarios' 100/s because + # half of it stops the moment the fault lands: the number that has to be + # large is the *control* group's, since a collateral-damage finding is a + # percentage change in its throughput and a small sample cannot show one. + intervalMillis: 1000 + # The most of its baseline the isolated group may keep for the partition + # to count as observed. Not zero: an operation submitted late in the fault + # window can still be waiting when the link returns, and then confirms. + # At this cadence that is a handful against a baseline of hundreds, so + # anything near 25% means the cut did not take hold. + isolatedCeilingPercent: 25 + # The least of its baseline the control group must keep. Below this, + # serving an unreachable executor cost the agents that had nothing to do + # with it — the sharpest finding this scenario can produce. + # + # 75% rather than 95%: worker-service invalidates one process-wide routing + # table entry on every failed call, so some shared cost is expected by + # construction. `invalidation_min_delay` bounds it to twice a second, + # which should keep it far above this line. + controlFloorPercent: 75 + # What being served again may cost an isolated agent once the link is + # back. Recorded, not asserted: nothing has to be recovered here — no + # shard moves and no worker restarts — so this is really a measure of how + # long worker-service's own retry loop takes to notice, and how long that + # may be is a judgement. + recoveryBudgetSecs: 60 + + retryPolicy: + # Identical to the others and load-bearing for the same reason, with one + # consequence specific to S3: the retry is what turns a single 120s stall + # into a second attempt under the same key. If the link heals in between, + # that retry succeeds against work the first attempt may already have + # started — which is exactly the case the exactly-once probe is here to + # rule on. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S7 — executor pod kill during agent state revert (GOL-371). + # + # The only scenario that disturbs work the platform is trying to *undo*. Each + # agent builds its counter up with a run of increments and then asks for some + # of them back, over and over, while an executor is killed underneath. + # + # It is also the only read-back in the suite with no band of doubt in it. The + # last increment of a round returns the counter's value, so the driver knows + # exactly what the agent was worth before the revert and exactly how many + # invocations it asked to take back. Afterwards there are two legal values and + # nothing between them, which is why S7's findings fail the run outright. + # + # Like S8, S10, S11 and S3 the driver names the pod and keeps driving the + # agents on the other executor as a control group. + - code: S7 + name: executor-crash-during-revert + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, as in every other pinned scenario. + mode: one + durationSecs: 60 + + phases: + # Long enough for cold starts to settle, so the kill lands on a population + # that has been cycling rounds steadily rather than one still arriving. + baselineSecs: 180 + # Covers the kill, the reschedule, and the shard reassignment that has to + # happen before anything can revert an agent the dead executor owned. + faultSecs: 120 + # Rounds continue throughout. Long enough that every agent runs several + # whole rounds after the kill, so a round disturbed by it is followed by + # ones that were not. + recoverySecs: 300 + + revert: + # 200 agents, split by shard ownership across the two executors. Also the + # resolution of the report: a torn revert localises to one agent out of + # two hundred. + agents: 200 + # Four increments then a revert of two, so a completed round is worth +2. + # + # `revertInvocations` must not exceed `incrementsPerRound`: a revert that + # reached further back would land in the region an earlier revert already + # deleted, and the platform refuses that outright — every round would fail + # and the run would measure nothing. The driver checks this before + # starting rather than discovering it per round. + incrementsPerRound: 4 + revertInvocations: 2 + # Between rounds. With five operations per round this keeps roughly one + # agent in six mid-round at any instant, which is the population a kill + # has something to land in. It also has to leave the round long enough + # that the workflow's inject-and-verify path — signal poll (5s) plus + # `kubectl apply` plus waiting for `AllInjected` — does not fall entirely + # between two rounds. + intervalMillis: 500 + # What recovering a reverted agent may cost. Recorded, not asserted: the + # floor is a shard reassignment plus the worker recovery that replays the + # oplog up to the revert, and how much more than that is acceptable is a + # judgement. + recoveryBudgetSecs: 60 + + retryPolicy: + # Applies to the increments only. **The revert is never retried**, and + # that exception is load-bearing rather than an oversight: a revert has no + # idempotency key and is not idempotent — asking twice for "the last two + # invocations" takes back four. A retried revert that had actually landed + # would look exactly like the platform tearing a truncation, which is the + # finding this scenario exists to make. See `chaos/reverts.rs`. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S6 — executor pod kill during agent deletion (GOL-372). + # + # S7 asks the platform to forget some of an agent's work. S6 asks it to forget + # the agent, and kills the executor while it is doing so. + # + # The same two-value oracle, one step further: invoking a deleted id creates a + # *new* agent, so when a slot is used again it must report either 1 (the + # deletion took) or its old value plus one (it did not). Neither is a defect + # alone — a lost response leaves the question open — but "the platform + # confirmed the deletion and the agent is still worth what it was" is the + # resurrection this scenario is named for, and it fails the run. + # + # Worth knowing before reading a result: the happy path is already defended. + # `Worker::start_deleting` stops the background status flush and checkpointer + # first, specifically so neither can resurrect the cached status after the + # durable removal. The question is whether that defence survives the pod dying + # between the mark and the removal. + - code: S6 + name: executor-crash-during-deletion + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, as in every other pinned scenario. + mode: one + durationSecs: 60 + + phases: + baselineSecs: 180 + faultSecs: 120 + # Rounds continue throughout, so every slot is reused several times after + # the kill and a round it disturbed is followed by ones it did not. + recoverySecs: 300 + + delete: + # 200 slots, split by shard ownership. Also the resolution of the report: + # a resurrection localises to one slot out of two hundred. + agents: 200 + # Three increments before each delete, so an agent is worth 3 when it + # goes. More than one is load-bearing rather than arbitrary: with a single + # increment a fresh agent and a resurrected one both read 1 and the oracle + # cannot tell them apart. The driver refuses to start below 2. + incrementsPerRound: 3 + # Between rounds. A round is four operations, so this keeps the delete a + # meaningful share of each slot's cycle and gives the kill something to + # land in. + intervalMillis: 500 + # What recovering a slot may cost once its executor is back. Recorded, not + # asserted. + recoveryBudgetSecs: 60 + + retryPolicy: + # Applies to the increments only. **The delete is never retried**, for a + # reason specific to this scenario: `delete_worker_internal` starts with a + # metadata lookup and returns `worker_not_found` when there is nothing + # there, so deleting twice does not succeed twice. A delete whose response + # was lost but which landed would come back on retry as a refusal, and the + # run would record "refused, and gone anyway" — one of the violations this + # scenario exists to detect. See `chaos/deletions.rs`. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S9 — executor pod kill during a component rollback (GOL-369). + # + # S5 moves agents forward onto a new build and kills an executor while that is + # happening. S9 moves them forward, waits for that to land, and then moves them + # back — killing an executor during the return leg. + # + # The return leg is the one that matters operationally. A rollback is what you + # reach for when the new build is already going wrong, so a rollback happening + # under a dying executor is the situation you would actually be in. + # + # The rollback is a redeploy: `agent-counters` is uploaded again as a new + # revision and every agent is asked to move to it. `Counter::component_version` + # is compiled into each build, so an agent that has genuinely returned reports + # 1 from the code that is running rather than from metadata about what the + # platform believes. + - code: S9 + name: executor-crash-during-rollback + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # `one` plus the namespace and opt-in label selectors in podchaos-s9.yaml + # bound the blast radius. Unlike S8 the driver does not name the pod: + # agents are mid-rollback across both executors, so either one interrupts + # returns in flight and which one carries no information. + mode: one + durationSecs: 60 + + phases: + # Long enough for cold starts and route warm-up to settle, so the roll + # forward is applied to a population that is running rather than arriving. + baselineSecs: 300 + # Has to cover the kill, the reschedule, and the rollback finishing on + # both the surviving executor and the replacement. + faultSecs: 120 + # Returns continue after the kill and agents have to converge before the + # census decides which build they are on. + recoverySecs: 300 + + workload: + # Durable agents are the population being rolled back, so the pool is + # large: a stuck agent localises to one out of 200. + durableAgents: 200 + # Not rolled back — they exist to show whether acceptance degraded during + # the kill, separately from the rollback. + ephemeralAgents: 50 + # Durable state the rollback does not touch, so they are the control: + # their read-back should be unaffected. + scheduledAgents: 50 + promiseAgents: 0 + # No quota stream, for the same reason as S5: a lease that cannot be + # renewed would add a second failure mode to a scenario that has one. + quotaAgents: 0 + ratePerSec: 100 + + rollback: + # How long to let the roll-forward land before measuring it. Too short and + # the census reads a population still in transit, and the gate below + # refuses a rollback that would have been fine. + settleSecs: 90 + # The share of agents that must actually report the new build before the + # rollback is attempted at all. Below it the driver stops: rolling agents + # back to a build they never left would pass every check and prove + # nothing, which is the worst artifact this suite can produce. + rolledForwardFloorPercent: 90 + # Retries for the rollback *control plane*, counted apart from the + # workload's own. Different question: a request refused because its + # agent's executor just died says nothing about correctness, but an agent + # nobody successfully asked to come back explains a stale agent later + # without excusing one. + controlRetries: 2 + controlRetryDelaySecs: 5 + # How far into the rollback to ask for the kill. As in S5 this is when the + # driver *asks*; applying the PodChaos and confirming it takes a few + # seconds more, and the result records both so how far in it landed is + # readable rather than assumed. + killDelaySecs: 2 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S16 — key-value PostgreSQL outage recovery (GOL-379). + # + # The first fault in the suite aimed at something the platform depends on + # rather than at the platform itself. The executors keep running, keep their + # shards and keep answering the shard-manager. What they lose is the Aurora + # cluster underneath them, which on golem-dev carries promises, the + # running-workers set, user key-value data and the scheduler's own schema. + # + # The oplog is on a different cluster and the worker-status hot cache is in + # Redis, and neither is touched. That is the interesting part: the platform + # keeps the ability to record what it did and loses the ability to know what + # it is doing. + # + # There is no control group of executors, because all three share the cluster. + # The control is the baseline window, and the first thing the report has to + # establish is that throughput collapsed at all — a partition that silently + # failed to take hold would otherwise produce a clean-looking account of an + # undisturbed cluster. + - code: S16 + name: keyvalue-postgres-outage + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, unlike every other partition in this suite. A storage + # outage that only some executors could see would be a routing fault + # wearing a database's clothes, and S3 already covers that shape. + mode: all + # Must equal phases.faultSecs. Chaos Mesh recovers a NetworkChaos on its + # own `duration`, while the workflow holds the phase for faultSecs and + # only then deletes the object. Setting this lower lifts the partition + # early and leaves the tail of the fault phase measuring a healed + # cluster, which is how the first S16 run came to report 16.87% of + # baseline throughput during a total outage. + durationSecs: 60 + + phases: + # Long enough for cold starts, component loading and route warm-up to + # settle, and for the scheduler to be firing steadily, so the cut lands on + # a cluster in steady state. + baselineSecs: 300 + # Sized to the outage this scenario models rather than to the caller's + # timeout: the worst case an AWS storage failover imposes, which is an + # Aurora reader promotion of about a minute and an ElastiCache Multi-AZ + # promotion of about thirty seconds. The key-value retry budget is built + # to cover exactly that, so an outage of this length is one the platform + # is expected to absorb without losing an executor. + # + # Deliberately shorter than S3's 180s, and not for the reason S3 gives. + # S3 needs the caller's 120s attempt timeout to fire inside the window + # because pending-then-timeout is the behaviour it exists to show. Here + # the opposite is the claim: invocations issued during the cut should + # still be waiting when the link returns and then complete. Throughput is + # attributed by completion, so the fault window still reads as a collapse + # and the recovery window as the catch-up, which is what says the + # partition took hold. + # + # Do not raise this past the retry budget without meaning to. The budget + # is 15 attempts, ~93s of capped backoff plus a 5s pool acquire apiece + # against a blackholed endpoint, so roughly 168s. Beyond that the + # recovery-index write in `status_flusher` exhausts and panics by design, + # and the scenario stops measuring absorption and starts measuring the + # crash it is supposed to have removed. + faultSecs: 60 + # Every stream has to get several whole operations after the heal, so a + # stream that recovered slowly is distinguishable from one that never + # did. Scheduled actions registered during the outage also have to fall + # due and fire inside the run rather than after it. + recoverySecs: 420 + + workload: + # The population read back agent by agent, so a lost increment localises + # to one agent out of 200. + durableAgents: 200 + # No durable state of their own, but they still need the running-workers + # set to start, so they say whether acceptance degraded separately from + # whether state survived. + ephemeralAgents: 50 + # Zero on purpose. The scheduled stream is driven by the `scheduled` block + # below instead, because only its registrations carry a token into the + # target's fire log and scheduler lag cannot be measured without one. + # Setting both is refused at load time. + scheduledAgents: 0 + # Promises live in the cluster being taken away, so this stream is + # directly on the fault path rather than incidental to it. + promiseAgents: 50 + # No quota stream: a lease renewal failing would add a shard-manager + # failure mode to a scenario that is about storage. + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + # 100 targets, each with its own emitter, same as S10. Also the resolution + # of the report: a lost action localises to one target out of a hundred. + targets: 100 + intervalMillis: 2000 + # 500 actions accepted and not yet run at any instant, so the outage + # begins with a large population of work the platform has promised to do + # and cannot currently read. + leadSecs: 10 + # What recovering a pending action may cost. Larger than S10's 60s and + # deliberately so: there is no shard reassignment here, but the scheduler + # cannot claim or acknowledge anything for the whole 180s outage, so the + # floor is the outage itself. 240s covers the outage plus a lease TTL and + # some catch-up without being so generous that a regression hides inside + # it. + # + # Recorded, not asserted, like every other budget in the suite. + leaseBudgetSecs: 240 + + storage: + # The Aurora writer endpoint the workflow names in the NetworkChaos + # manifest's `externalTargets`. Recorded here so the archived result says + # which storage the run was about; the driver never resolves or contacts + # it. + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + # The least of the fault window every stream must answer nothing at all + # for, as a share of that window, for the outage to count as observed. + # + # This was a ceiling on during-fault throughput as a share of baseline. + # That number is still reported, but it cannot be the verdict: it is a + # rate averaged over the whole window, while an absorbed outage does all + # its serving in the few seconds at the window's edges. The same handful + # of edge confirmations reads as ~8% of a 180s window and ~26% of a 60s + # one, so the threshold tracked the window length rather than the + # platform, and shortening this window to 60s duly tripped it on a + # partition that had plainly landed (run 33099228775: quiet 58.0s of + # 62.3s on every stream, yet 25.79% against a 15% ceiling). + # + # Quiet time is measured against the window's own edges, so it means the + # same thing whatever the window is. 50% is far from both outcomes it has + # to tell apart: a landed outage is quiet for over 90% of the window, an + # undisturbed cluster for a few percent. + expect: + # Everything is behind this cut, so every stream that was serving is + # expected to stop. See `OutageExpectation` for why the rule is named + # per scenario rather than shared: S18 cuts a store only part of the + # workload touches, and this rule reads that arrangement backwards. + kind: whole-workload + quietFloorPercent: 50 + # What serving again may cost once the cluster is reachable. Recorded, not + # asserted: this is really a measure of how long sqlx's pool takes to + # notice, plus whatever recovery scan the executors run, and how long that + # may be is a judgement. + recoveryBudgetSecs: 120 + + retryPolicy: + # Identical to the others and load-bearing for the same reason, with one + # consequence specific to S16: the retry is what turns a single stalled + # attempt into a second one under the same key. If the database comes back + # in between, that retry lands against work the first attempt may already + # have half-done, which is exactly what the exactly-once probe is here to + # rule on. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S22 — the long-outage companion to S16 (GOL-499). Same cut, same driver, + # held past the point where the platform can absorb it. + # + # S16 asks whether golem rides out a storage failover. This asks what happens + # when the storage does not come back in time. The key-value retry budget is + # 15 attempts, roughly 93s of capped backoff plus a 5s pool acquire apiece + # against a blackholed endpoint, so about 168s. Past that the recovery-index + # write in `status_flusher` panics on purpose, on the grounds that an executor + # which cannot maintain its own running-workers index should be replaced + # rather than left running with agents it can no longer account for. + # + # So executor restarts are the *expected* outcome here, not a defect, and + # nothing in this entry treats them as one. What the run has to establish is + # that the exit is the intended one and that nothing is lost across it: the + # scheduled-fire account and the exactly-once account are the same assertions + # S16 makes, and they are the ones that fail the build. + - code: S22 + name: keyvalue-postgres-outage-past-budget + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, as in S16. A budget exhausted on only some of them would + # leave the survivors serving and make the replay cost unreadable. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 300 + + phases: + # Same as S16: long enough for cold starts, component loading and route + # warm-up to settle, and for the scheduler to be firing steadily. + baselineSecs: 300 + # Comfortably past the ~168s budget rather than just over it. A window + # that only just exceeds the budget would make the run's outcome depend + # on which second an operation happened to start in, so a rerun could + # come back with restarts or without them for no reason worth reporting. + # At 300s every tracked transition in the window exhausts. + faultSecs: 300 + # Longer than S16's 420s, and for a reason S16 does not have: the + # executors are replaced here, so recovery is not just a pool noticing + # its database came back. It is pods rescheduling, rejoining the routing + # table, and every agent that was on them replaying from oplog or + # snapshot. Scheduled actions registered during the outage also have to + # fall due and fire inside the run rather than after it. + recoverySecs: 600 + + workload: + # Identical to S16 on purpose. The two runs are meant to be read side by + # side, and a different population would make the difference between them + # a question about the workload rather than about the outage length. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Wider than S16's 240s. The lease has to outlast the outage plus the + # replay behind it, or actions would be judged late for the recovery this + # scenario exists to measure. + leaseBudgetSecs: 480 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + # Same rule and floor as S16, and it should be met even more comfortably: + # an outage this long leaves the streams silent for nearly the whole + # window. + expect: + kind: whole-workload + quietFloorPercent: 50 + # Recorded, not asserted, as everywhere else — but read differently here. + # In S16 this measures a connection pool noticing. Here it measures a + # cluster rebuilding itself, and it is the number that says whether the + # deliberate crash is an acceptable trade. + recoveryBudgetSecs: 300 + + retryPolicy: + # Identical to S16's, for the same reason. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S14 — the oplog half of the storage matrix (GOL-376). Same driver as S16 and + # S22, same cut, aimed at the other Aurora cluster. + # + # S16 takes away the cluster that says what the platform is doing: promises, + # the running-workers set, user key-value data and the scheduler's schema. S14 + # takes away the one that says what it did. The oplog is the only thing on the + # indexed cluster, so under this cut promises still resolve, the scheduler + # still claims and acknowledges on time, and the running-workers set is still + # writable — and nothing durable can be committed. The two scenarios are + # near-mirror images and are meant to be read side by side. + # + # The retry budget is not the interesting quantity here the way it is in S16 + # and S22. golem-dev gives indexed storage 200 attempts with a 10s cap, which + # is over half an hour of stalling, so this window cannot exhaust it. What is + # worth watching instead is the error classification: `retry_storage_op` + # panics immediately on any indexed-storage error it was not handed as + # transient, and transient means `PoolTimedOut` or `Io` and nothing else. An + # executor lost during this run is a misclassification finding, not an + # exhaustion one. + - code: S14 + name: indexed-oplog-postgres-outage + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, as in S16 and for the same reason: a storage outage only + # some executors could see would be a routing fault wearing a database's + # clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 60 + + phases: + # Same as S16, so the two are comparable: long enough for cold starts, + # component loading and route warm-up to settle, and for the scheduler to + # be firing steadily. + baselineSecs: 300 + # The same worst-case Aurora writer promotion S16 is sized to, about a + # minute. GOL-376 was written with 30s, before that number had been + # worked out; 60s is the failover this is supposed to model and matching + # S16 is what makes the two results readable against each other. + faultSecs: 60 + # As S16: every stream needs several whole operations after the heal, and + # scheduled actions registered during the outage have to fall due and fire + # inside the run. + recoverySecs: 420 + + workload: + # Identical to S16, deliberately. The comparison between the two runs is + # the point, and a different population would turn the difference into a + # question about the workload. + durableAgents: 200 + ephemeralAgents: 50 + # Zero for the same reason as S16: the scheduled stream comes from the + # block below, because only its registrations carry a token into the + # target's fire log. Setting both is refused at load time. + scheduledAgents: 0 + # Promises live on the cluster this cut leaves alone, so unlike in S16 + # this stream is not directly on the fault path. It still degrades, + # because completing a promise wakes an agent that then cannot commit, + # and that gap between "the promise was written" and "the agent got + # anywhere" is worth having on the record. + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Same as S16's, and it should be met with room to spare. The scheduler is + # outside this cut, so claims and acknowledgements keep their timing and + # the delay reduces to how long the fired invocation could not commit. + # That makes S14 the cleanest read of scheduler lag in the suite: in S16 + # and S22 the scheduler is inside the outage and the two costs cannot be + # separated. + leaseBudgetSecs: 240 + + storage: + # The indexed writer endpoint, not the key-value one. This is the only + # field that differs materially from S16. + endpoint: golem-postgres-dev-indexed.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + # Same rule, floor and reasoning as S16. + expect: + kind: whole-workload + quietFloorPercent: 50 + # Recorded, not asserted. Expected to be small: nothing is replaced here, + # so this is a connection pool noticing its database is back plus whatever + # the stalled commits still have to do. + recoveryBudgetSecs: 120 + + retryPolicy: + # Identical to S16's, for the same reason. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S18 — Redis outage (GOL-384) + # + # The fourth scenario on the storage driver and the first that does not cut a + # database. golem-dev runs the key-value layer as `NamespaceRouted`: the + # `Worker`, `AgentStatus` and `AgentStatusCheckpoint` namespaces go to a Redis + # cache and everything else goes to Postgres. S16 cut the Postgres half and + # left this one standing; S18 cuts exactly what S16 left. + # + # It is sized differently from the other three, and the reason is in the + # config rather than in the ambition. Those three are measured against a + # platform retry budget — S16 and S14 stay inside one, S22 runs past one. This + # client has no budget to run past: golem-dev sets the cache's + # RETRIES__MAX_ATTEMPTS to 0, which the code turns into a fred reconnect + # policy with unlimited attempts, and fred's defaults then give the client no + # command timeout and an unbounded command buffer. A cache write during the + # cut does not fail. It waits. + # + # So the only clock that can run out is the caller's, and the window is set + # against that instead: past the 120s attempt timeout, so operations time out + # and retry under the same idempotency key while the original write is still + # queued inside fred. Two landings of one operation is what this run is for. + # --------------------------------------------------------------------------- + - code: S18 + name: redis-cache-outage + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, as in the other three: a cache outage only some + # executors could see would be a routing fault wearing a cache's clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 180 + + phases: + # Same as the other three, so all four are comparable. + baselineSecs: 300 + # Past the workload's 120s ATTEMPT_TIMEOUT plus its 5s retry delay, so an + # operation that starts early in the window times out, retries, and has + # that retry still in flight when the heal arrives. At 60s nothing would + # time out and the run would only show slower operations completing, which + # the config already tells us without spending a run. + faultSecs: 180 + # As the others. The drain here is larger than S14's — three minutes of + # coalesced status writes and queued commands come back at once — so this + # needs to cover a burst rather than a trickle. + recoverySecs: 420 + + workload: + # Identical to S16 and S14, deliberately, so the difference between the + # four results is the fault and not the population. + durableAgents: 200 + ephemeralAgents: 50 + # Zero for the same reason as the others: the scheduled stream comes from + # the block below. Setting both is refused at load time. + scheduledAgents: 0 + # Expected to be one of the two streams that actually stops. Completing a + # promise suspends and wakes an agent, and a suspend is a lifecycle + # boundary, which is one of the few things that still crosses this cache + # synchronously. + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Raised from the 240 the other three use, because the window itself is + # 180s. A fire at the start of the outage is delayed by roughly the whole + # window before it can make progress, and then joins a drain queue behind + # three minutes of backlog. 300 leaves headroom for that tail without + # making a delay the platform should be ashamed of look acceptable. + leaseBudgetSecs: 300 + + storage: + # The Redis primary endpoint, not either Aurora writer. Both Postgres + # clusters stay reachable for the whole run. + endpoint: master.golem-redis-dev.intsie.use1.cache.amazonaws.com + # The one scenario in this family that does not use the whole-workload + # rule, because this cut is partial by construction and that rule reads a + # partial cut backwards. Run 33130077355 proved it: `durable` held 100% + # of its baseline throughout, exactly as NamespaceRoutedKeyValueStorage + # says it should, and the shared rule reported the partition as one that + # never landed. + # + # `ephemeral` is the only stream named. Its agents are created and torn + # down per operation, so every one of them crosses a lifecycle boundary, + # which is one of the few things that still reaches this cache + # synchronously. It went silent for 99.997% of that first window. + # + # `promise` was expected to be the second and was not: the mixed + # workload's promise stream is get_promise+complete in one round trip + # against a durable agent and never suspends. The suspending variant is + # `promise-wait`, which S11 drives and this scenario does not enable. + # + # The serving floor is the half a shared rule cannot state. Durable, + # scheduled and promise are claimed to be off this fault path because of + # where the code routes their namespaces, and that claim is worth + # asserting: a stall there would mean the status blob is not off the + # commit path the way AgentStatusFlusher describes. All three held + # 99.94-100.06% on the first run, so 50 catches a real regression without + # tripping on noise. + expect: + kind: partial-workload + silenced: [ephemeral] + quietFloorPercent: 50 + servingFloorPercent: 50 + # Recorded, not asserted. Expected to be larger than S14's: nothing is + # replaced, but three minutes of coalesced status writes and buffered + # commands all arrive at once when the connection returns. + recoveryBudgetSecs: 180 + + retryPolicy: + # Identical to the other three, and load-bearing here rather than + # incidental. Retrying transport errors once under the same idempotency + # key is the whole mechanism by which a stalled operation and its retry + # can both land, which is the question the run exists to answer. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S17 — Redis latency degradation (GOL-375) + # + # The fifth scenario on the storage driver, and the first that does not break + # anything. S18 took the Redis cache away; this leaves it reachable and makes + # it slow. That is a different question rather than a milder version of the + # same one: S18 asked whether the platform survives losing its worker-status + # store, and this asks whether it degrades or breaks when that store gets + # slower. "It went quiet" would be a failure here, not the expected result. + # + # GOL-375 asks for three things S18 could not give it — invocation latency + # under load, RSS peak, and pending-work evidence — and for a determination of + # whether worker-status writes block invocation completion or are best effort. + # S18 answered half of that: best effort for a resident durable agent, because + # AgentStatusFlusher takes the blob off the commit path, and blocking for + # anything crossing a lifecycle boundary. This run gets the other half, which + # is what that costs when the store is merely slow rather than gone. + # --------------------------------------------------------------------------- + - code: S17 + name: redis-cache-latency + enabled: true + + fault: + # Recorded for the archive rather than acted on: the workflow selects the + # manifest by scenario code and reads the Chaos Mesh kind out of the + # manifest itself. It compares this string only against `none`, + # `network-partition` and `pod-kill`, none of which this is. + kind: network-delay + target: worker-executor + # Every executor, as in the other four. A cache that is slow for only some + # executors is a routing fault wearing a cache's clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 300 + + phases: + # Same as the other four, so all five are comparable. + baselineSecs: 300 + # Longer than any of the others, and not for the reason S18's window is + # long. Nothing times out under a 500ms delay — an ephemeral operation + # goes from about 212ms to under a second, nowhere near the 120s attempt + # timeout — so there is no caller clock to outrun. The length is for the + # two things GOL-375 asks for that need time to develop: a memory series + # long enough to show whether the executor's dirty-status set grows, and + # a steady state rather than a transient. At the workflow's 15s sampling + # interval this is about 20 memory samples inside the window. + faultSecs: 300 + # As the others. + recoverySecs: 420 + + workload: + # Identical to S14, S16, S18 and S22, so the difference between the five + # results is the fault and not the population. + durableAgents: 200 + ephemeralAgents: 50 + # Zero for the same reason as the others: the scheduled stream comes from + # the block below. Setting both is refused at load time. + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # The scheduler keeps its own schema on Postgres, which this fault does + # not touch, so its claims and acknowledgements should be unaffected. S18 + # confirmed that: during-fault p99 was 2094ms against a 2099ms baseline. + # 240 as the other three, and it should not be approached. + leaseBudgetSecs: 240 + + storage: + # The same Redis endpoint S18 cuts. Both Aurora clusters stay reachable + # and unslowed for the whole run. + endpoint: master.golem-redis-dev.intsie.use1.cache.amazonaws.com + + expect: + # Neither of the rules the other four use. Under a delay nothing is + # supposed to fall silent, so silence is not available as evidence that + # the fault landed and time has to stand in for it. + kind: latency-degradation + + # `ephemeral` alone, for the reason S18 measured rather than predicted: + # its agents are created and torn down per operation, so every one + # crosses a lifecycle boundary, and that is one of the few things still + # reaching this cache synchronously. Under S18 it was the only stream + # that stopped. + slowed: [ephemeral] + + # Ephemeral's baseline median was 212ms in S18's run. The delay is 500ms + # applied to egress only (`direction: to`, which Chaos Mesh's webhook + # requires for netem against an external target), so one Redis round + # trip inside the operation puts it at about 712ms, or 3.4x. Two put it + # near 5.7x. + # + # 2.0 sits below the single-round-trip case with room to spare while + # still failing the run that matters: a netem rule that never applied + # leaves the stream at roughly 1.0x, and the report would otherwise be + # full of healthy numbers describing an experiment that never happened. + slowdownFloor: 2.0 + + # The other three, and the claim that makes the 6.59x above attributable + # to this cache rather than to a slower cluster. Their agents are + # resident and durable, so `AgentStatusFlusher` keeps the status blob off + # the commit path and nothing they do crosses Redis synchronously — and + # the first run bore that out exactly, at 61ms, 100ms and 91ms in all + # three windows. S15 makes the mirror claim with the two lists swapped. + steady: [durable, promise, scheduled] + + # Room for the spill of sharing executors and a runtime with a stream + # that is six times slower than usual, and nothing like the multiple a + # real dependency on this cache would produce. Measured at 1.00x. + steadyCeiling: 1.5 + + # A delay should cost time, not work. Throughput is not expected to move + # much: the per-stream in-flight budget is MAX_IN_FLIGHT/4 = 256, and at + # 33 operations per second and 712ms each the stream sits around 23 in + # flight, so nothing queues. A stream falling below half its baseline + # therefore means the added latency broke something rather than slowed + # it — a timeout fired, a pool drained — and that is worth failing on. + servingFloorPercent: 50 + + # Recorded, not asserted. Expected to be near zero: nothing was ever + # unreachable, so there is no reconnect to wait for and no backlog to + # drain, only the last few in-flight operations finishing at their slower + # pace. + recoveryBudgetSecs: 60 + + retryPolicy: + # Identical to the other four. Not load-bearing here the way it is in S18, + # because nothing should time out and so nothing should retry — but a + # retry count above zero would itself be a finding, and leaving the policy + # the same is what makes that comparable. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S15 — key-value PostgreSQL latency degradation (GOL-374) + # + # S17 with the two halves of the key-value layer swapped. That fault slows the + # Redis cache and this one slows the Aurora cluster behind it, and because + # `NamespaceRoutedKeyValueStorage` splits the namespaces between them, the two + # runs should move opposite sets of streams. Reading them side by side is the + # point: each is the other's control. + # + # ── Which streams actually reach this cluster ────────────────────────────── + # + # Worth spelling out, because "the workload is promise- and schedule-heavy" + # undersells it and the durable stream is the surprise. + # + # * durable — `AgentStatusFlusher::on_status_changed` updates the + # `RunningWorkers` recovery index *synchronously on the hot path*, and + # only when the tracking predicate flips. An idle agent invoked once goes + # Idle → Running → Idle, so the predicate flips twice and the invocation + # waits on two writes to this cluster. The status blob itself is off the + # commit path; the recovery index deliberately is not. + # * promise — `create` is one `set_if_not_exists` and `complete` is an + # `exists` followed by another, all in the `Promise` namespace, which + # routes here. + # * scheduled — the emitter's `schedule_poll_at` registers into the + # scheduler's own schema, which lives on this same cluster under + # `GOLEM__SCHEDULER_STORAGE__CONFIG__HOST`. + # * ephemeral — reaches none of it. `on_status_changed` returns immediately + # for an ephemeral agent, and its status goes to the Redis cache, so this + # is the one stream the delay should not touch. Under S17 it was the only + # one that did. + # + # ── What the first run measured, which was not what it predicted ────────── + # + # Run 33210769609. The prediction here was 17x on durable and promise and 6x + # on scheduled, from counting round trips, with a note that the connection + # pools would fit "neither by much". The measurement was 644x, 939x and + # 1640x: durable went from 61ms to 39s, promise from 92ms to 86s, scheduled + # from 62ms to 102s. A 500ms delay came out three orders of magnitude larger + # at the caller. + # + # So this is not a degradation scenario in practice. Counting round trips + # gives the cost of one operation running alone; it says nothing about what + # happens when every operation is doing the same thing to a pool of 64 + # connections, and queueing is the whole answer. Nothing was lost — every key + # exactly once, every schedule fired once, 300/300 agents consistent, every + # stream serving again within 4s — but the platform did not stay usable. + # + # Two predictions that were wrong in an informative direction: + # + # * No executor was lost. `PoolTimedOut` never reached the panic, so the + # S16 finding was not reproduced from latency alone. The queue built in + # front of the pool rather than inside its acquire timeout. + # * `ephemeral`, which reaches none of this cluster, ran 235x slower. 46 + # seconds is ninety 500ms round trips and the stream makes none, so it was + # queueing behind the three streams that do. Whatever gate that is, it is + # not the per-account concurrency semaphore — golem-dev sets + # `MAX_CONCURRENT_AGENTS_PER_EXECUTOR` to the unlimited sentinel, so that + # path is bypassed. See the runbook. + # --------------------------------------------------------------------------- + - code: S15 + name: keyvalue-postgres-latency + enabled: true + + fault: + # As S17. Recorded for the archive rather than acted on: the workflow + # picks the manifest by scenario code and reads the Chaos Mesh kind out of + # the manifest itself. + kind: network-delay + target: worker-executor + # Every executor. A store that is slow for only some of them is a routing + # fault wearing a database's clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 300 + + phases: + # S17's, unchanged, so the two delays differ only in what they aim at. + baselineSecs: 300 + # 300s for S17's reasons and one more. Nothing here should approach the + # 120s attempt timeout, so there is no caller clock to outrun; the length + # is for a memory series long enough to have a slope and for a steady + # state rather than a transient. The one more is the pool arithmetic + # above: a pool that is going to saturate does it by filling, and filling + # takes time that a 60s window does not give it. + faultSecs: 300 + recoverySecs: 420 + + workload: + # Identical to the other five, deliberately, even though GOL-374 asks for + # "promise- and schedule-heavy" work and a reweighted population would + # press the pools harder. Two of the six storage runs are a matched pair + # and comparability is what makes the pair worth anything; and the durable + # stream turns out to be the heaviest user of this cluster anyway, at two + # synchronous writes per invocation. Deliberately overloading the pool is + # a different experiment — S22 is to S16 what that would be to this — and + # it should be its own scenario rather than a thumb on this one's scale. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # As the others. Unlike S17 the scheduler is *inside* this delay, so its + # claims and acknowledgements each gain a round trip — but 240s of lease + # against a claim that costs milliseconds plus 500ms is not a number this + # run can approach, and a lease expiry here would be a real finding rather + # than a threshold that was set too tight. + leaseBudgetSecs: 240 + + storage: + # The key-value Aurora cluster, which is also where the scheduler keeps + # its schema. The same endpoint S16 and S22 cut. Neither Redis nor the + # indexed-oplog cluster is touched: slowing a second store would leave + # every finding unattributable. + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + # As S17. Under a delay nothing is supposed to fall silent, so silence + # is not available as evidence that the fault landed and time has to + # stand in for it. + kind: latency-degradation + + # The three streams that reach this cluster synchronously, per the + # walk-through above. Measured at 644x, 939x and 1640x — see the note + # above for why the round-trip arithmetic underestimated that by two + # orders of magnitude. + slowed: [durable, promise, scheduled] + + # 2.0, as S17, and for the same reason rather than for the arithmetic + # above: the floor exists to fail a run where the netem rule never + # applied, which leaves every stream near 1.0x. Setting it near the + # predicted 6x would instead be a second, unannounced assertion that the + # prediction is right, and the prediction is the thing under test. + slowdownFloor: 2.0 + + # The control, and the mirror image of S17's list. An ephemeral agent + # never enters the recovery index and its status goes to Redis, so if + # this stream slows with the others, the delay reached something wider + # than this cluster and the run cannot say what it measured. + steady: [ephemeral] + + # As S17. Ephemeral is the noisiest stream in the suite at rest — its + # p99 runs to seven times its median — but the check is on medians, and + # the median has held at 196-212ms across every storage run so far. + # + # The first run broke this at 235x, and deliberately left as it is: the + # ceiling did its job. Raising it to accommodate the result would turn + # the one number that caught the coupling into a number that describes + # it. + steadyCeiling: 1.5 + + # A delay should cost time, not work. Held at 50 for comparability, but + # the first run showed this floor cannot be read on its own once the + # in-flight budget saturates: throughput is concurrency over latency, so + # a stream 900x slower cannot hold its rate however healthy it is. + # Promise came in at 11.9% of baseline while carrying a hundred times + # more work in flight than at rest. `outage.rs` now suppresses the + # finding when the stream's own slowdown accounts for the shortfall, + # which is the only case where the two floors were measuring one fact. + servingFloorPercent: 50 + + # Recorded, not asserted, and expected to be near zero: nothing was ever + # unreachable, so there is no reconnect to wait for. If an executor was + # lost to the pool timeout this will be the number that shows it, as the + # replacement has to reload its agents before it can serve. + recoveryBudgetSecs: 60 + + retryPolicy: + # Identical to the other five. Nothing should time out and so nothing + # should retry; a retry count above zero is itself a finding. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S15A / S15B / S15C — the S15 elimination series + # + # S15's first run left one thing unexplained. `ephemeral` slowed 235x under a + # delay aimed at a cluster it never touches, and the search for a shared gate + # inside the executor came up empty: worker-service was exonerated by the + # traces, agent creation ran in single-digit milliseconds, memory and + # filesystem pools had headroom, the per-account concurrency semaphore is + # bypassed on golem-dev, the tokio global queue was flat at zero with the busy + # ratio *falling*, and neither the oplog cluster nor Redis moved. Everything + # was parked, nothing was competing, and no mechanism in the source accounts + # for it. + # + # So the next step is elimination rather than more forensics: hold the fault + # and the ephemeral stream fixed, and add one stream at a time until ephemeral + # starts to suffer. Whichever addition does it names the interaction, and the + # code reading has somewhere to start. + # + # S15A ephemeral the control: nothing should move + # S15B ephemeral + durable durable is the heaviest user + # S15C ephemeral + durable + promise + # S15 ephemeral + durable + promise + scheduled already run: 235x + # + # ── Why ratePerSec differs between them, which is not a knob being fiddled ── + # + # `workload::start` divides `ratePerSec` evenly across the streams that have + # agents, so a constant here would give ephemeral 100/s alone and 33/s in a + # threesome, and the comparison would be between three different ephemeral + # workloads rather than between three fault environments. Each entry sets the + # rate that holds ephemeral at the ~34/s S15 measured it at. + # + # One confound cannot be removed the same way. `MAX_IN_FLIGHT / active.len()` + # gives ephemeral a budget of 1024, 512 and 341 across the three, against 256 + # in S15, and that ceiling is a constant in the driver rather than a suite + # setting. It bounds throughput, not latency, and latency is what this series + # reads, so it is recorded rather than corrected. + # + # ── Why the scheduled block is present with zero targets ─────────────────── + # + # `storage_fault::run` requires the block. Zero targets means no registrations + # and no fires, so the scheduled stream is not driven and `drives_stream` + # reports it as absent, which is what keeps it out of the expectations below. + # --------------------------------------------------------------------------- + - code: S15A + name: keyvalue-postgres-latency-ephemeral-only + enabled: true + + fault: + kind: network-delay + target: worker-executor + mode: all + # The same file S15 applies, not a copy of it. Four manifests that must + # stay identical by hand is four chances for the fault to diverge from the + # run it is being compared against. + manifest: networkchaos-s15.yaml + durationSecs: 300 + + phases: + # S15's exactly. Shorter windows would be cheaper and the effect is large + # enough to survive them, but the whole value of this series is that its + # ephemeral numbers are comparable with S15's, and a median taken over a + # different window length is a different measurement. + baselineSecs: 300 + faultSecs: 300 + recoverySecs: 420 + + workload: + durableAgents: 0 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 0 + quotaAgents: 0 + # One active stream, so ephemeral gets all of it. + ratePerSec: 34 + + scheduled: + targets: 0 + intervalMillis: 2000 + leadSecs: 10 + leaseBudgetSecs: 240 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + kind: latency-degradation + + # Empty, and the only entry in the suite that is. Nothing this workload + # drives reaches the delayed cluster, so there is no stream whose + # slowdown could evidence the fault. That makes this a control rather + # than an experiment, and it means the run cannot fail on + # `slowdown-not-observed`: it cannot tell a delay that landed and did + # not matter from one that never applied. + # + # The evidence has to come from outside the workload, and it does. The + # scheduler polls its own schema on this cluster whether or not anything + # is scheduled, so `db_success_seconds{svc="scheduler_storage"}` moves + # under the fault regardless of what the workload does. Check it before + # believing a clean result here. + slowed: [] + + # Required by the schema and inert here, because the floor is only ever + # applied to the streams `slowed` names and it names none. Left at the + # series' value rather than at something arbitrary, so a later edit that + # gives this run a slowed stream inherits a floor that means what it + # does everywhere else. + slowdownFloor: 2.0 + + # The whole question. If ephemeral holds its 196ms median with nothing + # else running, then the 235x in S15 came from the company it was + # keeping and not from the fault. + steady: [ephemeral] + steadyCeiling: 1.5 + + servingFloorPercent: 50 + + recoveryBudgetSecs: 60 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + - code: S15B + name: keyvalue-postgres-latency-plus-durable + enabled: true + + fault: + kind: network-delay + target: worker-executor + mode: all + manifest: networkchaos-s15.yaml + durationSecs: 300 + + phases: + baselineSecs: 300 + faultSecs: 300 + recoverySecs: 420 + + workload: + # S15's durable population, unchanged, so the load this adds is the load + # S15 had rather than a scaled-down version of it. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 0 + quotaAgents: 0 + # Two active streams, so 34/s each. + ratePerSec: 68 + + scheduled: + targets: 0 + intervalMillis: 2000 + leadSecs: 10 + leaseBudgetSecs: 240 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + kind: latency-degradation + # Durable waits on two synchronous writes to this cluster per + # invocation, so it should move a long way. S15 measured 644x. + slowed: [durable] + slowdownFloor: 2.0 + # And this is the reading that matters. If ephemeral moves here, durable + # alone is enough to do it and the interaction is between those two. + steady: [ephemeral] + steadyCeiling: 1.5 + servingFloorPercent: 50 + + recoveryBudgetSecs: 60 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + - code: S15C + name: keyvalue-postgres-latency-plus-promise + enabled: true + + fault: + kind: network-delay + target: worker-executor + mode: all + manifest: networkchaos-s15.yaml + durationSecs: 300 + + phases: + baselineSecs: 300 + faultSecs: 300 + recoverySecs: 420 + + workload: + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + # Three active streams, so 33.3/s each — the rate S15 ran them at. + ratePerSec: 100 + + scheduled: + targets: 0 + intervalMillis: 2000 + leadSecs: 10 + leaseBudgetSecs: 240 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + kind: latency-degradation + # Promise makes the most calls of any stream: three of its own, plus the + # two its durable agent pays on each of the two invocations an operation + # takes. S15 measured 939x. + slowed: [durable, promise] + slowdownFloor: 2.0 + steady: [ephemeral] + steadyCeiling: 1.5 + servingFloorPercent: 50 + + recoveryBudgetSecs: 60 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index af889b5156..bac31f25be 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -592,6 +592,21 @@ async fn run_chaos( ChaosScenarioArg::S5 => chaos::ScenarioCode::S5, ChaosScenarioArg::S12 => chaos::ScenarioCode::S12, ChaosScenarioArg::S13 => chaos::ScenarioCode::S13, + ChaosScenarioArg::S10 => chaos::ScenarioCode::S10, + ChaosScenarioArg::S11 => chaos::ScenarioCode::S11, + ChaosScenarioArg::S3 => chaos::ScenarioCode::S3, + ChaosScenarioArg::S7 => chaos::ScenarioCode::S7, + ChaosScenarioArg::S6 => chaos::ScenarioCode::S6, + ChaosScenarioArg::S9 => chaos::ScenarioCode::S9, + ChaosScenarioArg::S16 => chaos::ScenarioCode::S16, + ChaosScenarioArg::S22 => chaos::ScenarioCode::S22, + ChaosScenarioArg::S14 => chaos::ScenarioCode::S14, + ChaosScenarioArg::S18 => chaos::ScenarioCode::S18, + ChaosScenarioArg::S17 => chaos::ScenarioCode::S17, + ChaosScenarioArg::S15 => chaos::ScenarioCode::S15, + ChaosScenarioArg::S15A => chaos::ScenarioCode::S15A, + ChaosScenarioArg::S15B => chaos::ScenarioCode::S15B, + ChaosScenarioArg::S15C => chaos::ScenarioCode::S15C, }; let config = suite .scenario(code, allow_disabled) @@ -623,6 +638,38 @@ async fn run_chaos( chaos::ScenarioCode::S13 => { chaos::scenarios::s13::run(&config, &manifest, &deps, &signals, &outputs).await } + chaos::ScenarioCode::S10 => { + chaos::scenarios::s10::run(&config, &manifest, &deps, &signals, &outputs).await + } + chaos::ScenarioCode::S11 => { + chaos::scenarios::s11::run(&config, &manifest, &deps, &signals, &outputs).await + } + chaos::ScenarioCode::S3 => { + chaos::scenarios::s3::run(&config, &manifest, &deps, &signals, &outputs).await + } + chaos::ScenarioCode::S7 => { + chaos::scenarios::s7::run(&config, &manifest, &deps, &signals, &outputs).await + } + chaos::ScenarioCode::S6 => { + chaos::scenarios::s6::run(&config, &manifest, &deps, &signals, &outputs).await + } + chaos::ScenarioCode::S9 => { + chaos::scenarios::s9::run(&config, &manifest, &deps, &signals, &outputs).await + } + code @ (chaos::ScenarioCode::S14 + | chaos::ScenarioCode::S15 + | chaos::ScenarioCode::S15A + | chaos::ScenarioCode::S15B + | chaos::ScenarioCode::S15C + | chaos::ScenarioCode::S16 + | chaos::ScenarioCode::S17 + | chaos::ScenarioCode::S18 + | chaos::ScenarioCode::S22) => { + chaos::scenarios::storage_fault::run( + code, &config, &manifest, &deps, &signals, &outputs, + ) + .await + } }; deps.kill_all().await; @@ -855,7 +902,7 @@ async fn run_density( // Emit the cell result as a single-result suite so the JSON shape // matches cloud-perf (BenchmarkSuiteResultCollection), and the - // buildspec can upload it directly to S3. + // buildspec can upload it directly to golem-bench-results. let mut suite_result = BenchmarkSuiteResult::new(&format!("density-{section}")); suite_result.add(result); if let Some(run_id) = cloud_bench_run_id() { diff --git a/integration-tests/src/benchmarks/density/agent.rs b/integration-tests/src/benchmarks/density/agent.rs index 96dba8f7bc..0b3fcc28fd 100644 --- a/integration-tests/src/benchmarks/density/agent.rs +++ b/integration-tests/src/benchmarks/density/agent.rs @@ -185,7 +185,8 @@ pub struct CellConfig { } impl CellConfig { - /// The cell's name, used in S3 paths and result identifiers. Encodes the + /// The cell's name, used in `golem-bench-results` keys and result identifiers. + /// Encodes the /// full axis set in human-readable terms (no cryptic single letters). pub fn cell_name(&self) -> String { let mut parts = vec![ diff --git a/integration-tests/src/benchmarks/density/prep.rs b/integration-tests/src/benchmarks/density/prep.rs index 209076716e..f5e208b473 100644 --- a/integration-tests/src/benchmarks/density/prep.rs +++ b/integration-tests/src/benchmarks/density/prep.rs @@ -20,7 +20,8 @@ //! //! The buildspec runs prep exactly once at suite start (against a freshly-wiped //! cluster) and writes the resulting [`PrepManifest`] to a file that is -//! uploaded to S3 and passed to every per-cell invocation via +//! uploaded to the `golem-bench-results` bucket and passed to every per-cell +//! invocation via //! `--prep-manifest`. The manifest carries the account token and all component //! IDs so per-cell invocations need no by-name lookup and no re-tokenization — //! this is also the resume mechanism: a resumed run reloads the same manifest. diff --git a/integration-tests/src/chaos/deletions.rs b/integration-tests/src/chaos/deletions.rs new file mode 100644 index 0000000000..ab5a974cdc --- /dev/null +++ b/integration-tests/src/chaos/deletions.rs @@ -0,0 +1,384 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Agents that are built up and then deleted outright (GOL-372). +//! +//! One emitter per agent slot, running rounds. A round is +//! `increments_per_round` increments followed by one `delete`. Then the slot is +//! used again: invoking a deleted agent id creates a **new** agent, so the next +//! round's first increment returns `1` if the deletion took and `V + 1` if the +//! old agent is still there. +//! +//! That is the same probe [`crate::chaos::reverts`] uses and it works for the +//! same reason: the round's last increment reports the value, so there are +//! exactly two legal answers afterwards and nothing between them. +//! +//! ### The delete must never be retried +//! +//! `delete_worker_internal` in the executor starts with `get_latest_metadata` +//! and returns `worker_not_found` when there is nothing there. So deleting an +//! agent twice does **not** return success twice: the second call is an error. +//! +//! A delete whose response was lost but which actually landed would therefore, +//! on retry, come back as a refusal — and the driver would record "the platform +//! refused, and the agent is gone anyway", which is one of the violations this +//! scenario exists to detect. The retry is switched off for the delete alone, +//! exactly as it is for a revert, and left on for the increments around it. +//! +//! ### What the kill is aimed at +//! +//! Deleting is four steps: interrupt the running worker, `start_deleting`, +//! remove it from the worker service, remove it from the active set. Only the +//! third is durable. `start_deleting` exists to stop a background status flush +//! from — in the executor's own words — "resurrecting the cached status" after +//! the removal, so the happy path is already defended against exactly the thing +//! this scenario is named for. The question is whether that defence survives the +//! pod dying between the mark and the removal. + +use crate::chaos::history::{Outcome, Stream}; +use crate::chaos::workload::{self, WorkloadContext}; +use crate::chaos::{DeleteConfig, RetryPolicy}; +use chrono::{DateTime, Utc}; +use golem_test_framework::dsl::TestDsl; +use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::task::JoinSet; +use tracing::info; + +/// The counter a freshly created agent carries. +/// +/// Named rather than written as `0` at the comparison, because it is the whole +/// definition of "the deletion took": an agent id that was deleted and then +/// invoked again is a *new* agent, and a new counter starts from nothing. +pub const COUNTER_OF_A_NEW_AGENT: u64 = 0; + +/// One round, as the driver observed it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeleteRound { + pub agent: String, + pub round: u32, + /// The value the round's last increment returned, so what the agent was + /// worth immediately before it was deleted. `None` when an increment did + /// not answer, which leaves the round unjudgeable rather than failed. + pub before_delete: Option, + /// What the delete call itself returned. + pub outcome: Outcome, + pub submitted_at: DateTime, + pub completed_at: Option>, + /// Whether the platform refused this delete by saying the agent was not + /// there, as opposed to refusing it for any other reason. + /// + /// Load-bearing rather than diagnostic. Deleting is not idempotent, and + /// worker-service retries a call whose executor became unreachable — so a + /// delete that *succeeded* on a pod that then died comes back to the caller + /// as not-found. Without this flag that reads as "the platform refused and + /// the agent is gone anyway", which is one of this scenario's violations. + #[serde(default)] + pub rejected_as_not_found: bool, + /// The counter the deletion left behind: `0` if the id came back as a new + /// agent, `before_delete` if the old one is still there. + /// + /// One meaning, whichever probe produced it, which is the point. A round is + /// normally probed by the *next* round's first increment, and an increment + /// reports the counter it just raised — so the value it leaves behind is + /// that minus one. The last round of a run has no increment after it and is + /// closed by a plain read instead, which reports the counter directly. + /// Storing the increment's own return would make those two probes disagree + /// by one, and only ever on the last round of each slot. That is exactly + /// the bug the first S6 run reported 125 times. + pub observed_after: Option, +} + +/// The counter an operation left behind, given what the *next* increment +/// reported. +/// +/// One line, and it earns a name because getting it wrong is invisible. The +/// account judges `observed_after` as the counter a delete left behind, and the +/// two probes that produce it disagree by one: an increment reports the counter +/// it just raised, a plain read reports the counter itself. The first S6 run +/// stored the increment's own return and reported 125 partial-state findings — +/// exactly the slots whose last round was closed by a read. +pub fn counter_left_by(increment_returned: u64) -> u64 { + increment_returned.saturating_sub(1) +} + +/// A running deletion workload. +pub struct DeleteHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, + rounds: Arc>>, +} + +impl DeleteHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + pub fn rounds(&self) -> Vec { + self.rounds.lock().map(|r| r.clone()).unwrap_or_default() + } + + pub async fn stop(mut self) -> Vec { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + let rounds = self.rounds(); + info!( + "Chaos delete workload stopped after {} operations across {} rounds", + self.submitted(), + rounds.len() + ); + rounds + } +} + +/// The agent slots a run of `count` emitters drives, in index order. +pub fn agent_names(ctx: &WorkloadContext, count: u32) -> Vec { + (0..count) + .map(|index| ctx.agent_name(Stream::Delete, index)) + .collect() +} + +/// Builds one agent up, deletes it, and checks it came back new. +/// +/// Run once before the baseline, against a throwaway id. It exists because the +/// first S11 run did not have its equivalent: a scenario whose premise is wrong +/// spends its whole baseline before the numbers say so, and the maintenance +/// window is gone. If deletion does not behave the way this whole account +/// assumes, this fails in seconds with the platform's own error. +pub async fn smoke_round(ctx: &WorkloadContext, config: &DeleteConfig) -> anyhow::Result<()> { + let agent = format!("{}-delete-smoke", ctx.key_prefix); + let mut value = 0; + for step in 0..config.increments_per_round.max(1) { + value = workload::increment_counter( + ctx, + Stream::Delete, + &agent, + ctx.idempotency_key(&agent, step as u64), + ) + .await + .value + .map(u64::from) + .ok_or_else(|| { + anyhow::anyhow!("smoke round: increment {step} on {agent} did not answer") + })?; + } + + let agent_id = workload::counter_agent_id(ctx, &agent); + ctx.user + .delete_worker(&agent_id) + .await + .map_err(|e| anyhow::anyhow!("smoke round: deleting {agent} failed: {e:#}"))?; + + let after = workload::increment_counter( + ctx, + Stream::Delete, + &agent, + ctx.idempotency_key(&agent, u64::from(config.increments_per_round) + 1), + ) + .await + .value + .map(u64::from) + .ok_or_else(|| anyhow::anyhow!("smoke round: {agent} did not answer after being deleted"))?; + + if after != COUNTER_OF_A_NEW_AGENT + 1 { + anyhow::bail!( + "smoke round: {agent} was worth {value}, was deleted, and its next increment \ + returned {after} rather than {}. Deleting an agent does not behave the way this \ + scenario's whole account assumes, so the run would report a resurrection on every \ + round.", + COUNTER_OF_A_NEW_AGENT + 1 + ); + } + info!("S6: smoke round passed — a deleted agent came back as a new one"); + Ok(()) +} + +/// Starts one emitter per agent slot. +pub fn start(ctx: WorkloadContext, config: &DeleteConfig) -> DeleteHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let rounds: Arc>> = Arc::new(Mutex::new(Vec::new())); + let mut tasks = JoinSet::new(); + + info!( + "Chaos delete workload starting: {} emitters, {} increments then a delete per round, \ + {:?} between rounds", + config.agents, + config.increments_per_round, + config.interval() + ); + + for index in 0..config.agents { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + let rounds = rounds.clone(); + let config = config.clone(); + + tasks.spawn(async move { + let agent = ctx.agent_name(Stream::Delete, index); + let mut round: u32 = 0; + let mut pending: Option = None; + + while stop.load(Ordering::Relaxed) == 0 { + let mut last_value = None; + let mut all_answered = true; + for step in 0..config.increments_per_round { + if stop.load(Ordering::Relaxed) != 0 { + break; + } + let seq = round as u64 * config.increments_per_round as u64 + step as u64; + submitted.fetch_add(1, Ordering::Relaxed); + let value = workload::increment_counter( + &ctx, + Stream::Delete, + &agent, + ctx.idempotency_key(&agent, seq), + ) + .await + .value + .map(u64::from); + + // The first increment of a round says what the previous + // round's delete left behind. + if let Some(slot) = pending.take() + && let Some(observed) = value + && let Ok(mut rounds) = rounds.lock() + && let Some(entry) = rounds.get_mut(slot) + { + entry.observed_after = Some(counter_left_by(observed)); + } + + match value { + Some(v) => last_value = Some(v), + None => all_answered = false, + } + } + if stop.load(Ordering::Relaxed) != 0 { + break; + } + + let submitted_at = Utc::now(); + submitted.fetch_add(1, Ordering::Relaxed); + let (outcome, rejected_as_not_found) = delete_once(&ctx, &agent, round).await; + + if let Ok(mut rounds) = rounds.lock() { + rounds.push(DeleteRound { + agent: agent.clone(), + round, + before_delete: all_answered.then_some(last_value).flatten(), + outcome, + rejected_as_not_found, + submitted_at, + completed_at: Some(Utc::now()), + observed_after: None, + }); + pending = Some(rounds.len() - 1); + } + + round += 1; + if stop.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(config.interval()).await; + } + } + }); + } + + DeleteHandle { + stop, + tasks, + submitted, + rounds, + } +} + +/// The error code the platform returns when there is nothing to delete. +const AGENT_NOT_FOUND: &str = "AGENT_NOT_FOUND"; + +/// One delete, with retries switched off. See the module docs. +/// +/// Returns the outcome and whether a refusal was specifically "no such agent", +/// which is the signature of a delete that had already taken effect. +async fn delete_once(ctx: &WorkloadContext, agent: &str, round: u32) -> (Outcome, bool) { + let mut once = ctx.clone(); + once.retry = RetryPolicy { + transport_only: true, + max_retries: 0, + delay_secs: 0, + }; + + let key = format!("{agent}-delete-{round:08}"); + let agent_id = workload::counter_agent_id(&once, agent); + let ctx2 = once.clone(); + + let result = workload::run_operation( + &once, + Stream::Delete, + agent.to_string(), + "delete", + key, + |_| { + let ctx = ctx2.clone(); + let agent_id = agent_id.clone(); + async move { + ctx.user.delete_worker(&agent_id).await?; + Ok(None) + } + }, + ) + .await; + + let not_found = result + .error + .as_deref() + .is_some_and(|e| e.contains(AGENT_NOT_FOUND)); + (result.outcome, not_found) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + /// The conversion the first S6 run got wrong, and which no test covered + /// because it lived in a spawned task rather than in the account. + /// + /// `observed_after` means the counter the delete left behind. An increment + /// reports the counter it just raised, so it is one more than that; a plain + /// read reports the counter directly and needs no conversion. Both probes + /// have to land on the same scale or every slot's last round — the only one + /// closed by a read — reads as a partial state. + #[test] + fn the_two_probes_agree_on_what_a_delete_left_behind() { + // A deleted agent: the next increment creates it and reports 1. + assert_eq!(counter_left_by(1), COUNTER_OF_A_NEW_AGENT); + // A survivor worth 3: the next increment reports 4. + assert_eq!(counter_left_by(4), 3); + // A plain read of the same two cases reports 0 and 3 with no + // conversion, which is what `close_last_rounds` stores. + for counter in [COUNTER_OF_A_NEW_AGENT, 3] { + assert_eq!(counter_left_by(counter + 1), counter); + } + } + + /// An increment that somehow reported nothing must not underflow into a + /// value the account would read as a survivor. + #[test] + fn a_zero_increment_does_not_wrap_around() { + assert_eq!(counter_left_by(0), 0); + } +} diff --git a/integration-tests/src/chaos/fires.rs b/integration-tests/src/chaos/fires.rs new file mode 100644 index 0000000000..4d2ee1b13b --- /dev/null +++ b/integration-tests/src/chaos/fires.rs @@ -0,0 +1,965 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The scheduled-fire account (GOL-378). +//! +//! Every other oracle in this suite reduces to arithmetic over counts: the +//! driver submitted N, the durable state says M, and the gap between them is +//! the finding. That works because an increment is interchangeable with every +//! other increment. A scheduled action is not. The question S10 asks is whether +//! *this* action, claimed by an executor that then died, fired — once — and how +//! long the lease took to hand it to somebody else. +//! +//! So the target agent records a token per fire rather than a tally, and this +//! module pairs those tokens against the registrations the driver made. Pairing +//! is what makes the verdicts facts about a named action rather than a +//! judgement about a distribution, which is the same reason S8 probes keys +//! individually. +//! +//! ## What fails a run, and what only gets reported +//! +//! Three things fail it, and all three are statements about one token: +//! +//! - a **confirmed** registration whose action never fired +//! - a token that fired **more than once** +//! - a registration the platform **refused** that fired anyway +//! +//! Everything else is reported. In particular an *indeterminate* registration +//! that never fired is not a finding: the driver never learned whether the +//! registration landed, so an action that never fired is one of the two +//! legitimate answers. Those are counted, so a clean verdict over many of them +//! reads as weaker than a clean verdict with none. +//! +//! The same care applies to the read itself. A target whose fire log could not +//! be read, or whose log hit the component's cap, cannot testify about its own +//! registrations — those become unverifiable rather than lost. Reporting an +//! unreadable agent as lost work would turn a failed read into a correctness +//! defect, which is the exact mistake this suite exists to avoid. +//! +//! ## Delay, and why it is grouped the way it is +//! +//! Fire delay is `observed - scheduled`: how far past its due time the platform +//! actually ran the action. It is grouped two ways at once, because either +//! alone is misleading. +//! +//! By **window**, because an action due while the executor was gone is the only +//! one whose delay says anything about recovery. By **group**, because on a +//! two-executor cluster roughly half the targets were never on the pod that +//! died: mixing them in drags the percentile down until a lease recovery that +//! took its full TTL looks like a healthy p99. +//! +//! Delays are measured across two clocks — the driver mints the due time, the +//! executor stamps the fire — so a small negative delay is skew rather than an +//! action that fired early. `minDelayMs` is reported per group so that skew is +//! visible instead of silently folded into the percentiles. +//! +//! ### Actions that were late before the scheduler saw them +//! +//! The due time is minted *before* the registering invocation goes out, so a +//! registration that takes longer than the lead to complete describes an action +//! that was already overdue the moment it was accepted. The platform runs it +//! immediately and correctly, and the arithmetic still reports it as tens of +//! seconds late. +//! +//! This is not hypothetical: killing an executor stalls the registrations that +//! were in flight to it, and a stall longer than the lead puts every one of +//! those actions into the delay cells as tens of seconds late even though each +//! fired the instant it was accepted. Worse, it lands in *both* groups, because +//! emitters are spread across executors independently of their targets — so an +//! untouched control group can be made to look exactly as damaged as the killed +//! executor's. +//! +//! So they are separated. A fire whose registration completed after its due time +//! is counted as `overdueOnArrival` and kept out of the delay cells, because the +//! delay cells answer "how late was the scheduler" and this is the answer to +//! "how late was the client". Both are reported; neither is allowed to +//! impersonate the other. + +use crate::chaos::history::{FireRecord, OperationRecord, Outcome, Stream, TargetFireLog}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, TimeDelta, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +/// Ceiling on how many findings the report carries. +/// +/// A scenario that lost every action would otherwise produce tens of thousands +/// of them and an artifact nobody can open. The count is reported separately, +/// so truncation is stated rather than inferred from a suspiciously round +/// number of findings. +const MAX_FINDINGS: usize = 200; + +/// The fault window, as the workflow reported it. See +/// [`crate::chaos::split::FaultWindow`]. +pub type FaultWindow = crate::chaos::split::FaultWindow; + +/// Which side of the fault an action was due on. +/// +/// An alias rather than a type of its own: every scenario that reports against a +/// fault window asks the same question, and `fireWindow` is the key an archived +/// S10 result already carries. +pub type FireWindow = crate::chaos::split::Window; + +/// Whether an action's target was on the executor the fault killed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum TargetGroup { + /// Owned by the killed executor when the driver signalled readiness. + OnKilledExecutor, + /// Owned by an executor the fault left alone: the run's own control group. + Elsewhere, +} + +impl TargetGroup { + pub fn as_str(self) -> &'static str { + match self { + TargetGroup::OnKilledExecutor => "on-killed-executor", + TargetGroup::Elsewhere => "elsewhere", + } + } +} + +impl std::fmt::Display for TargetGroup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// What a token did that it should not have. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum FireViolation { + /// A registration the platform accepted whose action never ran. + NeverFired, + /// One registration, two or more fires. + FiredMoreThanOnce, + /// A registration the platform definitively refused, whose action ran + /// anyway. + FiredDespiteRejection, +} + +impl FireViolation { + pub fn as_str(self) -> &'static str { + match self { + FireViolation::NeverFired => "never-fired", + FireViolation::FiredMoreThanOnce => "fired-more-than-once", + FireViolation::FiredDespiteRejection => "fired-despite-rejection", + } + } +} + +impl std::fmt::Display for FireViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, against one token. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FireFinding { + pub violation: FireViolation, + pub token: String, + pub agent: String, + pub window: FireWindow, + pub detail: String, +} + +/// Fire delay for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FireDelayStats { + pub group: TargetGroup, + pub window: FireWindow, + /// Percentiles over delays clamped at zero, so skew cannot flatter them. + pub delay: LatencyStats, + /// The most negative delay seen, which is the clock skew between the driver + /// and the executor rather than an action that fired early. + pub min_delay_ms: i64, + /// Fires whose delay exceeded the configured lease budget. + pub over_budget: u64, +} + +/// The scheduled-fire account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScheduleFireReport { + /// What a lease recovery is allowed to cost, from the suite YAML. Recorded + /// so a percentile in an archived result can be read years later against + /// the number it was judged by rather than against today's config. + pub lease_budget_ms: u64, + pub registrations_confirmed: u64, + pub registrations_indeterminate: u64, + pub registrations_rejected: u64, + /// Fires the targets recorded, including any whose token is unknown. + pub fires_recorded: u64, + /// Accepted registrations paired with exactly one fire. + pub fired_once: u64, + /// Registrations the driver was never sure of that fired anyway — doubt the + /// platform resolved in its own favour. + pub indeterminate_that_fired: u64, + /// Registrations the driver was never sure of that never fired. Not a + /// finding: the registration may never have landed. + pub inconclusive: u64, + /// Registrations whose target could not testify, because its log was + /// unreadable or truncated. + pub unverifiable: u64, + /// Fires whose token no registration claims. Zero on a healthy run: agent + /// names carry the run nonce, so nothing from an earlier run can appear. + pub unknown_tokens: u64, + /// Targets the read-back could not reach at all. + pub targets_unreadable: Vec, + /// Targets whose fire log hit the component's cap. + pub targets_truncated: Vec, + pub delay: Vec, + /// Fires whose registration only landed after the action was already due, so + /// the platform could not have run them on time whatever the scheduler did. + /// Held out of [`Self::delay`] — see the module docs. + pub overdue_on_arrival: u64, + /// How late those were, for the record. This is client-side registration + /// latency wearing a scheduler's clothes. + pub overdue_delay: LatencyStats, + pub findings: Vec, + /// Findings past [`MAX_FINDINGS`], which the report drops rather than + /// carries. Non-zero means `findings` is a sample. + pub findings_omitted: u64, +} + +impl ScheduleFireReport { + /// Pairs registrations against fires. + /// + /// `records` is the whole history; only the scheduled stream is considered. + /// `lead` is how far ahead registrations were made, used to say when an + /// action that never fired was due — the fire log is the only place the + /// exact due time survives, and an action that never fired left no entry. + pub fn build( + records: &[OperationRecord], + logs: &[TargetFireLog], + lead: Duration, + fault: Option, + killed_targets: &BTreeSet, + lease_budget: Duration, + ) -> Self { + let lead = TimeDelta::from_std(lead).unwrap_or(TimeDelta::zero()); + let budget_ms = lease_budget.as_millis().min(u64::MAX as u128) as u64; + + let mut fires_by_token: BTreeMap<&str, Vec<&FireRecord>> = BTreeMap::new(); + let mut complete: BTreeMap<&str, bool> = BTreeMap::new(); + let mut targets_unreadable = Vec::new(); + let mut targets_truncated = Vec::new(); + let mut fires_recorded = 0u64; + + for log in logs { + complete.insert(log.agent.as_str(), log.is_complete()); + if log.error.is_some() { + targets_unreadable.push(log.agent.clone()); + } else if !log.is_complete() { + targets_truncated.push(log.agent.clone()); + } + for fire in &log.fires { + fires_recorded += 1; + fires_by_token + .entry(fire.token.as_str()) + .or_default() + .push(fire); + } + } + + let mut report = Self { + lease_budget_ms: budget_ms, + registrations_confirmed: 0, + registrations_indeterminate: 0, + registrations_rejected: 0, + fires_recorded, + fired_once: 0, + indeterminate_that_fired: 0, + inconclusive: 0, + unverifiable: 0, + unknown_tokens: 0, + targets_unreadable, + targets_truncated, + delay: Vec::new(), + overdue_on_arrival: 0, + overdue_delay: LatencyStats::default(), + findings: Vec::new(), + findings_omitted: 0, + }; + + let mut claimed: BTreeSet<&str> = BTreeSet::new(); + let mut findings: Vec = Vec::new(); + // When each registration actually landed, which is what decides whether + // its action still had a chance of being on time. + let mut registered_at: BTreeMap<&str, DateTime> = BTreeMap::new(); + + for record in records.iter().filter(|r| r.stream == Stream::Scheduled) { + let token = record.idempotency_key.as_str(); + claimed.insert(token); + if let Some(completed) = record.completed_at { + registered_at.insert(token, completed); + } + let fires = fires_by_token.get(token).map(Vec::as_slice).unwrap_or(&[]); + // The due time comes from the fire itself when there is one, because + // that is what the platform was actually told. Without a fire the + // driver only knows what it asked for. + let due = fires + .first() + .map(|f| f.scheduled_at) + .unwrap_or(record.submitted_at + lead); + let window = FireWindow::of(due, fault); + let can_testify = complete + .get(record.agent.as_str()) + .copied() + .unwrap_or(false); + + match record.outcome { + Outcome::Confirmed => report.registrations_confirmed += 1, + Outcome::Indeterminate => report.registrations_indeterminate += 1, + Outcome::Rejected => report.registrations_rejected += 1, + } + + match (record.outcome, fires.len()) { + (_, count) if count > 1 => findings.push(FireFinding { + violation: FireViolation::FiredMoreThanOnce, + token: token.to_string(), + agent: record.agent.clone(), + window, + detail: format!( + "one registration, {count} fires at {} — the action ran more than once", + fires + .iter() + .map(|f| f.observed_at.to_rfc3339()) + .collect::>() + .join(", ") + ), + }), + (Outcome::Rejected, 1) => findings.push(FireFinding { + violation: FireViolation::FiredDespiteRejection, + token: token.to_string(), + agent: record.agent.clone(), + window, + detail: "the platform refused the registration, then ran the action" + .to_string(), + }), + (Outcome::Confirmed, 1) => report.fired_once += 1, + (Outcome::Indeterminate, 1) => { + report.fired_once += 1; + report.indeterminate_that_fired += 1; + } + (Outcome::Confirmed, 0) if !can_testify => report.unverifiable += 1, + (Outcome::Confirmed, 0) => findings.push(FireFinding { + violation: FireViolation::NeverFired, + token: token.to_string(), + agent: record.agent.clone(), + window, + detail: format!( + "accepted registration due at {} never fired", + due.to_rfc3339() + ), + }), + (Outcome::Indeterminate, 0) if !can_testify => report.unverifiable += 1, + (Outcome::Indeterminate, 0) => report.inconclusive += 1, + (Outcome::Rejected, 0) => {} + (_, _) => {} + } + } + + report.unknown_tokens = fires_by_token + .iter() + .filter(|(token, _)| !claimed.contains(*token)) + .map(|(_, fires)| fires.len() as u64) + .sum(); + + report.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + report.findings = findings; + let (delay, overdue) = delay_stats(logs, fault, killed_targets, budget_ms, ®istered_at); + report.delay = delay; + report.overdue_on_arrival = overdue.len() as u64; + report.overdue_delay = + LatencyStats::from_durations(overdue.iter().map(|d| (*d).max(0) as u64).collect()); + report + } + + /// The three conditions that fail the scenario. + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() + } + + /// The p99 of the cell that the SLO is about: actions due while the + /// executor was gone, on targets it owned. + pub fn fault_window_p99_ms(&self) -> Option { + self.delay + .iter() + .find(|d| { + d.group == TargetGroup::OnKilledExecutor && d.window == FireWindow::DuringFault + }) + .map(|d| d.delay.p99_ms) + } + + /// Lines an operator should see next to the read-back verdicts. + pub fn attention_lines(&self) -> Vec { + let mut lines = Vec::new(); + for finding in &self.findings { + lines.push(format!( + "{}: token {} on target {} ({}) — {}", + finding.violation, finding.token, finding.agent, finding.window, finding.detail + )); + } + if self.findings_omitted > 0 { + lines.push(format!( + "scheduled-fire findings are a sample: {} more were dropped", + self.findings_omitted + )); + } + if !self.targets_unreadable.is_empty() { + lines.push(format!( + "{} scheduled targets could not be read back, so {} registrations are \ + unverifiable rather than accounted for", + self.targets_unreadable.len(), + self.unverifiable + )); + } + if !self.targets_truncated.is_empty() { + lines.push(format!( + "{} scheduled targets filled their fire log and dropped entries — raise the \ + cadence or shorten the run before reading this as exactly-once evidence", + self.targets_truncated.len() + )); + } + if self.unknown_tokens > 0 { + lines.push(format!( + "{} fires carried a token no registration claims, which should be impossible \ + within one run nonce", + self.unknown_tokens + )); + } + if let Some(p99) = self.fault_window_p99_ms() + && p99 > self.lease_budget_ms + { + lines.push(format!( + "scheduled-fire p99 during the fault was {p99}ms against a {}ms lease budget", + self.lease_budget_ms + )); + } + lines + } + + /// Lines that explain the account without claiming anything is wrong. + /// + /// Overdue-on-arrival lives here rather than in [`Self::attention_lines`] + /// because it describes the client, not the platform: the due time is + /// minted before the registering invocation goes out, so a slow + /// registration describes an action that was already late when it arrived + /// and which the scheduler then ran correctly and at once. + /// + /// Read its count together with the worst case, never alone. The + /// classification catches anything slower than the lead, so a fix that + /// shortens a two-minute stall to twelve seconds moves entries *into* this + /// count while making the platform strictly better. + pub fn note_lines(&self) -> Vec { + let mut lines = Vec::new(); + if self.overdue_on_arrival > 0 { + lines.push(format!( + "{} already overdue when the registration landed (worst {}ms late). That is \ + client-side registration latency, not scheduler delay, so they are excluded \ + from the delay percentiles and counted here instead", + if self.overdue_on_arrival == 1 { + "1 action was".to_string() + } else { + format!("{} actions were", self.overdue_on_arrival) + }, + self.overdue_delay.max_ms + )); + } + lines + } +} + +/// Delay percentiles per (group, window) cell, and the fires held out of them. +/// +/// A fire is held out when its registration completed after the action was +/// already due: nothing the scheduler did could have made it on time, so +/// counting it as scheduler delay would blame the platform for the client. +fn delay_stats( + logs: &[TargetFireLog], + fault: Option, + killed_targets: &BTreeSet, + budget_ms: u64, + registered_at: &BTreeMap<&str, DateTime>, +) -> (Vec, Vec) { + let mut cells: BTreeMap<(TargetGroup, FireWindow), Vec> = BTreeMap::new(); + let mut overdue: Vec = Vec::new(); + + for log in logs { + let group = if killed_targets.contains(&log.agent) { + TargetGroup::OnKilledExecutor + } else { + TargetGroup::Elsewhere + }; + for fire in &log.fires { + let late_on_arrival = registered_at + .get(fire.token.as_str()) + .is_some_and(|landed| *landed > fire.scheduled_at); + if late_on_arrival { + overdue.push(fire.delay_ms()); + continue; + } + cells + .entry((group, FireWindow::of(fire.scheduled_at, fault))) + .or_default() + .push(fire.delay_ms()); + } + } + + let cells: Vec = cells + .into_iter() + .map(|((group, window), delays)| { + let min = delays.iter().copied().min().unwrap_or(0); + let over_budget = delays.iter().filter(|d| **d > budget_ms as i64).count() as u64; + FireDelayStats { + group, + window, + delay: LatencyStats::from_durations( + delays.iter().map(|d| (*d).max(0) as u64).collect(), + ), + min_delay_ms: min, + over_budget, + } + }) + .collect(); + (cells, overdue) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{AttemptRecord, Phase}; + use test_r::test; + + fn at(offset_secs: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + offset_secs, 0).unwrap() + } + + const LEAD: Duration = Duration::from_secs(10); + const BUDGET: Duration = Duration::from_secs(30); + + fn record( + token: &str, + agent: &str, + outcome: Outcome, + submitted_at: DateTime, + ) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Scheduled, + phase: Phase::Fault, + agent: agent.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: token.to_string(), + submitted_at, + completed_at: Some(submitted_at), + attempts: 1, + outcome, + duration_ms: 12, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 12, + returned_value: None, + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + fn fire(token: &str, scheduled: DateTime, delay_ms: i64) -> FireRecord { + FireRecord { + token: token.to_string(), + scheduled_at: scheduled, + observed_at: scheduled + TimeDelta::milliseconds(delay_ms), + } + } + + /// A log the agent answered in full. + fn log(agent: &str, fires: Vec) -> TargetFireLog { + TargetFireLog { + agent: agent.to_string(), + polls: Some(fires.len() as u64), + fires, + error: None, + } + } + + fn build(records: &[OperationRecord], logs: &[TargetFireLog]) -> ScheduleFireReport { + ScheduleFireReport::build(records, logs, LEAD, None, &BTreeSet::new(), BUDGET) + } + + /// The healthy shape: one registration, one fire. + #[test] + fn a_registration_paired_with_one_fire_is_not_a_finding() { + let r = record("t-0", "target-0", Outcome::Confirmed, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-0", at(10), 40)])], + ); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!(report.fired_once, 1); + assert_eq!(report.registrations_confirmed, 1); + } + + /// Accepted work has to happen. This is the loss half of the guarantee. + #[test] + fn a_confirmed_registration_that_never_fired_fails_the_run() { + let r = record("t-1", "target-0", Outcome::Confirmed, at(0)); + let report = build(std::slice::from_ref(&r), &[log("target-0", vec![])]); + assert!(report.has_violations()); + assert_eq!(report.findings[0].violation, FireViolation::NeverFired); + assert_eq!(report.findings[0].token, "t-1"); + assert!( + report.findings[0].detail.contains(&at(10).to_rfc3339()), + "the finding must say when the action was due: {}", + report.findings[0].detail + ); + } + + /// The duplicate half, and the reason the target records tokens rather than + /// a tally: a lease recovered while the original claim was still running + /// runs the action twice, and no count over a busy target would show it. + #[test] + fn a_token_that_fired_twice_fails_the_run() { + let r = record("t-2", "target-0", Outcome::Confirmed, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log( + "target-0", + vec![fire("t-2", at(10), 40), fire("t-2", at(10), 31_000)], + )], + ); + assert!(report.has_violations()); + assert_eq!( + report.findings[0].violation, + FireViolation::FiredMoreThanOnce + ); + } + + /// A definite refusal means nothing was accepted, so an action that runs + /// anyway is the platform contradicting its own answer. + #[test] + fn a_refused_registration_that_fired_anyway_fails_the_run() { + let r = record("t-3", "target-0", Outcome::Rejected, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-3", at(10), 5)])], + ); + assert!(report.has_violations()); + assert_eq!( + report.findings[0].violation, + FireViolation::FiredDespiteRejection + ); + } + + /// The distinction the whole verdict rests on: the driver never learned + /// whether this registration landed, so an action that never fired is one + /// of two legitimate answers. + #[test] + fn an_indeterminate_registration_that_never_fired_is_counted_rather_than_failed() { + let r = record("t-4", "target-0", Outcome::Indeterminate, at(0)); + let report = build(std::slice::from_ref(&r), &[log("target-0", vec![])]); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!( + report.inconclusive, 1, + "but it must be counted, so a clean verdict over many of them reads as weaker" + ); + } + + /// ...and when it did fire, the doubt resolved in the platform's favour. + #[test] + fn an_indeterminate_registration_that_fired_resolves_the_doubt() { + let r = record("t-5", "target-0", Outcome::Indeterminate, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-5", at(10), 20)])], + ); + assert!(!report.has_violations()); + assert_eq!(report.fired_once, 1); + assert_eq!(report.indeterminate_that_fired, 1); + } + + /// A read that failed is not evidence about the platform. Turning it into + /// one would report a network problem as lost durable work. + #[test] + fn an_unreadable_target_leaves_its_registrations_unverifiable_rather_than_lost() { + let r = record("t-6", "target-0", Outcome::Confirmed, at(0)); + let unreadable = TargetFireLog { + agent: "target-0".to_string(), + polls: None, + fires: Vec::new(), + error: Some("timed out after 30s".to_string()), + }; + let report = build(std::slice::from_ref(&r), &[unreadable]); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!(report.unverifiable, 1); + assert_eq!(report.targets_unreadable, vec!["target-0".to_string()]); + } + + /// Same reasoning for a log that filled up: the fire may have happened and + /// been dropped. `polls` is what makes that detectable at all. + #[test] + fn a_truncated_fire_log_leaves_its_registrations_unverifiable_rather_than_lost() { + let r = record("t-7", "target-0", Outcome::Confirmed, at(0)); + let truncated = TargetFireLog { + agent: "target-0".to_string(), + // The agent fired 500 actions and kept 2 of them. + polls: Some(500), + fires: vec![fire("t-other", at(10), 5), fire("t-more", at(11), 5)], + error: None, + }; + let report = build(std::slice::from_ref(&r), &[truncated]); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!(report.unverifiable, 1); + assert_eq!(report.targets_truncated, vec!["target-0".to_string()]); + } + + /// The grouping that makes the percentile mean anything. Both cells are + /// reported; folding them together would let the untouched half hide what + /// the recovery cost. + #[test] + fn delay_is_split_by_whether_the_target_was_on_the_killed_executor() { + let fault = FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + }; + let killed = BTreeSet::from(["target-killed".to_string()]); + let logs = vec![ + log("target-killed", vec![fire("a", at(150), 28_000)]), + log("target-survivor", vec![fire("b", at(150), 40)]), + ]; + let report = ScheduleFireReport::build(&[], &logs, LEAD, Some(fault), &killed, BUDGET); + + let killed_cell = report + .delay + .iter() + .find(|d| d.group == TargetGroup::OnKilledExecutor) + .expect("the killed executor's targets need their own cell"); + assert_eq!(killed_cell.window, FireWindow::DuringFault); + assert_eq!(killed_cell.delay.p99_ms, 28_000); + + let survivor_cell = report + .delay + .iter() + .find(|d| d.group == TargetGroup::Elsewhere) + .expect("the control group needs its own cell"); + assert_eq!(survivor_cell.delay.p99_ms, 40); + + assert_eq!(report.fault_window_p99_ms(), Some(28_000)); + } + + /// Actions are placed by when they were *due*, not when they fired: an + /// action due during the outage that landed afterwards is exactly the + /// population the scenario is about. + #[test] + fn an_action_is_placed_in_the_window_its_due_time_fell_in() { + let fault = Some(FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + }); + assert_eq!(FireWindow::of(at(99), fault), FireWindow::BeforeFault); + assert_eq!(FireWindow::of(at(100), fault), FireWindow::DuringFault); + // Due mid-fault, fired long after it healed: still a fault-window action. + assert_eq!(FireWindow::of(at(199), fault), FireWindow::DuringFault); + assert_eq!(FireWindow::of(at(200), fault), FireWindow::AfterFault); + assert_eq!(FireWindow::of(at(150), None), FireWindow::Unknown); + } + + /// A fault that never cleared leaves everything after injection inside it, + /// rather than inventing an end. + #[test] + fn a_fault_that_never_cleared_has_no_after_window() { + let fault = Some(FaultWindow { + injected_at: at(100), + recovered_at: None, + }); + assert_eq!(FireWindow::of(at(10_000), fault), FireWindow::DuringFault); + } + + /// Two clocks measure this, so a small negative delay is skew. It stays + /// visible as a minimum instead of being folded into the percentiles. + #[test] + fn clock_skew_shows_as_a_negative_minimum_rather_than_flattering_the_percentiles() { + let logs = vec![log( + "target-0", + vec![fire("a", at(10), -35), fire("b", at(11), 60)], + )]; + let report = ScheduleFireReport::build(&[], &logs, LEAD, None, &BTreeSet::new(), BUDGET); + let cell = &report.delay[0]; + assert_eq!(cell.min_delay_ms, -35); + assert_eq!(cell.delay.p50_ms, 0, "the negative sample clamps to zero"); + assert_eq!(cell.delay.max_ms, 60); + } + + /// Fires over the lease budget are counted per cell, which is what turns a + /// percentile into SLO evidence rather than a number. + #[test] + fn fires_past_the_lease_budget_are_counted() { + let logs = vec![log( + "target-0", + vec![ + fire("a", at(10), 29_000), + fire("b", at(11), 31_000), + fire("c", at(12), 45_000), + ], + )]; + let report = ScheduleFireReport::build(&[], &logs, LEAD, None, &BTreeSet::new(), BUDGET); + assert_eq!(report.delay[0].over_budget, 2); + } + + /// An artifact nobody can open is not evidence. Truncation is stated. + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let records: Vec = (0..MAX_FINDINGS + 25) + .map(|i| record(&format!("t-{i}"), "target-0", Outcome::Confirmed, at(0))) + .collect(); + let report = build(&records, &[log("target-0", vec![])]); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 25); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("are a sample")), + "an operator must be told the list is partial" + ); + } + + /// Agent names carry the run nonce, so this cannot happen within a run. If + /// it ever does, the pairing is answering a different question than it + /// thinks it is, and the report says so. + #[test] + fn a_fire_whose_token_no_registration_claims_is_counted() { + let r = record("t-8", "target-0", Outcome::Confirmed, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log( + "target-0", + vec![fire("t-8", at(10), 5), fire("stranger", at(10), 5)], + )], + ); + assert_eq!(report.unknown_tokens, 1); + assert!(!report.has_violations()); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("no registration claims")) + ); + } + + /// Only the scheduled stream is paired: a durable operation carries an + /// idempotency key too, and pairing it against a fire log would invent + /// findings out of an unrelated stream. + #[test] + fn operations_from_other_streams_are_not_paired() { + let mut durable = record("t-9", "target-0", Outcome::Confirmed, at(0)); + durable.stream = Stream::Durable; + let report = build(std::slice::from_ref(&durable), &[log("target-0", vec![])]); + assert_eq!(report.registrations_confirmed, 0); + assert!(!report.has_violations()); + } + + /// The correction the first S10 run forced. + /// + /// An executor kill stalls the invocations in flight to it. Those + /// registrations land late, describing actions that were already overdue, + /// and the platform then runs them immediately and correctly. Counting that + /// as scheduler delay put 13 fires over budget in the killed executor's + /// targets and 13 in the control group — which is the tell, since the + /// control group's shards never moved. + #[test] + fn a_fire_whose_registration_landed_late_is_held_out_of_the_delay_cells() { + // Registered at t=0 for t=10, but the invocation only returned at t=125 + // after a stalled attempt and a retry. The action fires at once. + let mut r = record("stalled", "target-0", Outcome::Confirmed, at(0)); + r.completed_at = Some(at(125)); + r.duration_ms = 125_000; + let prompt = record("prompt", "target-0", Outcome::Confirmed, at(20)); + + let report = build( + &[r, prompt], + &[log( + "target-0", + vec![ + fire("stalled", at(10), 115_200), + fire("prompt", at(30), 1_100), + ], + )], + ); + + assert_eq!(report.overdue_on_arrival, 1); + assert_eq!(report.overdue_delay.max_ms, 115_200); + assert_eq!( + report.delay.iter().map(|d| d.delay.count).sum::(), + 1, + "only the registration that landed before its due time belongs in the cells" + ); + assert_eq!(report.delay[0].delay.max_ms, 1_100); + assert_eq!( + report.delay[0].over_budget, 0, + "the stalled registration must not put the scheduler over budget" + ); + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("client-side registration latency")), + "an operator has to be told why the count is held out" + ); + assert!( + report.attention_lines().is_empty(), + "holding a fire out of the cells is context, not a finding: CI raises an \ + annotation on attention, and a run where the platform did nothing wrong must \ + not raise one" + ); + // Still exactly-once: both actions ran once. + assert!(!report.has_violations()); + assert_eq!(report.fired_once, 2); + } + + /// The boundary: landing exactly on the due time is not late. + #[test] + fn a_registration_that_landed_on_its_due_time_stays_in_the_cells() { + let mut r = record("t-0", "target-0", Outcome::Confirmed, at(0)); + r.completed_at = Some(at(10)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-0", at(10), 900)])], + ); + assert_eq!(report.overdue_on_arrival, 0); + assert_eq!(report.delay[0].delay.count, 1); + } + + /// The p99 an operator is judged on is the fault-window cell for the + /// targets that were on the dead pod. With no such cell there is nothing to + /// judge, and the report says nothing rather than substituting another. + #[test] + fn the_reported_p99_is_absent_when_the_fault_window_produced_no_fires() { + let report = ScheduleFireReport::build(&[], &[], LEAD, None, &BTreeSet::new(), BUDGET); + assert_eq!(report.fault_window_p99_ms(), None); + } +} diff --git a/integration-tests/src/chaos/history.rs b/integration-tests/src/chaos/history.rs index 9a00ab5185..f225da4a7a 100644 --- a/integration-tests/src/chaos/history.rs +++ b/integration-tests/src/chaos/history.rs @@ -67,6 +67,32 @@ pub enum Stream { /// one known executor, so mixing them into the durable population would /// blur two different experiments. PinnedHttp, + /// `Counter.increment` rounds on agents that are then deleted outright + /// (GOL-372). + /// + /// Excluded from the generic read-back for the same reason as + /// [`Stream::Revert`], one step further: a deleted agent's counter is + /// *supposed* to be gone, so comparing it against everything confirmed + /// against it would report the whole agent as lost work. See + /// [`crate::chaos::resurrection`]. + Delete, + /// `Counter.increment` rounds that are deliberately taken back again, and + /// the `revert` calls that take them (GOL-371). + /// + /// Distinct from [`Stream::Durable`] even though both land on `Counter` + /// agents, and for a reason that is not cosmetic: a reverted increment is + /// acknowledged work that the platform was then *asked* to forget, so the + /// generic read-back — which compares a counter against everything + /// confirmed against it — would report every one of them as lost. S7 + /// computes each agent's exact expected value from its own round history + /// instead, which is strictly stronger than a range. + Revert, + /// `PromiseWaiter.arm` / `wait` / the external completion that resolves it + /// (GOL-377). Distinct from `Promise` even though both land on the promise + /// component: that stream creates and resolves a promise in one breath with + /// nobody suspended on it, and this one exists precisely to leave an agent + /// parked across a fault. + PromiseWait, } impl Stream { @@ -78,6 +104,9 @@ impl Stream { Stream::Promise => "promise", Stream::Quota => "quota", Stream::PinnedHttp => "pinned-http", + Stream::PromiseWait => "promise-wait", + Stream::Delete => "delete", + Stream::Revert => "revert", } } @@ -91,6 +120,11 @@ impl Stream { /// - `Promise` operations resolve a one-shot promise rather than advancing a /// counter, so there is no accumulated number to read. They are reported /// on created/completed counts and latency. + /// - `PromiseWait` agents *do* keep a durable count, but comparing totals + /// would be strictly weaker than what S11 already does with them: every + /// completion carries a token into the waiter's wakeup log, so the report + /// pairs individual completions against individual wakeups instead of + /// arguing about sums. See [`crate::chaos::wakeups`]. pub fn has_readback(self) -> bool { matches!( self, @@ -98,13 +132,16 @@ impl Stream { ) } - pub const ALL: [Stream; 6] = [ + pub const ALL: [Stream; 9] = [ Stream::Durable, Stream::Ephemeral, Stream::Scheduled, Stream::Promise, Stream::Quota, Stream::PinnedHttp, + Stream::PromiseWait, + Stream::Revert, + Stream::Delete, ]; } @@ -312,6 +349,88 @@ impl OperationRecord { pub fn had_successful_attempt(&self) -> bool { self.outcome == Outcome::Confirmed || self.attempt_log.iter().any(|a| a.succeeded) } + + /// Attempts at this operation that hit the client's attempt timeout rather + /// than answering. + /// + /// Matched on the message [`crate::chaos::workload`] writes for a timed-out + /// attempt. A structured flag would be better, but the attempt log is an + /// archived shape that older results already carry, and this reads it + /// without changing it. + /// + /// The distinction it draws is the one a stalled dependency needs: an + /// operation that timed out and then answered on its retry was rescued by + /// the caller, not returned by the platform, and the outcome alone cannot + /// say so. + pub fn attempts_timed_out(&self) -> u64 { + self.attempt_log + .iter() + .filter(|a| { + a.error + .as_deref() + .is_some_and(|e| e.contains("attempt timed out")) + }) + .count() as u64 + } +} + +/// What a scheduled action recorded when it ran, and the log it was recorded +/// in (GOL-378). +/// +/// These live here rather than in [`crate::chaos::fires`] for the same reason +/// [`OperationRecord`] does: they are what was *observed*, and the analysis +/// that reduces them is a separate thing that a later ticket may want to redo +/// over an archived run. The first S10 run learned that the expensive way — its +/// delay percentiles turned out to need a correction that could not be applied +/// afterwards, because only the reduced numbers had been archived. +/// +/// This one is a single fire, as the target agent recorded it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FireRecord { + /// The registering invocation's idempotency key. + pub token: String, + /// When the action was due, as the driver asked for it. + pub scheduled_at: DateTime, + /// When the platform ran it, as the executor's clock saw it. + pub observed_at: DateTime, +} + +impl FireRecord { + /// How far past its due time the action ran. Negative means clock skew + /// between the driver and the executor, not an action that fired early. + pub fn delay_ms(&self) -> i64 { + (self.observed_at - self.scheduled_at).num_milliseconds() + } +} + +/// Everything read back from one target agent. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TargetFireLog { + pub agent: String, + /// `ScheduleCounter.polls`, which keeps counting past the log's cap and is + /// therefore what says whether the log below is complete. + pub polls: Option, + pub fires: Vec, + /// Why the agent could not be read, when it could not be. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl TargetFireLog { + /// Whether this log can testify about its own registrations. + /// + /// Two ways it cannot: the read failed outright, or the component's fire log + /// hit its cap and dropped entries. Both leave an absent fire ambiguous. + pub fn is_complete(&self) -> bool { + match (self.error.is_some(), self.polls) { + (true, _) => false, + (false, Some(polls)) => self.fires.len() as u64 >= polls, + // No `polls` read means no way to tell whether the log is whole. + (false, None) => false, + } + } } /// The persisted history document. @@ -324,6 +443,76 @@ pub struct HistoryDocument { /// so a reader never mistakes a partial history for a short one. pub partial: bool, pub operations: Vec, + /// Per-target fire logs, for the scenarios that drive scheduled actions. + /// Empty for every other scenario rather than absent, so a reader never + /// wonders whether the section was dropped. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub scheduled_fires: Vec, + /// Per-waiter wakeup logs, for the scenarios that park agents on promises. + /// Empty for every other scenario rather than absent, for the same reason as + /// `scheduled_fires`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub promise_wakeups: Vec, +} + +/// One wakeup, as the waiter agent recorded it (GOL-377). +/// +/// The times are the *cluster's*, stamped inside the agent, which is what makes +/// this log the authority on whether a completion landed. The driver's own view +/// is in the operation record for the `wait` invocation, and during the fault +/// that view is frequently just a broken connection. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupRecord { + /// The round's idempotency key, carried in by `arm` and back out by the + /// wakeup. This is what pairs a completion to the wakeup it caused. + pub token: String, + /// When the waiter armed the promise. + pub armed_at: DateTime, + /// When the platform resumed the waiter. + pub woken_at: DateTime, +} + +impl WakeupRecord { + /// How long the waiter was parked, on one clock. + /// + /// Both ends are stamped by the executor, so this is free of the driver ↔ + /// cluster skew that the completion-to-wakeup delay carries. It is not the + /// delay itself — it also contains the round's deliberate dwell — but it is + /// what lets a reader tell a slow wakeup from a skewed clock. + pub fn parked_ms(&self) -> i64 { + (self.woken_at - self.armed_at).num_milliseconds() + } +} + +/// Everything read back from one waiter agent. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WaiterWakeupLog { + pub agent: String, + /// `PromiseWaiter.wakes`, which keeps counting past the log's cap and is + /// therefore what says whether the log below is complete. + pub wakes: Option, + pub wakeups: Vec, + /// Why the agent could not be read, when it could not be. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl WaiterWakeupLog { + /// Whether this log can testify about its own completions. + /// + /// Same two ways it cannot as [`TargetFireLog::is_complete`], and the same + /// consequence: without a whole log an absent wakeup is ambiguous between a + /// lost completion and a dropped log entry, and S11 must not call the second + /// one a finding. + pub fn is_complete(&self) -> bool { + match (self.error.is_some(), self.wakes) { + (true, _) => false, + (false, Some(wakes)) => self.wakeups.len() as u64 >= wakes, + (false, None) => false, + } + } } /// Append-only operation log, shared across the concurrent workload streams. @@ -335,6 +524,8 @@ pub struct OperationHistory { scenario_code: String, inner: Arc>>, next_id: Arc, + fire_logs: Arc>>, + wakeup_logs: Arc>>, } impl OperationHistory { @@ -343,6 +534,8 @@ impl OperationHistory { scenario_code: scenario_code.into(), inner: Arc::new(Mutex::new(Vec::new())), next_id: Arc::new(std::sync::atomic::AtomicU64::new(0)), + fire_logs: Arc::new(Mutex::new(Vec::new())), + wakeup_logs: Arc::new(Mutex::new(Vec::new())), } } @@ -383,12 +576,27 @@ impl OperationHistory { .count() as u64 } + /// Archives the fire logs read back at the end of a scheduled scenario. + /// + /// Called once, after read-back, before the history is written. Kept here + /// rather than in the result because the result is the reduced report and + /// this is the raw material it was reduced from. + pub fn record_fire_logs(&self, logs: Vec) { + *self.fire_logs.lock().unwrap() = logs; + } + + pub fn record_wakeup_logs(&self, logs: Vec) { + *self.wakeup_logs.lock().unwrap() = logs; + } + pub fn document(&self, partial: bool) -> HistoryDocument { HistoryDocument { schema_version: HISTORY_SCHEMA_VERSION, scenario_code: self.scenario_code.clone(), partial, operations: self.snapshot(), + scheduled_fires: self.fire_logs.lock().unwrap().clone(), + promise_wakeups: self.wakeup_logs.lock().unwrap().clone(), } } diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 2404c937a8..c639914b47 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -18,8 +18,10 @@ //! Cloud-mode Golem while a bounded fault is injected, then reports what //! happened. The suite's shape follows density: the workflow drives one scenario //! per invocation, each scenario is independently selectable through a YAML -//! `enabled` flag, and results are archived to S3 per scenario so an interrupted -//! run resumes rather than restarts. +//! `enabled` flag, and results are archived to the `golem-bench-results` bucket +//! per scenario so an interrupted run resumes rather than restarts. +//! +//! `S3` in this module always means the scenario code, never the bucket. //! //! Two boundaries define this module: //! @@ -32,18 +34,32 @@ //! engine here — see [`summary`] for what is measured and the narrow set of //! conditions that fail a run outright. +pub mod deletions; pub mod errors; +pub mod fires; pub mod history; +pub mod outage; pub mod ownership; pub mod pinned; pub mod prep; pub mod probe; +pub mod reachability; pub mod result; +pub mod resurrection; +pub mod reverts; +pub mod rollback; pub mod scenarios; +pub mod scheduled; pub mod signal; +pub mod split; +pub mod steady; pub mod summary; +pub mod truncation; +pub mod waiters; +pub mod wakeups; pub mod workload; +use crate::chaos::history::Stream; use anyhow::Context; use serde::{Deserialize, Serialize}; use std::path::Path; @@ -63,6 +79,43 @@ pub enum ScenarioCode { S12, /// Rolling executor restarts under load. S13, + /// Executor pod kill while scheduled actions are between claim and fire. + S10, + /// Executor pod kill while agents are suspended on promises being completed. + S11, + /// Executor cut off from worker-service while it keeps its shards. + S3, + /// Executor pod kill while agents are having their state reverted. + S7, + /// Executor pod kill while agents are being deleted. + S6, + /// Executor pod kill while a component rollback is in flight. + S9, + /// Executors cut off from the key-value PostgreSQL cluster for about as + /// long as an AWS storage failover takes. + S16, + /// The same cut, held for longer than the key-value retry budget, so the + /// executors are expected to be replaced rather than to ride it out. + S22, + /// Executors cut off from the indexed-oplog PostgreSQL cluster, the other + /// Aurora cluster underneath them, for the length of a writer failover. + S14, + /// Executors cut off from the Redis cache that fronts the key-value layer, + /// for longer than a caller is willing to wait. + S18, + /// The same Redis cache still reachable but slowed, to ask whether the + /// platform degrades or breaks when its worker-status store gets slower. + S17, + /// The key-value PostgreSQL cluster still reachable but slowed. S17's + /// mirror on the other half of the split key-value layer. + S15, + /// S15 driving `ephemeral` alone. The control: that stream reaches none of + /// the delayed cluster, so nothing should move. + S15A, + /// S15A plus `durable`, the heaviest user of the delayed cluster. + S15B, + /// S15B plus `promise`. The remaining step, `scheduled`, is S15 itself. + S15C, } impl ScenarioCode { @@ -73,18 +126,48 @@ impl ScenarioCode { ScenarioCode::S5 => "S5", ScenarioCode::S12 => "S12", ScenarioCode::S13 => "S13", + ScenarioCode::S10 => "S10", + ScenarioCode::S11 => "S11", + ScenarioCode::S3 => "S3", + ScenarioCode::S7 => "S7", + ScenarioCode::S6 => "S6", + ScenarioCode::S9 => "S9", + ScenarioCode::S16 => "S16", + ScenarioCode::S22 => "S22", + ScenarioCode::S14 => "S14", + ScenarioCode::S18 => "S18", + ScenarioCode::S17 => "S17", + ScenarioCode::S15 => "S15", + ScenarioCode::S15A => "S15A", + ScenarioCode::S15B => "S15B", + ScenarioCode::S15C => "S15C", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 5] = [ + pub const ALL: [ScenarioCode; 20] = [ ScenarioCode::S1, + ScenarioCode::S3, ScenarioCode::S5, + ScenarioCode::S6, + ScenarioCode::S7, ScenarioCode::S8, + ScenarioCode::S9, + ScenarioCode::S10, + ScenarioCode::S11, ScenarioCode::S12, ScenarioCode::S13, + ScenarioCode::S14, + ScenarioCode::S15, + ScenarioCode::S15A, + ScenarioCode::S15B, + ScenarioCode::S15C, + ScenarioCode::S16, + ScenarioCode::S17, + ScenarioCode::S18, + ScenarioCode::S22, ]; pub fn parse(s: &str) -> Option { @@ -172,6 +255,17 @@ pub struct FaultConfig { /// half the executors" is only meaningful next to how many that was. #[serde(default, skip_serializing_if = "Option::is_none")] pub target_count: Option, + /// The Chaos Mesh manifest to apply, by file name, when it is not the one + /// the workflow would find from the scenario code. + /// + /// Exists for the elimination variants. S15A, S15B and S15C inject exactly + /// the fault S15 injects and differ only in which streams the workload + /// drives, so copying `networkchaos-s15.yaml` three times would make four + /// files that must be kept identical by hand — and a run whose fault + /// silently diverged from the run it is being compared against is worse + /// than no run. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub manifest: Option, pub duration_secs: u64, } @@ -301,6 +395,581 @@ fn default_candidate_pool_multiplier() -> u32 { 8 } +/// Shape of the scheduled-registration workload (GOL-378). +/// +/// A third experiment shape next to [`WorkloadConfig`] and [`PinnedConfig`], not +/// a variation on either. The mixed workload asks what a stream of invocations +/// does when the platform is disturbed, and the pinned workload asks what +/// happens to specific invocations that were running on the pod that died. This +/// one asks about work the driver is not holding a connection to at all: an +/// action the platform promised to run later, whose executor died in between. +/// +/// The two numbers that decide whether the run measures anything are `leadSecs` +/// and `intervalMillis`. See [`crate::chaos::scheduled`] for why. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScheduledConfig { + /// Target agents, each with its own emitter. Also the resolution of the + /// report: a finding localises to one target out of this many. + pub targets: u32, + /// Milliseconds between registrations on one target. The offered rate is + /// `targets / interval`. + pub interval_millis: u64, + /// How far ahead each action is registered. With the cadence above this + /// sets how many actions stand accepted but not yet run at any instant, + /// which is the population a kill has to land in the middle of. + pub lead_secs: u64, + /// What recovering a scheduled action is allowed to cost, which the + /// fire-delay percentiles are reported against. An SLO the run records + /// rather than a threshold it fails on: the floor is a shard reassignment, + /// or the executor's `lease_ttl` for the rarer case of an action that was + /// already claimed, and how much more than that is acceptable is a + /// judgement. + pub lease_budget_secs: u64, +} + +impl ScheduledConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn lead(&self) -> Duration { + Duration::from_secs(self.lead_secs) + } + pub fn lease_budget(&self) -> Duration { + Duration::from_secs(self.lease_budget_secs) + } +} + +/// Shape of the suspended-waiter workload (GOL-377). +/// +/// The fourth experiment shape, and the only one whose agents are *asleep* when +/// the fault lands. [`ScheduledConfig`] leaves work with the platform and walks +/// away; this one leaves an agent parked mid-invocation on a promise, so the +/// thing that has to survive the kill is not a queued action but a suspended +/// worker and the completion on its way to it. +/// +/// The number that decides whether the run measures anything is `waiters`: each +/// one holds exactly one promise at a time, so the pool size *is* the population +/// standing parked at the instant the pod dies. `dwellMillis` decides how much +/// of that population is also mid-completion — see [`crate::chaos::waiters`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PromiseConfig { + /// Waiter agents, each parked on at most one promise at a time. Also the + /// resolution of the report: a finding localises to one waiter out of this + /// many. + pub waiters: u32, + /// How long a waiter stays parked before its promise is completed. + /// + /// Sets the completion rate — `waiters / dwell` — and, with it, how many + /// completions are genuinely in flight when the pod dies. It has to + /// comfortably exceed the workflow's inject-and-verify path (signal poll, + /// `kubectl apply`, waiting for `AllInjected`) or every promise armed before + /// the kill would already have been completed by the time it landed. + pub dwell_millis: u64, + /// What resuming a suspended waiter is allowed to cost, which the wakeup + /// delay percentiles are reported against. An SLO the run records rather + /// than a threshold it fails on: the floor is a shard reassignment plus the + /// worker recovery that replays the waiter's oplog, and how much more than + /// that is acceptable is a judgement. + pub wakeup_budget_secs: u64, +} + +impl PromiseConfig { + pub fn dwell(&self) -> Duration { + Duration::from_millis(self.dwell_millis) + } + pub fn wakeup_budget(&self) -> Duration { + Duration::from_secs(self.wakeup_budget_secs) + } +} + +/// Shape of the reachability workload (GOL-370). +/// +/// The fifth experiment shape, and the only one where nothing about the +/// platform is broken at all. [`PinnedConfig`] asks what happens to operations +/// running on a pod that dies; this one asks what happens to operations bound +/// for a pod that is perfectly healthy and simply cannot be reached from the +/// tier that routes to it. The executor keeps its shards for the whole fault, +/// because the link it needs in order to keep them — to the shard-manager — is +/// not the one that was cut. +/// +/// Every agent gets its own emitter holding at most one operation, rather than +/// the shared per-stream budget [`WorkloadConfig`] drives. That is load-bearing +/// here and not a style choice: the stall this scenario induces is bounded by +/// *which executor owns the agent*, not by which stream it belongs to, so a +/// shared budget would be drained by the isolated half and would stop the +/// reachable half submitting too. The run would then report the control group +/// degrading, and the cause would be the driver. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IsolationConfig { + /// Durable counter agents, split by shard ownership into the ones the + /// isolated executor holds and the ones it does not. Also the resolution of + /// the report: a finding localises to one agent out of this many. + pub agents: u32, + /// Milliseconds between one agent's operations, measured from the end of + /// the previous one. The offered rate is `agents / interval`, and an agent + /// whose operation is stalled offers nothing at all — which is the signal, + /// not a gap in it. + pub interval_millis: u64, + /// The most of its own baseline throughput the isolated group may keep + /// during the fault, as a percentage, for the partition to count as + /// observed. + /// + /// A run above this line did not cut the executor off, whatever the fault + /// status says, and every other number in the report is then a measurement + /// of an undisturbed cluster. That is reported as inconclusive rather than + /// clean: a healthy-looking result from a fault that never landed is the + /// worst artifact this suite can produce. + pub isolated_ceiling_percent: f64, + /// The least of its own baseline throughput the control group must keep + /// during the fault, as a percentage. + /// + /// The sharpest thing S3 can find. The agents on the reachable executor + /// have nothing to do with the partition, so a drop here is collateral + /// damage from how worker-service handles an unreachable pod — its routing + /// table is one process-wide entry, and every stalled caller invalidating it + /// costs every other caller a shard-manager round trip. + pub control_floor_percent: f64, + /// What resuming an isolated agent may cost once the link is back, and the + /// number the recovery gap is reported against. Recorded rather than + /// asserted, like every other budget in the suite. + pub recovery_budget_secs: u64, +} + +impl IsolationConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } +} + +/// Shape of the revert workload (GOL-371). +/// +/// The sixth experiment shape, and the only one that asks the platform to +/// *destroy* durable state on purpose. Every other scenario disturbs work that +/// is trying to happen; this one disturbs work that is trying to be undone. +/// +/// Each agent repeats a round: increment `increments_per_round` times, then +/// revert the last `revert_invocations` of them. Both numbers are exact, and +/// that is the point — the driver knows the counter's value before the revert +/// from the last increment's own return value, so the value afterwards has +/// exactly two legitimate answers and no band of doubt between them. +/// +/// Reverting needs the worker stopped (`lock_stopped_worker` in +/// `golem-worker-executor/src/worker/mod.rs`), so a revert is not one atomic +/// instant but a stop, a commit and a status reattach. The truncation itself is +/// a single oplog entry and cannot tear; the window worth killing into is the +/// one around it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RevertConfig { + /// Counter agents running rounds, split by shard ownership around the + /// executor the kill is aimed at. Also the resolution of the report: a + /// torn revert localises to one agent out of this many. + pub agents: u32, + /// Increments before each revert. Must be at least `revert_invocations`, + /// or the revert would reach back into an already-deleted oplog region and + /// the platform would refuse it — see `find_nth_invocation_from_end`. + pub increments_per_round: u32, + /// How many of those increments each revert takes back. + pub revert_invocations: u32, + /// Milliseconds an agent waits between rounds. The share of the population + /// standing mid-revert at any instant is roughly one round-step in + /// `increments_per_round + 1`, so this and the round length together decide + /// how much of the mechanism a kill can land in. + pub interval_millis: u64, + /// What recovering a reverted agent may cost, and the number the + /// resume delay is reported against. Recorded rather than asserted, like + /// every other budget in the suite. + pub recovery_budget_secs: u64, +} + +impl RevertConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } + /// What one completed round adds to a counter. + pub fn net_per_round(&self) -> u32 { + self.increments_per_round + .saturating_sub(self.revert_invocations) + } +} + +/// Shape of the deletion workload (GOL-372). +/// +/// The seventh experiment shape, and one step past [`RevertConfig`]. A revert +/// asks the platform to forget some of an agent's work; this asks it to forget +/// the agent. Each slot builds a counter up, deletes it, and is used again — +/// invoking a deleted id creates a new agent, so the next round's first +/// increment says which of the two things happened. +/// +/// The failure mode it is named for has a defence in the executor already: +/// `start_deleting` stops a background status flush from "resurrecting the +/// cached status" after the durable removal. So the question is not whether +/// anyone thought about it, but whether the defence survives the pod dying +/// between the mark and the removal. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeleteConfig { + /// Agent slots running rounds, split by shard ownership around the executor + /// the kill is aimed at. Also the resolution of the report: a resurrection + /// localises to one slot out of this many. + pub agents: u32, + /// Increments before each delete. + /// + /// More than one, so that a *partial* survival is observable at all. The + /// two legal answers are always distinguishable — a fresh agent reports 1 + /// and a survivor reports `before + 1`, which never collide — but at one + /// increment there is no value *between* them, so a slot that came back + /// carrying some of a state it should have lost has nowhere to land and + /// [`crate::chaos::resurrection::ResurrectionViolation::PartialState`] can + /// never fire. Three of them leaves room for it. + pub increments_per_round: u32, + /// Milliseconds a slot waits between rounds. + pub interval_millis: u64, + /// What recovering a deleted agent's slot may cost. Recorded rather than + /// asserted, like every other budget in the suite. + pub recovery_budget_secs: u64, +} + +impl DeleteConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } +} + +/// Shape of the component rollback (GOL-369). +/// +/// S5 moves agents forward onto a new build and kills an executor while that is +/// happening. S9 moves them forward, waits for that to land, and then moves them +/// **back**, killing an executor during the return leg. +/// +/// The rollback is a redeploy: the original artifact is uploaded again as a new +/// revision, and every agent is asked to move to it. That is what "rollback" +/// means operationally, and it is what makes the evidence unambiguous — +/// `Counter::component_version` is compiled into each build, so an agent that +/// has genuinely returned reports `1` from the code that is actually running, +/// not from metadata about what the platform believes. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RollbackConfig { + /// How long to let the roll-forward settle before rolling back. + /// + /// Load-bearing. If the agents never reached the new build, the rollback + /// returns them to a build they never left and the run proves nothing, so + /// this has to be long enough for the forward leg to finish on a healthy + /// cluster. + pub settle_secs: u64, + /// The share of agents that must actually report the new build before the + /// rollback is worth attempting, as a percentage. + /// + /// Below it the driver stops rather than spending the maintenance window on + /// a return journey nobody made. The same instinct as S6's smoke round. + pub rolled_forward_floor_percent: f64, + /// Retries for the rollback *control plane* — the per-agent update + /// requests — recorded apart from the workload's own retries. + /// + /// Separate because they answer different questions. The workload's retry + /// exists to expose duplicate execution and is deliberately one attempt. + /// This one exists so that a refused rollback request is distinguishable + /// from a rollback that was never asked for, which matters when the + /// executor owning an agent is about to be killed. + pub control_retries: u32, + pub control_retry_delay_secs: u64, + /// How far into the rollback to ask for the kill. + pub kill_delay_secs: u64, +} + +impl RollbackConfig { + pub fn settle(&self) -> Duration { + Duration::from_secs(self.settle_secs) + } + pub fn control_retry_delay(&self) -> Duration { + Duration::from_secs(self.control_retry_delay_secs) + } + pub fn kill_delay(&self) -> Duration { + Duration::from_secs(self.kill_delay_secs) + } +} + +/// Settings for a storage-outage scenario (GOL-379). +/// +/// Not a workload shape, unlike the blocks above it: S16 drives the mixed +/// workload and the scheduled-registration workload that already exist, because +/// the fault is not about which agents are driven but about which *dependency* +/// is taken away from all of them. What this block carries is the thing being +/// taken away and the two numbers the account is judged by. +/// +/// The endpoint is recorded rather than acted on, like every other entry in +/// [`FaultConfig`]. The driver never resolves it, connects to it or cuts +/// anything off from it; the workflow does that, and this exists so an archived +/// result says which storage the run was about rather than leaving a reader to +/// infer it from the scenario name. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageConfig { + /// The storage endpoint the workflow partitions the executors from, as a + /// hostname. Chaos Mesh resolves it in the controller, so this is the same + /// string that appears in the NetworkChaos manifest's `externalTargets`. + pub endpoint: String, + /// What this scenario expects its cut to do to the workload, and therefore + /// what the run treats as evidence the cut landed. + /// + /// Deliberately per-scenario rather than one shared rule. The *account* the + /// driver produces is shared because it is factual: throughput per stream + /// per window, quiet time, latency, what was caught in flight. The + /// *verdict* is not, because what a cut is supposed to do depends on what + /// it cut, and a rule that reads correctly for one arrangement can be + /// confidently wrong about another. See [`OutageExpectation`]. + pub expect: OutageExpectation, + /// What serving again may cost once the storage is reachable, and the + /// number each stream's recovery gap is reported against. Recorded rather + /// than asserted, like every other budget in the suite: how long a + /// connection pool may take to notice its database came back is a + /// judgement, and the number is in the result either way. + pub recovery_budget_secs: u64, +} + +/// What a storage cut is expected to do to the workload. +/// +/// The two variants exist because the storage scenarios cut two different +/// *kinds* of thing, not two instances of one thing. S16, S22 and S14 each take +/// away a database every stream depends on, so "did anything keep serving" is a +/// sound test of whether the fault landed. S18 takes away a cache only part of +/// the workload touches, and under that arrangement the same rule is not merely +/// too strict — it is backwards. It reported S18's first run as an outage that +/// never happened while the partition had plainly landed, because `durable` +/// held 100% of its baseline throughout, exactly as the design says it should. +/// +/// Parameterising the old rule with a stream list would have fixed that one +/// message and left the deeper problem: for a partial cut, the streams that +/// *keep working* carry as much information as the ones that stop. If `durable` +/// had gone quiet under S18, the status blob would not be off the commit path +/// the way `AgentStatusFlusher` claims, and the shared rule would have called +/// that a clean pass. So the partial variant asserts both halves. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde( + tag = "kind", + rename_all = "kebab-case", + rename_all_fields = "camelCase" +)] +pub enum OutageExpectation { + /// Everything is behind the cut, so every stream that was serving before it + /// must fall silent. S16, S22 and S14. + WholeWorkload { + /// The least of the fault window each stream must answer nothing at all + /// for, as a percentage of that window. + /// + /// This used to be a ceiling on during-fault throughput as a share of + /// baseline, and that number is still recorded. It stopped being the + /// verdict because it is a rate averaged over the whole window while + /// all the serving in an absorbed outage happens in the seconds at its + /// edges, so the same handful of edge confirmations reads as 8% of a + /// 180s window and 26% of a 60s one. The threshold then tracks the + /// window length rather than the platform, and shortening S16's window + /// to 60s duly tripped it on a partition that had plainly landed. + /// + /// Quiet time has no such coupling: it is measured against the window's + /// own edges, so it means the same thing whatever the window is. + quiet_floor_percent: f64, + }, + /// Only part of the workload is behind the cut. The named streams must fall + /// silent and every other stream that was serving before it must keep + /// serving. S18. + PartialWorkload { + /// The streams this cut is expected to stop. Their silence is the + /// evidence the fault landed, so naming a stream the workload never + /// drives would leave the run with no such evidence and no complaint — + /// which `require_storage` refuses at load time rather than discovering + /// during a maintenance window. + silenced: Vec, + /// As [`Self::WholeWorkload::quiet_floor_percent`], but applied only to + /// the streams named above. + quiet_floor_percent: f64, + /// The least of its own baseline rate an *unnamed* stream must still + /// hold during the cut. + /// + /// This is the half a shared rule cannot express. A partial cut makes a + /// claim about the streams it does not touch, and that claim is worth + /// asserting: it is derived from where the code routes each namespace, + /// so a stream stalling here says the routing is not what the source + /// says it is. + serving_floor_percent: f64, + }, + /// The store is still reachable, only slow. Nothing is expected to fall + /// silent at all, so the evidence the fault landed is latency rather than + /// absence. S15 and S17. + /// + /// Worth being explicit that this is a different *question*, not a milder + /// version of the other two. Those ask whether the platform survives losing + /// a store. This asks whether it degrades or breaks when one gets slower, + /// and "it went quiet" would be a failure here rather than the expected + /// result. + LatencyDegradation { + /// The streams whose latency must rise for the injected delay to have + /// landed. Their slowdown plays the part silence plays elsewhere: it is + /// the only positive evidence the run has that anything happened. + slowed: Vec, + /// The least each named stream's during-fault median latency must be as + /// a multiple of its own before-fault median. + /// + /// A multiple rather than a millisecond figure, so the threshold does + /// not have to be re-derived per stream. Streams differ by an order of + /// magnitude at rest — S18's baselines ran from 56ms on `durable` to + /// 240ms on `ephemeral` — and a fixed ceiling would be slack for one + /// and impossible for the other. + slowdown_floor: f64, + /// The streams the delay is expected *not* to reach, and whose + /// steadiness is therefore the run's evidence that it reached only what + /// it aimed at. + /// + /// The latency counterpart to what + /// [`Self::PartialWorkload::serving_floor_percent`] does for a cut, and + /// it earns its place for the same reason. Both halves of the key-value + /// layer are delayed by a scenario in this suite, and each predicts a + /// different split: S17 slows the Redis half, where only a lifecycle + /// boundary crosses synchronously, so `ephemeral` moves and the other + /// three do not; S15 slows the PostgreSQL half, where the running-workers + /// recovery index, the promise keys and the scheduler's own schema all + /// live, so those three move and `ephemeral` does not. Naming the far + /// side turns each run into a test of that routing rather than a + /// measurement taken on faith. + /// + /// Optional, and empty means the run asserts nothing about the streams + /// it did not name. + #[serde(default)] + steady: Vec, + /// The most a steady stream's during-fault median may be as a multiple + /// of its own before-fault median. + /// + /// Not 1.0. A stream on the far side of the delay still shares + /// executors, connection pools and a tokio runtime with the streams on + /// the near side, so some spill is expected and is not itself a finding. + /// The number has to sit above that spill and below the multiple a real + /// dependency would produce, and those are far apart: S17 measured 6.59x + /// on the stream it delayed and no movement at all — 61ms, 100ms and + /// 91ms, identical across all three windows — on the three it did not. + #[serde(default = "default_steady_ceiling")] + steady_ceiling: f64, + /// The least of its own baseline rate *every* stream must still hold, + /// including the slowed ones. + /// + /// The claim that separates degradation from breakage. A delay should + /// make work take longer, not stop it, so a stream falling to nothing + /// here means the added latency cost more than time — a timeout fired, + /// a pool drained, a queue filled — and that is worth failing on. + serving_floor_percent: f64, + }, +} + +/// Applied when a `latency-degradation` expectation names steady streams +/// without saying how steady. Loose enough that ordinary run-to-run spread on +/// an undelayed stream does not trip it, which matters because the finding it +/// raises says the routing is wrong. +fn default_steady_ceiling() -> f64 { + 1.5 +} + +impl OutageExpectation { + /// The quiet floor, which both variants carry and apply to a different set. + pub fn quiet_floor_percent(&self) -> f64 { + match self { + OutageExpectation::WholeWorkload { + quiet_floor_percent, + } + | OutageExpectation::PartialWorkload { + quiet_floor_percent, + .. + } => *quiet_floor_percent, + // Nothing is expected to go quiet, so there is no floor to apply + // and reporting one would invite a reader to check it. + OutageExpectation::LatencyDegradation { .. } => 0.0, + } + } + + /// Whether this stream is one whose slowdown would prove a delay landed. + pub fn expects_slowdown(&self, stream: Stream) -> bool { + match self { + OutageExpectation::LatencyDegradation { slowed, .. } => slowed.contains(&stream), + _ => false, + } + } + + /// The multiple of its own baseline median a slowed stream must reach, or + /// `None` where the expectation makes no claim about latency. + pub fn slowdown_floor(&self) -> Option { + match self { + OutageExpectation::LatencyDegradation { slowdown_floor, .. } => Some(*slowdown_floor), + _ => None, + } + } + + /// Whether this stream is one the delay is expected not to reach. + pub fn expects_steady(&self, stream: Stream) -> bool { + match self { + OutageExpectation::LatencyDegradation { steady, .. } => steady.contains(&stream), + _ => false, + } + } + + /// The multiple of its own baseline median a steady stream may not exceed, + /// or `None` where the expectation names no steady streams to hold to it. + pub fn steady_ceiling(&self) -> Option { + match self { + OutageExpectation::LatencyDegradation { + steady, + steady_ceiling, + .. + } if !steady.is_empty() => Some(*steady_ceiling), + _ => None, + } + } + + /// Whether this stream is one whose silence would prove the cut landed. + /// + /// `WholeWorkload` says yes to everything, which is what makes it the + /// stricter rule rather than merely a different one. + pub fn expects_silence(&self, stream: Stream) -> bool { + match self { + OutageExpectation::WholeWorkload { .. } => true, + OutageExpectation::PartialWorkload { silenced, .. } => silenced.contains(&stream), + OutageExpectation::LatencyDegradation { .. } => false, + } + } + + /// The floor an unnamed stream's during-fault throughput must clear, or + /// `None` where the expectation makes no claim about streams still serving. + pub fn serving_floor_percent(&self) -> Option { + match self { + OutageExpectation::WholeWorkload { .. } => None, + OutageExpectation::PartialWorkload { + serving_floor_percent, + .. + } + | OutageExpectation::LatencyDegradation { + serving_floor_percent, + .. + } => Some(*serving_floor_percent), + } + } +} + +impl StorageConfig { + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } +} + /// One step of the executor scale schedule the workflow runs during the fault. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -372,6 +1041,29 @@ pub struct ScenarioConfig { /// The pinned in-flight workload. Absent for scenarios that do not run one. #[serde(default)] pub pinned: Option, + /// The scheduled-registration workload. Absent for scenarios that do not + /// run one. + #[serde(default)] + pub scheduled: Option, + /// The suspended-waiter workload. Absent for scenarios that do not run one. + #[serde(default)] + pub promise: Option, + /// The reachability workload. Absent for scenarios that do not run one. + #[serde(default)] + pub isolation: Option, + /// The revert workload. Absent for scenarios that do not run one. + #[serde(default)] + pub revert: Option, + /// The deletion workload. Absent for scenarios that do not run one. + #[serde(default)] + pub delete: Option, + /// The component rollback. Absent for scenarios that do not run one. + #[serde(default)] + pub rollback: Option, + /// Storage-outage settings. Absent for scenarios that do not take a + /// storage dependency away. + #[serde(default)] + pub storage: Option, /// Shard-ownership oracle settings. Absent for scenarios that do not sample /// executor assignments. #[serde(default)] @@ -425,6 +1117,329 @@ impl ScenarioConfig { }) } + /// The scheduled-registration block. See [`Self::require_workload`]. + pub fn require_scheduled(&self) -> anyhow::Result<&ScheduledConfig> { + self.scheduled.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `scheduled` block in the suite YAML", + self.code + ) + }) + } + + /// The suspended-waiter block. See [`Self::require_workload`]. + pub fn require_promise(&self) -> anyhow::Result<&PromiseConfig> { + self.promise.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `promise` block in the suite YAML", + self.code + ) + }) + } + + /// The reachability workload block. See [`Self::require_workload`]. + pub fn require_isolation(&self) -> anyhow::Result<&IsolationConfig> { + self.isolation.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs an `isolation` block in the suite YAML", + self.code + ) + }) + } + + /// The revert workload block. See [`Self::require_workload`]. + pub fn require_revert(&self) -> anyhow::Result<&RevertConfig> { + let config = self.revert.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `revert` block in the suite YAML", + self.code + ) + })?; + // Checked here rather than discovered mid-run: a revert reaching past + // its own round lands in an already-deleted oplog region and the + // platform refuses it, so every round would fail and the scenario would + // measure nothing. + if config.revert_invocations > config.increments_per_round { + anyhow::bail!( + "chaos scenario {}: revertInvocations ({}) exceeds incrementsPerRound ({}), \ + so every revert would reach into an already-deleted oplog region", + self.code, + config.revert_invocations, + config.increments_per_round + ); + } + if config.revert_invocations == 0 { + anyhow::bail!( + "chaos scenario {}: revertInvocations is 0, so the scenario would revert nothing", + self.code + ); + } + Ok(config) + } + + /// The deletion workload block. See [`Self::require_workload`]. + pub fn require_delete(&self) -> anyhow::Result<&DeleteConfig> { + let config = self.delete.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `delete` block in the suite YAML", + self.code + ) + })?; + // At one increment the two legal answers are adjacent — 1 and 2 — so + // nothing can land between them and the partial-state violation is + // structurally unobservable. A third of the oracle would be blind, and + // every run would look clean on that axis by construction. + if config.increments_per_round < 2 { + anyhow::bail!( + "chaos scenario {}: incrementsPerRound is {}, which leaves no value between \ + a fresh agent and a survivor, so a partial state could never be observed", + self.code, + config.increments_per_round + ); + } + Ok(config) + } + + /// The rollback block. See [`Self::require_workload`]. + pub fn require_rollback(&self) -> anyhow::Result<&RollbackConfig> { + self.rollback.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `rollback` block in the suite YAML", + self.code + ) + }) + } + + /// The storage-outage block. See [`Self::require_workload`]. + /// Whether this scenario's configuration actually produces operations on a + /// stream. + /// + /// Counts the blocks that drive traffic rather than the streams a run + /// happens to record, so it can be answered from the YAML alone before any + /// cluster time is spent. `Scheduled` has two possible sources and either + /// one counts; the streams that belong to other scenario shapes are not + /// reachable from a `storage` scenario's config at all. + fn drives_stream(&self, stream: Stream) -> bool { + let workload = self.workload.as_ref(); + match stream { + Stream::Durable => workload.is_some_and(|w| w.durable_agents > 0), + Stream::Ephemeral => workload.is_some_and(|w| w.ephemeral_agents > 0), + Stream::Promise => workload.is_some_and(|w| w.promise_agents > 0), + Stream::Quota => workload.is_some_and(|w| w.quota_agents > 0), + Stream::Scheduled => { + workload.is_some_and(|w| w.scheduled_agents > 0) + || self.scheduled.as_ref().is_some_and(|s| s.targets > 0) + } + Stream::PinnedHttp | Stream::PromiseWait | Stream::Delete | Stream::Revert => false, + } + } + + /// A floor of 100% can never be met: a stream is only quiet between the + /// answers it did give, and an operation submitted just before the heal + /// confirms just after it, inside the window it was submitted in. Every run + /// would then report the outage as not observed, and the one verdict that + /// exists to catch a fault which never landed would be stuck on. A floor of + /// zero is the opposite failure: every run passes, including the ones where + /// nothing was ever cut off. + fn check_quiet_floor(&self, floor: f64) -> anyhow::Result<()> { + if !(0.0..100.0).contains(&floor) || floor <= 0.0 { + anyhow::bail!( + "chaos scenario {}: expect.quietFloorPercent is {floor}, which is not a share \ + of the fault window a real outage could be judged by", + self.code + ); + } + Ok(()) + } + + fn check_serving_floor(&self, floor: f64) -> anyhow::Result<()> { + if !(0.0..=100.0).contains(&floor) || floor <= 0.0 { + anyhow::bail!( + "chaos scenario {}: expect.servingFloorPercent is {floor}, which is not a share \ + of a stream's own baseline it could be held to", + self.code + ); + } + Ok(()) + } + + /// The streams whose behaviour is the run's only positive evidence that the + /// fault landed at all. + /// + /// Naming one the workload never drives leaves the run with no such + /// evidence and nothing to complain about — the exact failure the floors + /// exist to prevent, reintroduced one level up. Refused at load time so it + /// costs a build rather than a maintenance window. + fn check_evidence_streams( + &self, + field: &str, + streams: &[Stream], + evidence: &str, + ) -> anyhow::Result<()> { + if streams.is_empty() { + anyhow::bail!( + "chaos scenario {}: expect.{field} is empty, so nothing in the run would show \ + whether the fault landed at all", + self.code + ); + } + for stream in streams { + if !self.drives_stream(*stream) { + anyhow::bail!( + "chaos scenario {}: expect.{field} names `{stream}`, which this scenario's \ + workload never drives, so its {evidence} during the fault would prove \ + nothing", + self.code + ); + } + } + Ok(()) + } + + pub fn require_storage(&self) -> anyhow::Result<&StorageConfig> { + let config = self.storage.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `storage` block in the suite YAML", + self.code + ) + })?; + // A floor of 100% can never be met: a stream is only quiet between the + // answers it did give, and an operation submitted just before the heal + // confirms just after it, inside the window it was submitted in. Every + // run would then report the outage as not observed, and the one verdict + // that exists to catch a fault which never landed would be stuck on. A + // floor of zero is the opposite failure: every run passes, including + // the ones where nothing was ever cut off. + // Checked per variant rather than through the accessors, because the + // variants do not share a set of knobs and a single pass over "whatever + // is present" would silently skip whichever one this scenario actually + // relies on. + match &config.expect { + OutageExpectation::WholeWorkload { + quiet_floor_percent, + } => { + self.check_quiet_floor(*quiet_floor_percent)?; + } + OutageExpectation::PartialWorkload { + silenced, + quiet_floor_percent, + serving_floor_percent, + } => { + self.check_quiet_floor(*quiet_floor_percent)?; + self.check_serving_floor(*serving_floor_percent)?; + self.check_evidence_streams("silenced", silenced, "silence")?; + } + OutageExpectation::LatencyDegradation { + slowed, + slowdown_floor, + steady, + steady_ceiling, + serving_floor_percent, + } => { + // A floor of 1.0 or less asks a stream to be no slower than it + // already was, which every run satisfies including one where + // the delay never applied. That is the same hole a zero quiet + // floor leaves, in the units this variant uses. + if !slowdown_floor.is_finite() || *slowdown_floor <= 1.0 { + anyhow::bail!( + "chaos scenario {}: expect.slowdownFloor is {slowdown_floor}, and a \ + factor at or below 1.0 is met by a run where the delay never applied", + self.code + ); + } + self.check_serving_floor(*serving_floor_percent)?; + // A run may name no slowed streams at all, but only if it names + // steady ones — and then it is a control rather than an + // experiment. S15A drives `ephemeral` alone, which reaches none + // of the delayed store, so *nothing* in its workload should + // move and there is no slowdown available as evidence that the + // fault landed. + // + // The cost is real and worth stating: such a run cannot fail on + // `slowdown-not-observed`, so it cannot tell a delay that + // landed and did not matter from one that never applied. That + // evidence has to come from outside the workload, and for S15A + // it does — the scheduler polls its own schema on the delayed + // cluster whether or not anything is scheduled, so + // `db_success_seconds{svc="scheduler_storage"}` moves under the + // fault regardless. The runbook says to check it. + if slowed.is_empty() && steady.is_empty() { + anyhow::bail!( + "chaos scenario {}: expect.slowed and expect.steady are both empty, so \ + nothing in the run would show whether the fault landed or whether it \ + mattered", + self.code + ); + } + if !slowed.is_empty() { + self.check_evidence_streams("slowed", slowed, "slowdown")?; + } + // The steady list is optional, so it is checked only when + // present — but a present one carries the same two ways of + // being useless as the slowed list, plus one of its own. + for stream in steady { + if !self.drives_stream(*stream) { + anyhow::bail!( + "chaos scenario {}: expect.steady names `{stream}`, which this \ + scenario's workload never drives, so its steadiness during the \ + fault would prove nothing", + self.code + ); + } + if slowed.contains(stream) { + anyhow::bail!( + "chaos scenario {}: expect names `{stream}` as both slowed and \ + steady, so the run would demand the same stream both move and \ + stay put", + self.code + ); + } + } + // A ceiling at or below 1.0 asks an undelayed stream to run no + // slower than its own baseline median, which run-to-run spread + // alone breaks. The finding it would then raise says the + // platform routes a namespace somewhere other than where the + // source says, and that is far too strong a claim to make on + // noise. + if !steady.is_empty() && (!steady_ceiling.is_finite() || *steady_ceiling <= 1.0) { + anyhow::bail!( + "chaos scenario {}: expect.steadyCeiling is {steady_ceiling}, and a \ + factor at or below 1.0 is broken by ordinary run-to-run spread on a \ + stream the delay never reached", + self.code + ); + } + } + } + if config.endpoint.trim().is_empty() { + anyhow::bail!( + "chaos scenario {}: storage.endpoint is empty, so the result could not say which \ + storage the run took away", + self.code + ); + } + // Checked here rather than in the driver so a bad YAML fails the build + // instead of a maintenance window. Both blocks write to the scheduled + // stream, but only the `scheduled` block's registrations carry a token + // into the target's fire log — see `ScheduleEmitter::schedule_fire_at` + // in the counters component. Driving both would leave the fire account + // pairing the mixed workload's tokenless registrations against nothing + // and reporting every one of them as an action that never ran. + if let (Some(workload), Some(_)) = (&self.workload, &self.scheduled) + && workload.scheduled_agents > 0 + { + anyhow::bail!( + "chaos scenario {}: the mixed workload drives {} scheduled agents while a \ + `scheduled` block is also present, and only the latter's registrations carry a \ + token into the fire log, so the fire account would report every mixed-workload \ + registration as one that never fired", + self.code, + workload.scheduled_agents + ); + } + Ok(config) + } + /// The pinned workload block. See [`Self::require_workload`]. pub fn require_pinned(&self) -> anyhow::Result<&PinnedConfig> { self.pinned.as_ref().ok_or_else(|| { @@ -519,6 +1534,34 @@ mod tests { ); } + /// A `pod-kill` is instantaneous and its `duration` only governs how long + /// the object lingers, so those two numbers are free to differ. A + /// `network-partition` is not: Chaos Mesh takes the iptables rules down on + /// the object's own `duration`, while the workflow holds the phase for + /// `faultSecs` and only then deletes it. A shorter `durationSecs` therefore + /// heals the cluster part-way through the window the driver is still + /// attributing to the fault, and every during-fault number is measured + /// across a mix of the two states. + #[test] + fn a_partition_lasts_at_least_as_long_as_the_fault_phase_it_is_measured_over() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + for entry in &suite.scenarios { + if entry.fault.kind != "network-partition" { + continue; + } + assert!( + entry.fault.duration_secs >= entry.phases.fault_secs, + "{}: fault.durationSecs ({}) is shorter than phases.faultSecs ({}), \ + so the partition lifts {}s before the fault phase ends and the rest \ + of that phase measures a healed cluster", + entry.code, + entry.fault.duration_secs, + entry.phases.fault_secs, + entry.phases.fault_secs - entry.fault.duration_secs, + ); + } + } + #[test] fn every_suite_entry_resolves_to_an_implemented_scenario() { let suite = ChaosSuite::load(suite_path()).unwrap(); @@ -542,6 +1585,7 @@ mod tests { target: "shard-manager".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, phases: PhaseConfig { @@ -558,6 +1602,13 @@ mod tests { rate_per_sec: 1, }), pinned: None, + scheduled: None, + promise: None, + isolation: None, + revert: None, + delete: None, + rollback: None, + storage: None, ownership: None, scale_during_fault: None, retry_policy: RetryPolicy::default(), @@ -570,6 +1621,238 @@ mod tests { assert!(suite.scenario(ScenarioCode::S12, true).is_ok()); } + fn revert_config(increments: u32, revert: u32) -> ScenarioConfig { + ScenarioConfig { + code: "S7".to_string(), + name: "executor-crash-during-revert".to_string(), + enabled: true, + fault: FaultConfig { + kind: "pod-kill".to_string(), + target: "worker-executor".to_string(), + mode: "one".to_string(), + target_count: None, + manifest: None, + duration_secs: 60, + }, + phases: PhaseConfig { + baseline_secs: 1, + fault_secs: 1, + recovery_secs: 1, + }, + workload: None, + pinned: None, + scheduled: None, + promise: None, + isolation: None, + delete: None, + rollback: None, + storage: None, + revert: Some(RevertConfig { + agents: 10, + increments_per_round: increments, + revert_invocations: revert, + interval_millis: 500, + recovery_budget_secs: 60, + }), + ownership: None, + scale_during_fault: None, + retry_policy: RetryPolicy::default(), + signal_timeout_secs: 1, + } + } + + /// A revert reaching further back than its own round lands in the region an + /// earlier revert already deleted, and the platform refuses it outright. + /// Every round would fail and the run would measure nothing, so this is + /// caught before the maintenance window is spent rather than per round. + #[test] + fn a_revert_deeper_than_its_own_round_is_refused_before_the_run_starts() { + let error = revert_config(2, 3) + .require_revert() + .unwrap_err() + .to_string(); + assert!( + error.contains("already-deleted oplog region"), + "the message has to say why, got: {error}" + ); + } + + /// Reverting nothing would leave a scenario that builds state up and takes + /// none of it back, which is S8 with extra steps. + #[test] + fn a_revert_of_nothing_is_refused() { + assert!(revert_config(4, 0).require_revert().is_err()); + } + + /// The boundary case is legal: a round may take back everything it added. + #[test] + fn a_revert_of_exactly_one_round_is_allowed() { + let config = revert_config(3, 3); + assert_eq!(config.require_revert().unwrap().net_per_round(), 0); + } + + fn storage_config( + endpoint: &str, + quiet_floor: f64, + scheduled_agents: u32, + scheduled_block: bool, + ) -> ScenarioConfig { + ScenarioConfig { + code: "S16".to_string(), + name: "keyvalue-postgres-outage".to_string(), + enabled: true, + fault: FaultConfig { + kind: "network-partition".to_string(), + target: "worker-executor".to_string(), + mode: "all".to_string(), + target_count: None, + manifest: None, + duration_secs: 180, + }, + phases: PhaseConfig { + baseline_secs: 1, + fault_secs: 1, + recovery_secs: 1, + }, + workload: Some(WorkloadConfig { + durable_agents: 10, + ephemeral_agents: 0, + scheduled_agents, + promise_agents: 0, + quota_agents: 0, + rate_per_sec: 10, + }), + pinned: None, + scheduled: scheduled_block.then_some(ScheduledConfig { + targets: 10, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 240, + }), + promise: None, + isolation: None, + delete: None, + rollback: None, + storage: Some(StorageConfig { + endpoint: endpoint.to_string(), + expect: OutageExpectation::WholeWorkload { + quiet_floor_percent: quiet_floor, + }, + recovery_budget_secs: 120, + }), + revert: None, + ownership: None, + scale_during_fault: None, + retry_policy: RetryPolicy::default(), + signal_timeout_secs: 1, + } + } + + /// A floor of zero passes every run, including one where nothing was cut + /// off, so the verdict that exists to catch a fault which never landed + /// would never fire. + #[test] + fn a_storage_fault_quiet_floor_of_zero_is_refused() { + let error = storage_config("db.example", 0.0, 0, true) + .require_storage() + .unwrap_err() + .to_string(); + assert!( + error.contains("expect.quietFloorPercent"), + "the message has to name the knob, got: {error}" + ); + } + + /// A stream is only quiet between the answers it did give, so no real + /// outage can be quiet for the whole window and the verdict would be stuck + /// on for every run. + #[test] + fn a_storage_fault_quiet_floor_of_a_whole_window_is_refused() { + assert!( + storage_config("db.example", 100.0, 0, true) + .require_storage() + .is_err(), + "a floor of the whole window can never be met" + ); + } + + /// A control run: no stream is expected to slow, and the one named steady + /// is what the run is actually asking about. Allowed, because a series that + /// isolates an interaction needs a run with nothing in it to interact. + #[test] + fn a_latency_run_that_names_only_steady_streams_is_allowed() { + let mut config = storage_config("db.example", 50.0, 0, true); + config.workload.as_mut().unwrap().ephemeral_agents = 10; + config.storage.as_mut().unwrap().expect = OutageExpectation::LatencyDegradation { + slowed: Vec::new(), + slowdown_floor: 2.0, + steady: vec![Stream::Ephemeral], + steady_ceiling: 1.5, + serving_floor_percent: 50.0, + }; + assert!( + config.require_storage().is_ok(), + "a run may assert that nothing moved, as long as it names what should not have" + ); + } + + /// Both lists empty is the one shape that says nothing at all: no stream + /// would show the fault landing, and none would show it mattering either. + #[test] + fn a_latency_run_that_names_no_streams_at_all_is_refused() { + let mut config = storage_config("db.example", 50.0, 0, true); + config.storage.as_mut().unwrap().expect = OutageExpectation::LatencyDegradation { + slowed: Vec::new(), + slowdown_floor: 2.0, + steady: Vec::new(), + steady_ceiling: 1.5, + serving_floor_percent: 50.0, + }; + let error = config.require_storage().unwrap_err().to_string(); + assert!( + error.contains("expect.slowed and expect.steady are both empty"), + "the message has to say which pair is missing, got: {error}" + ); + } + + /// Without an endpoint the archived result could not say which storage the + /// run took away. + #[test] + fn a_storage_block_with_no_endpoint_is_refused() { + assert!( + storage_config(" ", 50.0, 0, true) + .require_storage() + .is_err() + ); + } + + /// Both blocks write to the scheduled stream and only one of them carries a + /// token into the fire log, so driving both would report every tokenless + /// registration as an action that never ran. Caught at load time rather + /// than discovered in the report. + #[test] + fn driving_the_scheduled_stream_from_both_blocks_is_refused() { + let error = storage_config("db.example", 15.0, 50, true) + .require_storage() + .unwrap_err() + .to_string(); + assert!( + error.contains("token into the fire log"), + "the message has to say why, got: {error}" + ); + } + + /// The same mixed-workload setting is fine on its own: without a + /// `scheduled` block there is only one writer to the stream. + #[test] + fn scheduled_agents_alone_are_allowed() { + assert!( + storage_config("db.example", 15.0, 50, false) + .require_storage() + .is_ok() + ); + } + /// The retry defaults are load-bearing for correctness, not just for load. #[test] fn retry_policy_defaults_to_one_same_key_transport_only_retry() { @@ -630,6 +1913,17 @@ mod tests { assert_eq!(ScenarioCode::parse("S12"), Some(ScenarioCode::S12)); assert_eq!(ScenarioCode::parse("s8"), Some(ScenarioCode::S8)); assert_eq!(ScenarioCode::parse("s1"), Some(ScenarioCode::S1)); + assert_eq!(ScenarioCode::parse("s3"), Some(ScenarioCode::S3)); + assert_eq!(ScenarioCode::parse("s7"), Some(ScenarioCode::S7)); + assert_eq!(ScenarioCode::parse("s6"), Some(ScenarioCode::S6)); + assert_eq!(ScenarioCode::parse("s9"), Some(ScenarioCode::S9)); + assert_eq!(ScenarioCode::parse("s16"), Some(ScenarioCode::S16)); + assert_eq!(ScenarioCode::parse("s14"), Some(ScenarioCode::S14)); + assert_eq!(ScenarioCode::parse("s18"), Some(ScenarioCode::S18)); + assert_eq!(ScenarioCode::parse("s17"), Some(ScenarioCode::S17)); + assert_eq!(ScenarioCode::parse("s15"), Some(ScenarioCode::S15)); + assert_eq!(ScenarioCode::parse("s15a"), Some(ScenarioCode::S15A)); + assert_eq!(ScenarioCode::parse("s15c"), Some(ScenarioCode::S15C)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -656,6 +1950,38 @@ mod tests { ScenarioCode::S13 => { entry.require_workload().unwrap(); } + ScenarioCode::S10 => { + entry.require_scheduled().unwrap(); + } + ScenarioCode::S11 => { + entry.require_promise().unwrap(); + } + ScenarioCode::S3 => { + entry.require_isolation().unwrap(); + } + ScenarioCode::S7 => { + entry.require_revert().unwrap(); + } + ScenarioCode::S6 => { + entry.require_delete().unwrap(); + } + ScenarioCode::S9 => { + entry.require_workload().unwrap(); + entry.require_rollback().unwrap(); + } + ScenarioCode::S14 + | ScenarioCode::S15 + | ScenarioCode::S15A + | ScenarioCode::S15B + | ScenarioCode::S15C + | ScenarioCode::S16 + | ScenarioCode::S17 + | ScenarioCode::S18 + | ScenarioCode::S22 => { + entry.require_workload().unwrap(); + entry.require_scheduled().unwrap(); + entry.require_storage().unwrap(); + } } } } diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs new file mode 100644 index 0000000000..ec7b0778af --- /dev/null +++ b/integration-tests/src/chaos/outage.rs @@ -0,0 +1,2014 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! What a storage outage cost, and whether the platform came back from it +//! (GOL-379). +//! +//! ### The control is time, not a pod +//! +//! Every other partition scenario in this suite keeps a group of agents on the +//! healthy side of the cut and reads the verdict off the disagreement between +//! the two groups. A storage outage has no healthy side. All three executors +//! share one key-value cluster, so cutting them off from it cuts off everything +//! at once, and an agent that happened to live elsewhere would be no better +//! served. +//! +//! So the comparison runs along the other axis. Each stream is measured against +//! **its own before-fault rate**, and the question the report answers first is +//! whether that rate collapsed at all. It has to be asked explicitly, because a +//! storage partition that silently failed to take hold produces a report full +//! of healthy numbers, and that must read as "this run tested nothing" rather +//! than as a pass. +//! +//! ### Throughput, not success rate +//! +//! The same reason [`crate::chaos::reachability`] gives, arriving by a +//! different route. The mixed workload gives every stream its own in-flight +//! budget ([`crate::chaos::workload::start`]), so a stream whose operations +//! stop returning does not fail over and over: it fills its budget, stops +//! submitting, and offers nothing further. A success rate would read that as a +//! handful of failures out of a handful of attempts. Confirmed operations per +//! second is the number that collapses. +//! +//! The per-stream budget is also what makes a per-stream row worth printing. +//! With one shared pool a single stalled stream drains the budget for all of +//! them and every row degrades together, which is exactly the unattributable +//! result S1 produced before the budgets were split. +//! +//! ### What the streams are actually testing +//! +//! The key-value cluster holds more than its name suggests. Reading the +//! executor's deployment: promises, the running-workers set and user key-value +//! data go to its `KeyValue` namespaces, and the scheduler keeps its own schema +//! on the same cluster. The oplog is on a different Aurora cluster and the +//! worker-status hot cache is in Redis, and neither is touched here. +//! +//! That is why every stream degrades under that cut rather than only the +//! obviously storage-shaped ones. A durable increment needs the running-workers +//! set before it can run at all, so `durable` is not a control group there and +//! must not be read as one. +//! +//! S18 cuts the Redis half instead, and the picture inverts. Only `Worker`, +//! `AgentStatus` and `AgentStatusCheckpoint` live there, the status blob is +//! written off the commit path by a background sweeper, and what still crosses +//! the cache synchronously is a lifecycle boundary or a `get_agent_mode` miss. +//! So `ephemeral` stops and `durable`, `scheduled` and `promise` carry on, and +//! the streams still answering are evidence rather than noise. +//! +//! S17 and S15 delay the two halves rather than cutting them, and the same +//! split decides which streams move. Under S17 that is `ephemeral` alone, which +//! its run confirmed at 6.59x against three streams that did not shift by a +//! millisecond. Under S15 it is the other three: a durable invocation flips its +//! agent between tracked and untracked in the running-workers index and waits +//! for the write, a promise operation reads and writes promise keys, and +//! registering a schedule writes to the scheduler's schema — all on the +//! PostgreSQL half — while an ephemeral agent is excluded from that index by +//! `AgentStatusFlusher::on_status_changed` and never touches it. +//! +//! ### What fails the run +//! +//! Five things, and all are statements about the experiment rather than about +//! latency: +//! +//! * [`OutageViolation::OutageNotObserved`] — a stream the cut was supposed to +//! stop kept working, so the fault did not land where the run says it did. +//! * [`OutageViolation::UnexpectedStall`] — a stream the cut was *not* supposed +//! to touch stopped anyway. +//! * [`OutageViolation::SlowdownNotObserved`] — a stream a delay was aimed at +//! ran no slower than at rest, so the netem rule proved nothing. +//! * [`OutageViolation::UnexpectedSlowdown`] — a stream a delay was *not* aimed +//! at slowed with it, which says the routing is not what the source says. +//! * [`OutageViolation::StreamNeverRecovered`] — a stream that was working +//! before the outage produced nothing at all after the heal. +//! +//! ### Why the verdict is per scenario and the account is not +//! +//! Everything above the verdict is factual and shared: throughput per stream +//! per window, quiet time, latency, what the fault caught in flight. The +//! verdict is supplied by the scenario, as an +//! [`OutageExpectation`](crate::chaos::OutageExpectation), because what a cut +//! is supposed to do depends on what it cut. +//! +//! This was one rule until S18. That rule — every stream must fall silent — +//! is sound for S16, S22 and S14, where a database every stream depends on goes +//! away. S18 cuts a cache only part of the workload touches, and there the same +//! rule is not merely too strict but backwards: it reported S18's first run as +//! an outage that never landed, on the grounds that `durable` held 100% of its +//! baseline, which is exactly what the routing in +//! `NamespaceRoutedKeyValueStorage` says should happen. +//! +//! Parameterising the old rule with a list of streams to exempt would have +//! silenced that message and left the deeper gap. Under a partial cut the +//! streams that keep working carry as much information as the ones that stop: +//! had `durable` gone quiet under S18, the status blob would not be off the +//! commit path the way `AgentStatusFlusher` describes, and a rule that only +//! looks for silence would have called that its cleanest possible pass. So the +//! partial expectation asserts both halves and the whole-workload one keeps +//! asserting the single half that is all it can know. +//! +//! Recovery time is recorded against the configured budget and never asserted +//! on, like every other budget in the suite. How long a connection pool may +//! take to notice its database is back is a judgement, and the number is in the +//! result either way. + +use crate::chaos::OutageExpectation; +use crate::chaos::errors::ErrorClass; +use crate::chaos::history::{OperationRecord, Outcome, Stream}; +use crate::chaos::split::{ + FaultWindow, Window, longest_silence_ms, round2, window_end, window_secs, window_start, +}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +/// What an outage finding is about. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OutageViolation { + /// The workload kept most of its baseline throughput while the storage was + /// supposed to be unreachable. Whatever the fault status said, the + /// executors could still reach the database, and every other number in this + /// report describes an undisturbed cluster. + OutageNotObserved, + /// A stream a partial cut was *not* supposed to touch stopped serving + /// anyway. + /// + /// The counterpart to [`Self::OutageNotObserved`], and the reason the + /// storage scenarios stopped sharing one rule. A partial cut is a claim + /// about routing: these namespaces go to the store being cut and those go + /// elsewhere. If a stream on the far side stalls, the routing is not what + /// the source says it is, and the older rule — which asked only whether + /// everything went quiet — would have called that its cleanest possible + /// pass. + UnexpectedStall, + /// A stream a delay was aimed at ran no slower than it did at rest. + /// + /// The latency equivalent of [`Self::OutageNotObserved`]. Where a cut + /// proves it landed by silence, a delay can only prove it by time, and a + /// netem rule that failed to apply leaves a run full of healthy numbers and + /// no error anywhere — the worst artifact this suite can produce. + SlowdownNotObserved, + /// A stream a delay was expected to leave alone slowed down with it. + /// + /// The counterpart to [`Self::SlowdownNotObserved`], and the same argument + /// [`Self::UnexpectedStall`] makes for a cut. A delay aimed at one half of + /// the key-value layer is a claim about routing: these namespaces go to the + /// store being slowed and those go elsewhere. A stream on the far side + /// moving with it says something is wrong with that picture — and a rule + /// that only looked for slowdown would have read it as its cleanest + /// possible pass, because everything the run aimed at did indeed get + /// slower. + /// + /// It says *something* rather than *what*, and S15's first run is why the + /// wording is careful. `ephemeral` came out at 234.83x there, which cannot + /// be a route it takes itself: 46 seconds is ninety 500ms round trips, and + /// the stream makes none. It was queueing behind the three streams that do, + /// through a gate the executor shares across an account. So the three + /// readings are that the routing is wrong, that the fault landed wider than + /// the manifest names, or that the platform couples streams that the + /// storage layer keeps apart — and the third is not the least interesting. + UnexpectedSlowdown, + /// A stream that was confirming operations before the outage confirmed + /// nothing at all after the heal. + StreamNeverRecovered, +} + +impl OutageViolation { + pub fn as_str(self) -> &'static str { + match self { + OutageViolation::OutageNotObserved => "outage-not-observed", + OutageViolation::UnexpectedStall => "unexpected-stall", + OutageViolation::SlowdownNotObserved => "slowdown-not-observed", + OutageViolation::UnexpectedSlowdown => "unexpected-slowdown", + OutageViolation::StreamNeverRecovered => "stream-never-recovered", + } + } +} + +impl std::fmt::Display for OutageViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, against one stream. The aggregate verdict carries no stream. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OutageFinding { + pub violation: OutageViolation, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream: Option, + pub detail: String, +} + +/// What one stream managed in one window. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamThroughputCell { + pub stream: Stream, + pub window: Window, + /// Agents of this stream that offered at least one operation in this + /// window. Below the stream's pool size means emitters were stalled across + /// the whole window rather than merely slowed. + pub agents_active: usize, + /// Operations *offered* in this window, and how they eventually ended up. + /// Attributed by submission time, so an operation counted here may not have + /// been answered until a later window. + pub submitted: u64, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Operations *answered* in this window, whenever they were offered. + /// + /// This is the one that says whether the platform was serving, and it is + /// deliberately not `confirmed`: during an outage the two differ by exactly + /// the work that was accepted while the storage was gone and answered only + /// once it came back. + pub served: u64, + /// Attempts that hit the client's attempt timeout rather than answering. + pub attempts_timed_out: u64, + pub window_secs: f64, + pub served_per_sec: f64, + /// This cell's rate against the same stream's own before-fault rate. `None` + /// for the before-fault cell itself, and for a stream that never had a + /// baseline to compare against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_of_baseline_percent: Option, + /// The longest the stream answered nothing at all, anywhere in this window. + /// + /// The number that stops a small non-zero during-fault rate being read as + /// residual service. Measured against the window's own edges, so a stream + /// that fell silent at the start or stayed silent to the end is caught by + /// it too. A `quietMs` close to `windowSecs` is a total outage however the + /// rate arithmetic came out, and it needs no threshold to say so. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quiet_ms: Option, + pub latency: LatencyStats, +} + +/// The operations the outage began underneath. +/// +/// These were submitted before the storage went away and were still running +/// when it did, which makes them the population at risk: the platform may have +/// executed them, may have half-executed them, and cannot tell the client +/// which. They cannot be read off the cells, because every trace of them is +/// attributed to the `before-fault` row they were submitted in. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamCaughtInFlight { + pub stream: Stream, + pub operations: u64, + /// Distinct agents they belonged to. + pub agents: usize, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Submission to final outcome, across every attempt. + pub duration: LatencyStats, + pub attempts_timed_out: u64, + /// The most attempts any one of them needed. An operation that stalled and + /// then answered on a later attempt was rescued by the caller's retry, not + /// returned by the platform, and the outcome alone cannot say so. + pub max_attempts: u32, + /// How many were still unresolved when the storage was reported reachable + /// again. + pub outlived_the_fault: u64, +} + +/// How long one stream took to serve anything again. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamRecovery { + pub stream: Stream, + /// Milliseconds from the heal to this stream's first confirmed operation. + /// `None` means it never confirmed one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub first_confirmed_ms: Option, + /// Whether that exceeded the configured budget. Recorded, not asserted. + pub over_budget: bool, +} + +/// How operations failed while the storage was unreachable. +/// +/// The acceptance criteria ask for fault-window failures, and a count alone +/// does not answer the question an operator has: whether the platform refused +/// the work definitively, or accepted it and then lost the ability to say what +/// happened. That is exactly the [`ErrorClass`] split, so the histogram is +/// keyed on it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FaultWindowErrors { + pub stream: Stream, + pub class: ErrorClass, + pub operations: u64, + /// One message, for the operator to paste into a log query. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub example: Option, +} + +/// The storage-outage account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageFaultReport { + /// The endpoint the workflow was asked to cut the executors off from, + /// recorded so an archived result says which storage the run was about + /// rather than leaving it to the scenario name. + pub endpoint: String, + /// The rule from the suite YAML, recorded whole so an archived cell can be + /// read years later against what it was judged by rather than against + /// today's config — including *which* rule, since the storage scenarios no + /// longer share one. + pub expect: OutageExpectation, + pub recovery_budget_ms: u64, + /// The whole workload's during-fault rate as a share of its own baseline. + /// `None` for a run that never learned when the fault was. + /// + /// Recorded, no longer the verdict. It is a rate averaged over the whole + /// fault window, so it moves with the window length even when the platform + /// behaves identically. See `outage_quiet_floor_percent`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_of_baseline_percent: Option, + /// The least any stream *expected to stop* stayed silent during the fault, + /// as a share of that window. This is what the quiet half of the verdict is + /// drawn from. + /// + /// Under `WholeWorkload` that is every stream, so this is the run-wide + /// minimum and means what it always did. Under `PartialWorkload` it covers + /// only the named streams, because the others are expected to keep + /// answering and their quiet time says nothing about whether the cut + /// landed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quietest_stream_percent: Option, + /// The least of its own baseline any stream *expected to keep serving* held + /// during the fault. `None` under `WholeWorkload`, which expects none to. + /// + /// The other half of a partial cut's verdict, and the half no shared rule + /// could state: a partial cut claims the streams it does not touch carry on, + /// and that claim comes from where the code routes each namespace rather + /// than from an assumption about faults. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub least_serving_stream_percent: Option, + /// The least any stream *expected to slow down* did, as a multiple of its + /// own before-fault median latency. `None` unless the expectation is + /// [`OutageExpectation::LatencyDegradation`]. + /// + /// A multiple of the stream's own baseline rather than a millisecond + /// figure, because the streams differ by an order of magnitude at rest and + /// a single absolute threshold would be slack for one and unreachable for + /// another. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub least_slowdown_factor: Option, + /// The most any stream the delay was expected *not* to reach moved, as a + /// multiple of its own before-fault median. `None` unless the expectation + /// names steady streams. + /// + /// Read together with `least_slowdown_factor`, the pair is the whole + /// experiment in two numbers: how far the streams behind the delayed store + /// moved, and how far the ones that should not be behind it did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub greatest_steady_factor: Option, + pub cells: Vec, + /// What the outage began underneath, per stream. Empty for a run that never + /// learned when the fault was. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub caught_in_flight: Vec, + pub recovery: Vec, + pub fault_window_errors: Vec, + pub findings: Vec, +} + +/// One stream's per-window accumulation, before it becomes a cell. +#[derive(Default)] +struct Tally { + agents: BTreeSet, + submitted: u64, + confirmed: u64, + rejected: u64, + indeterminate: u64, + attempts_timed_out: u64, + durations: Vec, + /// When this stream actually answered inside this window, sorted later. + /// + /// Keyed on the window a confirmation *landed* in rather than the one its + /// operation was offered in, which is the only way either of the numbers + /// derived from it means what it says. + served_at: Vec>, +} + +impl StorageFaultReport { + /// Builds the account from the operation history. + /// + /// `fault` is what the workflow reported. Without it every record lands in + /// [`Window::Unknown`] and the report carries counts but no verdict, which + /// is the honest outcome for a run that never learned when the fault was: + /// both thresholds are defined relative to a before-and-during comparison + /// that cannot be made. + pub fn build( + records: &[OperationRecord], + fault: Option, + endpoint: &str, + expect: OutageExpectation, + recovery_budget: Duration, + ) -> Self { + let mut tallies: BTreeMap<(Stream, Window), Tally> = BTreeMap::new(); + let mut first_submitted: Option> = None; + let mut last_completed: Option> = None; + + for record in records { + let window = Window::of(record.submitted_at, fault); + let tally = tallies.entry((record.stream, window)).or_default(); + + tally.agents.insert(record.agent.clone()); + tally.submitted += 1; + match record.outcome { + Outcome::Confirmed => { + tally.confirmed += 1; + tally.durations.push(record.duration_ms); + } + Outcome::Rejected => tally.rejected += 1, + Outcome::Indeterminate => tally.indeterminate += 1, + } + tally.attempts_timed_out += record.attempts_timed_out(); + + // Answered work is filed under the window it was answered in, which + // is usually but not always the one it was offered in. An operation + // offered during the outage and answered after the heal is service + // the fault window did not get, and counting it there is how a total + // outage reads as partial service. + if record.outcome == Outcome::Confirmed + && let Some(completed) = record.completed_at + { + tallies + .entry((record.stream, Window::of(completed, fault))) + .or_default() + .served_at + .push(completed); + } + + first_submitted = Some(match first_submitted { + Some(at) if at <= record.submitted_at => at, + _ => record.submitted_at, + }); + if let Some(completed) = record.completed_at { + last_completed = Some(match last_completed { + Some(at) if at >= completed => at, + _ => completed, + }); + } + } + + // Baselines first: every other cell is expressed as a share of its own + // stream's before-fault rate, so a stream the workload drives rarely + // cannot make the picture look better or worse than it was. + let mut baseline_rate: BTreeMap = BTreeMap::new(); + let mut cells: Vec = Vec::new(); + for ((stream, window), tally) in &tallies { + let secs = window_secs(*window, fault, first_submitted, last_completed); + let served = tally.served_at.len() as u64; + let rate = if secs > 0.0 { + served as f64 / secs + } else { + 0.0 + }; + if *window == Window::BeforeFault { + baseline_rate.insert(*stream, rate); + } + let quiet_ms = longest_silence_ms( + &tally.served_at, + window_start(*window, fault, first_submitted), + window_end(*window, fault, last_completed), + ); + cells.push(StreamThroughputCell { + stream: *stream, + window: *window, + agents_active: tally.agents.len(), + submitted: tally.submitted, + confirmed: tally.confirmed, + served, + rejected: tally.rejected, + indeterminate: tally.indeterminate, + attempts_timed_out: tally.attempts_timed_out, + window_secs: round2(secs), + served_per_sec: round2(rate), + share_of_baseline_percent: None, + quiet_ms, + latency: LatencyStats::from_durations(tally.durations.clone()), + }); + } + + for cell in &mut cells { + if cell.window == Window::BeforeFault { + continue; + } + if let Some(base) = baseline_rate.get(&cell.stream).copied() + && base > 0.0 + { + cell.share_of_baseline_percent = Some(round2(cell.served_per_sec / base * 100.0)); + } + } + cells.sort_by_key(|c| (c.stream, c.window)); + + let mut report = Self { + endpoint: endpoint.to_string(), + expect, + recovery_budget_ms: recovery_budget.as_millis().min(u64::MAX as u128) as u64, + share_of_baseline_percent: None, + quietest_stream_percent: None, + least_serving_stream_percent: None, + least_slowdown_factor: None, + greatest_steady_factor: None, + cells, + caught_in_flight: caught_in_flight(records, fault), + recovery: Vec::new(), + fault_window_errors: fault_window_errors(records, fault), + findings: Vec::new(), + }; + + report.judge_outage(fault, first_submitted, last_completed); + report.judge_recovery(records, fault, recovery_budget, &baseline_rate); + report + } + + /// Did the outage land? + /// + /// Judged on how long each stream answered *nothing*, as a share of the + /// fault window, and on the stream that managed it least. A stream still + /// serving is a fault that did not land on it, and averaging would let the + /// quiet ones outvote it. + /// + /// Quiet time rather than throughput because throughput is a rate over the + /// whole window while an absorbed outage does all its serving in the + /// seconds at the window's edges: the same handful of confirmations reads + /// as a small share of a long window and a large share of a short one, so a + /// throughput threshold tracks the window length rather than the platform. + /// Quiet time is measured against the window's own edges and does not move + /// when the window does. The share is still computed, and still reported, + /// as context. + /// + /// Streams with no before-fault serving are skipped. A stream the run never + /// got going says nothing about whether the storage went away, and the + /// driver's baseline gate already reports it. + fn judge_outage( + &mut self, + fault: Option, + first_submitted: Option>, + last_completed: Option>, + ) { + if fault.is_none() { + return; + } + let total = |window: Window| -> f64 { + let served: u64 = self + .cells + .iter() + .filter(|c| c.window == window) + .map(|c| c.served) + .sum(); + let secs = window_secs(window, fault, first_submitted, last_completed); + if secs > 0.0 { + served as f64 / secs + } else { + 0.0 + } + }; + + let baseline = total(Window::BeforeFault); + if baseline <= 0.0 { + return; + } + self.share_of_baseline_percent = + Some(round2(total(Window::DuringFault) / baseline * 100.0)); + + let served_before: BTreeSet = self + .cells + .iter() + .filter(|c| c.window == Window::BeforeFault && c.served > 0) + .map(|c| c.stream) + .collect(); + + // Split by what this scenario claims the cut does, not by what a cut + // does in general. The two sets are judged by opposite tests, and a + // stream in neither — one that never served before the fault — is + // judged by neither, because it has no baseline to be read against. + let during: Vec<&StreamThroughputCell> = self + .cells + .iter() + .filter(|c| c.window == Window::DuringFault && c.window_secs > 0.0) + .filter(|c| served_before.contains(&c.stream)) + .collect(); + + // Driven from `served_before` for the same reason as the serving list + // below: a stream that answered nothing at all leaves no during-fault + // cell, and that is total silence rather than missing data. Scoring the + // two sides of one fact differently — absence as 0% served but as + // "unknown" quiet — would let the same run read as a stall on one check + // and as no evidence on the other. + let mut quiet: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| self.expect.expects_silence(*stream)) + .filter_map(|stream| match during.iter().find(|c| c.stream == stream) { + Some(cell) => cell + .quiet_ms + .map(|ms| (stream, round2(ms as f64 / (cell.window_secs * 10.0)))), + None => Some((stream, 100.0)), + }) + .collect(); + quiet.sort_by(|a, b| a.1.total_cmp(&b.1)); + + if let Some(&(stream, quietest)) = quiet.first() { + self.quietest_stream_percent = Some(quietest); + let floor = self.expect.quiet_floor_percent(); + if quietest < floor { + self.findings.push(OutageFinding { + violation: OutageViolation::OutageNotObserved, + stream: Some(stream), + detail: format!( + "{stream} was expected to stop while {} was unreachable and kept \ + answering instead, silent for only {quietest}% of the fault window \ + against a {floor}% floor — so this run has no evidence the cut landed", + self.endpoint + ), + }); + } + } + + // The latency half, before the serving one, because a delay that never + // applied makes everything below it uninteresting: the streams would + // all be serving normally and the run would read as a clean pass of an + // experiment that never happened. + if let Some(slowdown_floor) = self.expect.slowdown_floor() { + let mut slowdowns: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| self.expect.expects_slowdown(*stream)) + .filter_map(|stream| { + // Each stream against its own before-fault median. Medians + // rather than means because a handful of retried operations + // drag a mean far enough to hide whether the typical + // operation moved at all, which is the question. + let baseline = self.median_ms(stream, Window::BeforeFault)?; + let during = self.median_ms(stream, Window::DuringFault)?; + (baseline > 0.0).then_some((stream, round2(during / baseline))) + }) + .collect(); + slowdowns.sort_by(|a, b| a.1.total_cmp(&b.1)); + + if let Some(&(stream, least)) = slowdowns.first() { + self.least_slowdown_factor = Some(least); + if least < slowdown_floor { + self.findings.push(OutageFinding { + violation: OutageViolation::SlowdownNotObserved, + stream: Some(stream), + detail: format!( + "{stream} was expected to slow down while {} was delayed and ran at \ + {least}x its own baseline median against a {slowdown_floor}x floor \ + — so this run has no evidence the delay reached the executors", + self.endpoint + ), + }); + } + } + } + + // The far side of the same claim. A delay aimed at one store predicts + // both which streams move and which do not, and only the second half + // can catch a delay that landed somewhere wider than the run says. + if let Some(steady_ceiling) = self.expect.steady_ceiling() { + let mut steadies: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| self.expect.expects_steady(*stream)) + .filter_map(|stream| { + let baseline = self.median_ms(stream, Window::BeforeFault)?; + let during = self.median_ms(stream, Window::DuringFault)?; + (baseline > 0.0).then_some((stream, round2(during / baseline))) + }) + .collect(); + steadies.sort_by(|a, b| b.1.total_cmp(&a.1)); + + if let Some(&(stream, most)) = steadies.first() { + self.greatest_steady_factor = Some(most); + if most > steady_ceiling { + self.findings.push(OutageFinding { + violation: OutageViolation::UnexpectedSlowdown, + stream: Some(stream), + detail: format!( + "{stream} does not depend on {} and was expected to run at its own \ + pace through the delay, but ran at {most}x its own baseline median \ + against a {steady_ceiling}x ceiling — so either it reaches that \ + storage after all, or it is queueing behind the streams that do", + self.endpoint + ), + }); + } + } + } + + let Some(serving_floor) = self.expect.serving_floor_percent() else { + return; + }; + // Driven from the streams that served *before* the cut rather than from + // the during-fault cells, because the worst case leaves no such cell at + // all: a stream whose operations all stall produces nothing to tally, + // and reading the cells alone would let total silence escape the one + // check that exists to catch it. Absence is the strongest evidence + // here, so it is scored as zero rather than skipped. + let mut serving: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| !self.expect.expects_silence(*stream)) + .map(|stream| { + let share = during + .iter() + .find(|c| c.stream == stream) + .and_then(|c| c.share_of_baseline_percent) + .unwrap_or(0.0); + (stream, share) + }) + .collect(); + serving.sort_by(|a, b| a.1.total_cmp(&b.1)); + + if let Some(&(stream, least)) = serving.first() { + self.least_serving_stream_percent = Some(least); + // Under a delay the two floors are not independent, and treating + // them as though they were reports the same fact twice. + // + // Throughput is concurrency over latency, and the workload offers a + // fixed in-flight budget per stream. So once a stream's latency + // rises far enough to saturate that budget, its throughput *must* + // fall in proportion, and the serving floor is then measuring the + // slowdown the run already reported rather than anything new. S15's + // first run made this concrete: promise held 11.88% of baseline + // while running 939x slower, which is 108 times more work in flight + // than at rest, not a stall. + // + // The invariant that separates them: share × slowdown is the ratio + // of during-fault concurrency to baseline concurrency. At or above + // 100% the stream carried more work than it did at rest and did not + // stop, whatever its throughput did. Below it, work is genuinely + // being lost, which is what this finding exists for. + let explained_by_slowdown = self + .expect + .slowdown_floor() + .and(self.median_ms(stream, Window::BeforeFault)) + .zip(self.median_ms(stream, Window::DuringFault)) + .map(|(before, during)| before > 0.0 && least * (during / before) >= 100.0) + .unwrap_or(false); + if least < serving_floor && !explained_by_slowdown { + let detail = if self.expect.slowdown_floor().is_some() { + format!( + "{stream} was expected to slow down while {} was delayed rather than \ + stop, but held only {least}% of its own baseline against a \ + {serving_floor}% floor, and its own latency does not account for the \ + shortfall — so the delay cost this stream work and not only time", + self.endpoint + ) + } else { + format!( + "{stream} does not depend on {} and was expected to carry on through \ + the cut, but held only {least}% of its own baseline against a \ + {serving_floor}% floor — the storage this stream actually reaches is \ + not what the routing says it is", + self.endpoint + ) + }; + self.findings.push(OutageFinding { + violation: OutageViolation::UnexpectedStall, + stream: Some(stream), + detail, + }); + } + } + } + + /// How long each stream took to serve again, and which never did. + /// + /// Only streams that were working *before* the outage are judged. A stream + /// the run never got going is a different problem, already reported by the + /// baseline gate in the driver, and calling it unrecovered here would put + /// the same failure in two places under two names. + fn judge_recovery( + &mut self, + records: &[OperationRecord], + fault: Option, + budget: Duration, + baseline_rate: &BTreeMap, + ) { + let Some(FaultWindow { + recovered_at: Some(recovered), + .. + }) = fault + else { + return; + }; + let budget_ms = budget.as_millis().min(i64::MAX as u128) as i64; + + for (stream, base) in baseline_rate { + if *base <= 0.0 { + continue; + } + let first = records + .iter() + .filter(|r| r.stream == *stream && r.outcome == Outcome::Confirmed) + .filter_map(|r| r.completed_at) + .filter(|at| *at >= recovered) + .min(); + + let first_confirmed_ms = + first.map(|at| (at - recovered).num_milliseconds().max(0) as u64); + let over_budget = first + .map(|at| (at - recovered).num_milliseconds() > budget_ms) + .unwrap_or(false); + + if first.is_none() { + self.findings.push(OutageFinding { + violation: OutageViolation::StreamNeverRecovered, + stream: Some(*stream), + detail: format!( + "the {stream} stream confirmed operations before the outage and none \ + after {} became reachable again", + self.endpoint + ), + }); + } + self.recovery.push(StreamRecovery { + stream: *stream, + first_confirmed_ms, + over_budget, + }); + } + } + + /// One stream's cell for one window, for the tests and for anything reading + /// the report without wanting to scan the whole list. + pub fn cell(&self, stream: Stream, window: Window) -> Option<&StreamThroughputCell> { + self.cells + .iter() + .find(|c| c.stream == stream && c.window == window) + } + + /// Whether the report found anything a human has to act on. + pub fn has_findings(&self) -> bool { + !self.findings.is_empty() + } + + /// Lines that need a human. Empty on a run where the outage landed and + /// everything came back. + pub fn attention_lines(&self) -> Vec { + self.findings + .iter() + .map(|f| match f.stream { + Some(stream) => format!("{}: {stream}: {}", f.violation, f.detail), + None => format!("{}: {}", f.violation, f.detail), + }) + .collect() + } + + /// One stream's median latency in a window, if it has a cell with samples. + fn median_ms(&self, stream: Stream, window: Window) -> Option { + self.cells + .iter() + .find(|c| c.stream == stream && c.window == window) + .filter(|c| c.latency.count > 0) + .map(|c| c.latency.p50_ms as f64) + } + + /// Context a reader needs in order to read the cells, which is not itself a + /// problem. + pub fn note_lines(&self) -> Vec { + let mut lines = Vec::new(); + if let Some(share) = self.share_of_baseline_percent { + // Says which streams the numbers are about, because under a partial + // cut they are about a subset and a reader who assumes otherwise + // draws the opposite conclusion from the same figure. + // Under a latency expectation nothing is meant to go quiet, so the + // quiet figure is absent by design rather than missing, and the + // slowdown takes its place as what says the fault landed. + let quiet = match (self.quietest_stream_percent, self.least_slowdown_factor) { + // A control run: the expectation is a delay, but it names no + // stream that should feel it, so there is no slowdown to report + // and its absence is the result rather than missing data. + (None, None) if self.expect.slowdown_floor().is_some() => { + "no stream in this run was expected to slow down, so whether the delay \ + landed has to be read from the storage metrics rather than from here" + .to_string() + } + (_, Some(factor)) => format!( + "the streams expected to slow down ran at least {factor}x their own baseline \ + median (floor {}x)", + self.expect.slowdown_floor().unwrap_or_default() + ), + (Some(q), None) => format!( + "the streams expected to stop were silent for at least {q}% of the fault \ + window (floor {}%)", + self.expect.quiet_floor_percent() + ), + (None, None) => { + "no stream had a before-fault baseline to be judged against".to_string() + } + }; + // The far side of a delay, when the scenario named one. Printed + // next to the slowdown rather than left in the cells, because the + // two numbers only mean anything together: a run where everything + // moved by the same multiple measured a slower cluster, not a + // slower store. + let steady = match (self.greatest_steady_factor, self.expect.steady_ceiling()) { + (Some(most), Some(ceiling)) => format!( + ", the streams expected to be left alone ran at no more than {most}x theirs \ + (ceiling {ceiling}x)" + ), + _ => String::new(), + }; + let serving = match ( + self.least_serving_stream_percent, + self.expect.serving_floor_percent(), + ) { + (Some(least), Some(floor)) => format!( + ", the streams expected to carry on held at least {least}% of their own \ + baseline (floor {floor}%)" + ), + _ => String::new(), + }; + lines.push(format!( + "Storage outage: {quiet}{steady}{serving} while {} was unreachable, and the workload held \ + {share}% of its baseline throughput across that window", + self.endpoint + )); + } else { + lines.push(format!( + "Storage outage: no fault window was reported, so the {} cells carry counts but \ + no verdict", + self.endpoint + )); + } + let over: Vec = self + .recovery + .iter() + .filter(|r| r.over_budget) + .map(|r| { + format!( + "{} ({}ms)", + r.stream, + r.first_confirmed_ms.unwrap_or_default() + ) + }) + .collect(); + if !over.is_empty() { + lines.push(format!( + "Storage outage: {} took longer than the {}ms recovery budget to serve again: {}", + over.len(), + self.recovery_budget_ms, + over.join(", ") + )); + } + lines + } +} + +/// The operations that were submitted before the outage and still running when +/// it started. +fn caught_in_flight( + records: &[OperationRecord], + fault: Option, +) -> Vec { + let Some(window) = fault else { + return Vec::new(); + }; + let mut by_stream: BTreeMap> = BTreeMap::new(); + for record in records { + // Submitted before the cut, and either still unfinished when it landed + // or finished after it. An operation with no completion at all is + // included: the driver never learned how it ended, which is the same + // doubt in a starker form. + if record.submitted_at >= window.injected_at { + continue; + } + let still_running = record + .completed_at + .map(|at| at >= window.injected_at) + .unwrap_or(true); + if still_running { + by_stream.entry(record.stream).or_default().push(record); + } + } + + by_stream + .into_iter() + .map(|(stream, caught)| StreamCaughtInFlight { + stream, + operations: caught.len() as u64, + agents: caught + .iter() + .map(|r| r.agent.as_str()) + .collect::>() + .len(), + confirmed: count_of(&caught, Outcome::Confirmed), + rejected: count_of(&caught, Outcome::Rejected), + indeterminate: count_of(&caught, Outcome::Indeterminate), + duration: LatencyStats::from_durations( + caught.iter().map(|r| r.duration_ms).collect::>(), + ), + attempts_timed_out: caught.iter().map(|r| r.attempts_timed_out()).sum(), + max_attempts: caught.iter().map(|r| r.attempts).max().unwrap_or(0), + outlived_the_fault: caught + .iter() + .filter(|r| match (r.completed_at, window.recovered_at) { + (Some(at), Some(recovered)) => at >= recovered, + (None, _) => true, + _ => false, + }) + .count() as u64, + }) + .collect() +} + +/// How the operations submitted during the outage failed, by stream and class. +fn fault_window_errors( + records: &[OperationRecord], + fault: Option, +) -> Vec { + let mut tallies: BTreeMap<(Stream, ErrorClass), (u64, Option)> = BTreeMap::new(); + for record in records { + if Window::of(record.submitted_at, fault) != Window::DuringFault { + continue; + } + let Some(class) = record.error_class else { + continue; + }; + let entry = tallies.entry((record.stream, class)).or_insert((0, None)); + entry.0 += 1; + if entry.1.is_none() { + entry.1.clone_from(&record.error); + } + } + + let mut rows: Vec = tallies + .into_iter() + .map( + |((stream, class), (operations, example))| FaultWindowErrors { + stream, + class, + operations, + example, + }, + ) + .collect(); + // Commonest first: an operator reading this wants the dominant failure mode + // before the long tail. + rows.sort_by(|a, b| { + b.operations + .cmp(&a.operations) + .then((a.stream, a.class).cmp(&(b.stream, b.class))) + }); + rows +} + +fn count_of(records: &[&OperationRecord], outcome: Outcome) -> u64 { + records.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{AttemptRecord, Phase}; + use chrono::TimeDelta; + use test_r::test; + + const ENDPOINT: &str = "golem-postgres-dev-keyvalue.cluster-example.rds.amazonaws.com"; + const QUIET_FLOOR: f64 = 50.0; + const AGENT: &str = "chaos-s16-durable-0000"; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-25T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(180)), + } + } + + /// One operation, submitted `offset` seconds from the moment the storage + /// was taken away. Negative offsets are the baseline. + fn op(stream: Stream, offset_secs: i64, outcome: Outcome) -> OperationRecord { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + OperationRecord { + op_id: 0, + stream, + phase: Phase::Baseline, + agent: AGENT.to_string(), + method: "increment".to_string(), + idempotency_key: format!("{stream}-{offset_secs}"), + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(20)), + attempts: 1, + outcome, + duration_ms: 20, + returned_value: Some(1), + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 20, + returned_value: Some(1), + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + /// An operation that hung until the client gave up, twice: the shape every + /// invocation takes while the database is unreachable. + fn stalled(stream: Stream, offset_secs: i64) -> OperationRecord { + let mut record = op(stream, offset_secs, Outcome::Indeterminate); + record.duration_ms = 245_000; + record.completed_at = Some(record.submitted_at + TimeDelta::seconds(245)); + record.returned_value = None; + record.attempts = 2; + record.error_class = Some(ErrorClass::Transport); + record.error = Some("attempt timed out after 120s".to_string()); + record.attempt_log = (1..=2) + .map(|attempt| AttemptRecord { + attempt, + started_at: record.submitted_at, + duration_ms: 120_000, + returned_value: None, + succeeded: false, + error_class: Some(ErrorClass::Transport), + error: Some("attempt timed out after 120s".to_string()), + }) + .collect(); + record + } + + /// 300s of baseline on both streams, then `during` confirmations while the + /// storage is gone, then 240s of recovery. + fn history(during: usize, recovery: bool) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, -second, Outcome::Confirmed)); + } + for i in 0..during { + records.push(op(Stream::Durable, i as i64, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, i as i64, Outcome::Confirmed)); + } + if recovery { + for second in 181..=420 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, second, Outcome::Confirmed)); + } + } + records + } + + const SERVING_FLOOR: f64 = 50.0; + + /// A workload shaped like S18's: `durable` on the far side of the cut and + /// `ephemeral` behind it. `durable_during` and `ephemeral_during` say how + /// many confirmations each managed while the storage was gone, so one + /// helper covers "behaved as designed", "the wrong stream stalled" and + /// "nothing stopped at all". + fn partial_history(durable_during: usize, ephemeral_during: usize) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, -second, Outcome::Confirmed)); + } + for i in 0..durable_during { + records.push(op(Stream::Durable, i as i64, Outcome::Confirmed)); + } + for i in 0..ephemeral_during { + records.push(op(Stream::Ephemeral, i as i64, Outcome::Confirmed)); + } + for second in 181..=420 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, second, Outcome::Confirmed)); + } + records + } + + /// The same shape as [`partial_history`], but with the operations that did + /// not confirm still *recorded* as stalled attempts. + /// + /// This is what a real run looks like: the workload keeps submitting into a + /// stall, so the stream has a during-fault cell showing what it offered and + /// nothing served. S18's first run recorded 321 such ephemeral operations. + /// [`partial_history`] covers the other shape, where a stream produced no + /// during-fault record at all, and the two must reach the same verdict. + fn partial_history_with_stalls( + durable_confirmed: usize, + ephemeral_confirmed: usize, + ) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, -second, Outcome::Confirmed)); + } + for (stream, confirmed) in [ + (Stream::Durable, durable_confirmed), + (Stream::Ephemeral, ephemeral_confirmed), + ] { + for i in 0..180 { + if i < confirmed { + records.push(op(stream, i as i64, Outcome::Confirmed)); + } else { + records.push(stalled(stream, i as i64)); + } + } + } + for second in 181..=420 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, second, Outcome::Confirmed)); + } + records + } + + const SLOWDOWN_FLOOR: f64 = 2.0; + const STEADY_CEILING: f64 = 1.5; + + /// A workload where every stream keeps serving and `ephemeral` is the one + /// the delay is aimed at. `ephemeral_ms` is what its operations take during + /// the fault, against a 200ms baseline. + fn latency_history(ephemeral_ms: u64, ephemeral_during: usize) -> Vec { + latency_history_with(ephemeral_ms, ephemeral_during, 50) + } + + /// As [`latency_history`], with the far-side stream's during-fault duration + /// under the test's control too, so a run where the delay reached more than + /// it was aimed at can be built. + fn latency_history_with( + ephemeral_ms: u64, + ephemeral_during: usize, + durable_during_ms: u64, + ) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(timed(Stream::Durable, -second, 50)); + records.push(timed(Stream::Ephemeral, -second, 200)); + } + for i in 0..180 { + records.push(timed(Stream::Durable, i as i64, durable_during_ms)); + } + for i in 0..ephemeral_during { + records.push(timed(Stream::Ephemeral, i as i64, ephemeral_ms)); + } + for second in 181..=420 { + records.push(timed(Stream::Durable, second, 50)); + records.push(timed(Stream::Ephemeral, second, 200)); + } + records + } + + /// A confirmed operation that took a stated time, so a window's median is + /// something the test controls rather than something it inherits. + fn timed(stream: Stream, offset_secs: i64, duration_ms: u64) -> OperationRecord { + let mut record = op(stream, offset_secs, Outcome::Confirmed); + record.duration_ms = duration_ms; + record.completed_at = + Some(record.submitted_at + TimeDelta::milliseconds(duration_ms as i64)); + record + } + + fn build_latency(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::LatencyDegradation { + slowed: vec![Stream::Ephemeral], + slowdown_floor: SLOWDOWN_FLOOR, + steady: Vec::new(), + steady_ceiling: STEADY_CEILING, + serving_floor_percent: SERVING_FLOOR, + }, + Duration::from_secs(120), + ) + } + + /// The same delay, with the claim S15 and S17 both make about the streams + /// on the far side of the store written down. + fn build_latency_steady(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::LatencyDegradation { + slowed: vec![Stream::Ephemeral], + slowdown_floor: SLOWDOWN_FLOOR, + steady: vec![Stream::Durable], + steady_ceiling: STEADY_CEILING, + serving_floor_percent: SERVING_FLOOR, + }, + Duration::from_secs(120), + ) + } + + fn build_partial(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::PartialWorkload { + silenced: vec![Stream::Ephemeral], + quiet_floor_percent: QUIET_FLOOR, + serving_floor_percent: SERVING_FLOOR, + }, + Duration::from_secs(120), + ) + } + + fn build(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::WholeWorkload { + quiet_floor_percent: QUIET_FLOOR, + }, + Duration::from_secs(120), + ) + } + + /// The healthy shape: throughput collapses while the database is gone and + /// comes back afterwards, and the report raises nothing. + #[test] + fn an_outage_that_lands_and_heals_has_no_findings() { + let report = build(&history(2, true)); + + assert!( + report.findings.is_empty(), + "expected no findings, got {:?}", + report.findings + ); + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert!( + report.quietest_stream_percent.unwrap() > QUIET_FLOOR, + "the streams should have gone silent, got {:?} from {during:?}", + report.quietest_stream_percent + ); + } + + /// The one thing this scenario cannot afford to be quiet about. A partition + /// that never took hold leaves every other number describing a cluster + /// nothing happened to, and a reader has to be told rather than left to + /// infer it from cells that all look fine. + #[test] + fn a_workload_that_kept_working_reports_the_outage_as_not_observed() { + // 180 confirmations per stream across the 180s fault window is the + // baseline cadence carrying straight on through it. + let report = build(&history(180, true)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "expected an outage-not-observed finding, got {:?}", + report.findings + ); + assert!( + report.quietest_stream_percent.unwrap() < QUIET_FLOOR, + "a stream answering every second is never quiet for long, got {:?}", + report.quietest_stream_percent + ); + } + + /// The regression S18's first run produced, expressed as a test. + /// + /// Under a partial cut one stream stops and the others carry on by design. + /// The whole-workload rule reads that as a fault that never landed, because + /// the only question it can ask is whether *everything* went quiet. Run + /// 33130077355 hit exactly this: `durable` held 100% of its baseline while + /// the Redis partition had plainly landed on `ephemeral`. + #[test] + fn a_partial_cut_is_not_judged_by_the_streams_it_was_never_going_to_stop() { + let records = partial_history(180, 0); + + let shared = build(&records); + assert!( + shared + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "the whole-workload rule is expected to misjudge this, or the test guards nothing" + ); + + let report = build_partial(&records); + assert!( + report.findings.is_empty(), + "one stream stopping and the rest carrying on is the expected outcome here, got {:?}", + report.findings + ); + } + + /// The half no shared rule could state. + /// + /// A partial cut claims the streams it does not touch keep working, and + /// that claim comes from where the code routes each namespace rather than + /// from an assumption about faults. A stall on the far side means the + /// routing is not what the source says, and the older rule would have + /// called it the cleanest possible pass: everything went quiet. + #[test] + fn a_stream_the_cut_should_not_have_touched_stalling_is_a_finding() { + let report = build_partial(&partial_history(0, 0)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall), + "durable does not depend on this endpoint and stopped anyway, got {:?}", + report.findings + ); + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "ephemeral did stop, so the cut was observed; got {:?}", + report.findings + ); + } + + /// The named stream refusing to stop still fails, which is the check the + /// partial rule inherits rather than replaces. + #[test] + fn a_partial_cut_whose_named_stream_kept_serving_is_still_not_observed() { + let report = build_partial(&partial_history(180, 180)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved + && f.stream == Some(Stream::Ephemeral)), + "ephemeral was the stream expected to stop, got {:?}", + report.findings + ); + } + + /// Both numbers are reported, and each covers only the streams its own rule + /// judges. Reading the quiet figure as run-wide is how the first S18 report + /// told its reader to draw the wrong conclusion. + #[test] + fn each_reported_share_covers_only_the_streams_its_rule_judges() { + let report = build_partial(&partial_history(180, 0)); + + assert!( + report.quietest_stream_percent.unwrap() > QUIET_FLOOR, + "the quiet figure must describe ephemeral, which stopped, not durable, got {:?}", + report.quietest_stream_percent + ); + assert!( + report.least_serving_stream_percent.unwrap() > SERVING_FLOOR, + "the serving figure must describe durable, which carried on, got {:?}", + report.least_serving_stream_percent + ); + } + + /// The same two verdicts against the shape a real run produces, where the + /// stalled operations are recorded rather than absent. A stream that + /// offered work and served none of it must read the same as one that + /// offered nothing at all. + #[test] + fn a_recorded_stall_reads_the_same_as_a_stream_that_went_missing() { + let as_designed = build_partial(&partial_history_with_stalls(180, 0)); + assert!( + as_designed.findings.is_empty(), + "ephemeral stalling and durable carrying on is the expected outcome, got {:?}", + as_designed.findings + ); + + let wrong_stream = build_partial(&partial_history_with_stalls(0, 0)); + assert!( + wrong_stream + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall + && f.stream == Some(Stream::Durable)), + "durable served nothing while off the fault path, got {:?}", + wrong_stream.findings + ); + } + + /// The delay landed and the platform degraded rather than broke, which is + /// the outcome S15 and S17 are written to confirm. + #[test] + fn a_slowed_stream_that_kept_serving_is_the_expected_outcome() { + // 200ms at rest, 1s under the delay, and still completing throughout. + let report = build_latency(&latency_history(1_000, 180)); + + assert!( + report.findings.is_empty(), + "slower but still serving is what a delay is supposed to produce, got {:?}", + report.findings + ); + assert_eq!( + report.least_slowdown_factor, + Some(5.0), + "1000ms against a 200ms baseline is 5x" + ); + assert!( + report.quietest_stream_percent.is_none(), + "nothing is expected to go quiet under a delay, so there is no quiet verdict to draw" + ); + } + + /// The netem rule that never applied. Without this the run is full of + /// healthy numbers and no error anywhere, which is the worst artifact this + /// suite can produce — the same failure `outage-not-observed` exists to + /// catch, in the units a delay is measured in. + #[test] + fn a_delay_that_did_not_reach_the_executors_is_a_finding() { + // 240ms against a 200ms baseline: 1.2x, well under the 2x floor. + let report = build_latency(&latency_history(240, 180)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::SlowdownNotObserved + && f.stream == Some(Stream::Ephemeral)), + "a stream that barely moved cannot evidence a delay, got {:?}", + report.findings + ); + } + + /// A delay is supposed to cost time, not work. A slowed stream that stops + /// entirely means the added latency broke something — a timeout fired, a + /// pool drained — and that is a different and worse outcome than slowness. + #[test] + fn a_delay_that_stopped_a_stream_rather_than_slowing_it_is_a_finding() { + // Slow enough to clear the slowdown floor, but only a trickle of + // operations got through, so it degraded past degradation. + let report = build_latency(&latency_history(1_000, 2)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall), + "a stream serving 2 operations where it served 180 has stopped, got {:?}", + report.findings + ); + } + + /// The throughput drop a delay forces on its own, which is not a second + /// finding. The workload offers each stream a fixed in-flight budget, so a + /// stream running a hundred times slower cannot hold its baseline rate no + /// matter how healthy it is — and reporting that as a stall restates the + /// slowdown under another name. S15's first run hit exactly this: promise + /// held 11.88% of baseline at 939x, which is a hundred times more work in + /// flight than at rest. + #[test] + fn a_stream_too_slow_to_hold_its_rate_has_not_stopped() { + // 100x slower, and serving 5 operations where it served 180. That is + // 2.8% of baseline, far under the 50% floor — and 2.8% x 100 is 278% of + // its baseline concurrency, so it carried more work rather than less. + let report = build_latency(&latency_history(20_000, 5)); + + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall), + "a stream whose own latency accounts for its rate has not stalled, got {:?}", + report.findings + ); + } + + /// Both halves of a delay's claim, held at once: the stream behind the + /// slowed store moved and the stream that is not behind it did not. This is + /// the shape S17 measured and the one S15 predicts with the streams + /// swapped, and it is what makes either run attributable to the store + /// rather than to a slower cluster. + #[test] + fn a_steady_stream_that_stayed_steady_is_what_makes_the_slowdown_attributable() { + let report = build_latency_steady(&latency_history_with(1_000, 180, 50)); + + assert!( + report.findings.is_empty(), + "one stream slower and the other unchanged is the expected outcome, got {:?}", + report.findings + ); + assert_eq!(report.least_slowdown_factor, Some(5.0)); + assert_eq!( + report.greatest_steady_factor, + Some(1.0), + "50ms against a 50ms baseline has not moved" + ); + } + + /// The finding a rule that only looks for slowdown cannot raise. Everything + /// the delay was aimed at did get slower, so the slowdown check passes and + /// the run reads as clean — while a stream the routing says is nowhere near + /// the delayed store moved with it, which means the routing is wrong or the + /// rule landed wider than the manifest says. + #[test] + fn a_stream_the_delay_should_not_have_touched_slowing_with_it_is_a_finding() { + // The delayed stream still clears its floor at 5x, so nothing else in + // the report objects. + let report = build_latency_steady(&latency_history_with(1_000, 180, 550)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedSlowdown + && f.stream == Some(Stream::Durable)), + "11x on a stream that does not depend on the delayed store is a finding, got {:?}", + report.findings + ); + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::SlowdownNotObserved), + "the aimed-at stream did slow down, so that check should be quiet: {:?}", + report.findings + ); + } + + /// A stream the run barely drives cannot flip the verdict on its own. It is + /// judged on how long it was silent, not on its rate against a baseline, so + /// a trickle reads as the near-total silence it is rather than as a stream + /// holding up. + #[test] + fn a_trickle_stream_does_not_make_the_outage_look_unobserved() { + let mut records = history(0, true); + // A trickle of a third stream that keeps working throughout: two + // operations in the whole fault window against a baseline of hundreds. + for i in 0..2 { + records.push(op(Stream::Promise, i, Outcome::Confirmed)); + } + for second in 1..=10 { + records.push(op(Stream::Promise, -second, Outcome::Confirmed)); + } + let report = build(&records); + + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "a trickle should not make the outage look unobserved, got {:?}", + report.findings + ); + } + + /// The inverse of the trickle case, and the one the old aggregate verdict + /// could miss. A single stream still answering all the way through is a + /// fault that did not land on it, and the silence of the others must not + /// vote it down. + #[test] + fn a_stream_still_answering_is_a_finding_even_when_the_others_are_silent() { + let mut records = history(0, true); + for second in 1..=10 { + records.push(op(Stream::Promise, -second, Outcome::Confirmed)); + } + // Answering every second of the fault window, unlike durable and + // scheduled which say nothing at all. + for second in 0..180 { + records.push(op(Stream::Promise, second, Outcome::Confirmed)); + } + let report = build(&records); + + let finding = report + .findings + .iter() + .find(|f| f.violation == OutageViolation::OutageNotObserved) + .expect("a stream answering throughout is a fault that did not land on it"); + assert_eq!(finding.stream, Some(Stream::Promise)); + } + + /// An absorbed outage: silence across the window with the serving bunched + /// into the seconds at its two edges, which is the shape the platform + /// produces once storage failures are retried rather than fatal. + fn absorbed(fault_secs: i64) -> StorageFaultReport { + let mut records = Vec::new(); + for second in 1..=300 { + for _ in 0..10 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, -second, Outcome::Confirmed)); + } + } + for edge in [0, fault_secs - 1] { + for _ in 0..60 { + records.push(op(Stream::Durable, edge, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, edge, Outcome::Confirmed)); + } + } + for second in fault_secs + 1..=fault_secs + 240 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, second, Outcome::Confirmed)); + } + StorageFaultReport::build( + &records, + Some(FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(fault_secs)), + }), + ENDPOINT, + OutageExpectation::WholeWorkload { + quiet_floor_percent: QUIET_FLOOR, + }, + Duration::from_secs(120), + ) + } + + /// The regression this floor exists for. + /// + /// An absorbed outage does all its serving in the seconds at the window's + /// edges, so a during-fault *rate* divides one fixed burst by the window + /// length: identical platform behaviour reads as a small share of a long + /// window and a large share of a short one. Shortening S16's window from + /// 180s to 60s duly tripped the old 15% ceiling on a partition that had + /// plainly landed. Quiet time is measured against the window's own edges + /// and does not move with it. + #[test] + fn the_verdict_does_not_move_when_the_fault_window_does() { + let long = absorbed(180); + let short = absorbed(60); + + for (secs, report) in [(180, &long), (60, &short)] { + assert!( + report + .findings + .iter() + .all(|f| f.violation != OutageViolation::OutageNotObserved), + "a {secs}s absorbed outage did land, got {:?}", + report.findings + ); + assert!( + report.quietest_stream_percent.unwrap() > QUIET_FLOOR, + "a {secs}s absorbed outage is silent for most of its window, got {:?}", + report.quietest_stream_percent + ); + } + + // And the number that used to decide it, kept as the demonstration: + // the same shape reads very differently at the two lengths, which is + // exactly why it could not stay the verdict. + let long_share = long.share_of_baseline_percent.unwrap(); + let short_share = short.share_of_baseline_percent.unwrap(); + // 15% was the old ceiling. The long window sits under it and the short + // one over it, on the same behaviour, which is the whole defect. + assert!( + long_share < 15.0 && short_share > 15.0, + "this test only means something if the old ceiling would have flipped between the \ + two windows, got {long_share}% over 180s against {short_share}% over 60s" + ); + } + + /// A stream that was working before the outage and confirmed nothing after + /// the heal is the platform losing a mechanism, not a slow recovery. + #[test] + fn a_stream_that_never_came_back_is_a_finding() { + let mut records = history(0, true); + records.retain(|r| !(r.stream == Stream::Scheduled && r.submitted_at > t0())); + let report = build(&records); + + let finding = report + .findings + .iter() + .find(|f| f.violation == OutageViolation::StreamNeverRecovered) + .expect("expected a stream-never-recovered finding"); + assert_eq!(finding.stream, Some(Stream::Scheduled)); + } + + /// A stream the run never got going is the driver's problem, already + /// reported by the baseline gate. Calling it unrecovered here would put one + /// failure in two places under two names. + #[test] + fn a_stream_that_never_worked_at_all_is_not_reported_as_unrecovered() { + let mut records = history(0, true); + records.retain(|r| r.stream != Stream::Scheduled); + // Present throughout, never confirming. + for second in 1..=300 { + records.push(op(Stream::Scheduled, -second, Outcome::Rejected)); + } + let report = build(&records); + + assert!( + !report + .findings + .iter() + .any(|f| f.stream == Some(Stream::Scheduled)), + "a stream that never worked should not be reported as unrecovered, got {:?}", + report.findings + ); + } + + /// Recovery is measured and reported against the budget, never asserted on. + #[test] + fn a_slow_recovery_is_recorded_rather_than_a_finding() { + let mut records = history(0, false); + // First confirmation 200s after the heal, past the 120s budget. + records.push(op(Stream::Durable, 380, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, 380, Outcome::Confirmed)); + let report = build(&records); + + assert!( + report.findings.is_empty(), + "a slow recovery must not be a finding, got {:?}", + report.findings + ); + assert!( + report.recovery.iter().all(|r| r.over_budget), + "both streams should be recorded as over budget, got {:?}", + report.recovery + ); + } + + /// Without a fault window nothing can be placed either side of the outage, + /// so the report carries counts and refuses to reach a verdict. + #[test] + fn without_a_fault_window_there_is_no_verdict() { + let report = StorageFaultReport::build( + &history(180, true), + None, + ENDPOINT, + OutageExpectation::WholeWorkload { + quiet_floor_percent: QUIET_FLOOR, + }, + Duration::from_secs(120), + ); + + assert!(report.findings.is_empty()); + assert_eq!(report.share_of_baseline_percent, None); + assert!(report.caught_in_flight.is_empty()); + assert!( + report + .cells + .iter() + .all(|c| c.window == Window::Unknown && c.share_of_baseline_percent.is_none()), + "every cell should be unplaceable, got {:?}", + report.cells + ); + } + + /// The number that stops a small non-zero during-fault cell reading as + /// residual service: it says the stream was silent for almost the whole + /// window and answered only at the end. + #[test] + fn the_during_fault_cell_says_how_long_the_stream_stayed_quiet() { + let mut records = history(0, true); + // One confirmation, answered 170s into the 180s outage. + records.push(op(Stream::Durable, 170, Outcome::Confirmed)); + let report = build(&records); + + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert_eq!(during.served, 1); + // Silent from the cut until that one answer, which is nearly the whole + // window — not the 10s that remained after it. + assert_eq!(during.quiet_ms, Some(170_020)); + } + + /// A stream answering steadily right through the window is the shape the + /// quiet number has to be able to tell apart from the one above. Both have + /// a non-zero during-fault count; only the silence separates them. + #[test] + fn a_stream_answering_throughout_the_window_is_never_quiet_for_long() { + let report = build(&history(180, true)); + + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert_eq!(during.served, 180); + assert!( + during.quiet_ms.is_some_and(|ms| ms < 2_000), + "a stream answering once a second should show no real silence, got {:?}", + during.quiet_ms + ); + } + + /// The regression the first S16 run turned up. + /// + /// The workload keeps offering work all through the outage, so anything + /// derived from submission times is busy no matter what the platform is + /// doing. Every one of these operations is offered during the fault and + /// answered only after the heal: the fault window served nothing, and the + /// report has to say so rather than crediting the window with work it did + /// not do. + #[test] + fn work_offered_during_the_outage_and_answered_after_it_is_not_during_fault_service() { + let mut records = history(0, true); + for second in 0..120 { + let mut record = op(Stream::Durable, second, Outcome::Confirmed); + // Offered inside the outage, answered once the storage returned. + record.completed_at = Some(t0() + TimeDelta::seconds(181)); + record.duration_ms = (181 - second) as u64 * 1_000; + records.push(record); + } + let report = build(&records); + + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert_eq!(during.submitted, 120, "they were offered during the fault"); + assert_eq!(during.confirmed, 120, "and they did all eventually confirm"); + assert_eq!( + during.served, 0, + "but none of it was served during the fault" + ); + assert_eq!(during.served_per_sec, 0.0); + assert_eq!( + during.quiet_ms, + Some(180_000), + "silent for the whole window" + ); + assert!( + report.findings.is_empty(), + "a total outage must not be reported as one that never landed, got {:?}", + report.findings + ); + } + + /// The operations at risk. They were submitted before the cut and are + /// attributed to the before-fault row, which is the last place anyone looks + /// for the damage. + #[test] + fn operations_running_when_the_storage_went_away_are_reported_apart() { + let mut records = history(0, true); + // Submitted 30s before the cut, still running when it landed. + for _ in 0..3 { + records.push(stalled(Stream::Durable, -30)); + } + let report = build(&records); + + let caught = report + .caught_in_flight + .iter() + .find(|c| c.stream == Stream::Durable) + .expect("expected a caught-in-flight row for the durable stream"); + assert_eq!(caught.operations, 3); + assert_eq!(caught.indeterminate, 3); + assert_eq!(caught.attempts_timed_out, 6); + assert_eq!(caught.max_attempts, 2); + // 245s from 30s before the cut lands 35s past the 180s heal. + assert_eq!(caught.outlived_the_fault, 3); + } + + /// An operator reading the fault window wants the dominant failure mode + /// first, and wants to know whether the platform refused the work or lost + /// track of it. + #[test] + fn fault_window_failures_are_grouped_by_class_commonest_first() { + let mut records = history(0, true); + for i in 0..5 { + records.push(stalled(Stream::Durable, i)); + } + let mut rejected = op(Stream::Durable, 10, Outcome::Rejected); + rejected.error_class = Some(ErrorClass::Response); + rejected.error = Some("agent not found".to_string()); + records.push(rejected); + let report = build(&records); + + assert_eq!(report.fault_window_errors.len(), 2); + assert_eq!(report.fault_window_errors[0].class, ErrorClass::Transport); + assert_eq!(report.fault_window_errors[0].operations, 5); + assert_eq!( + report.fault_window_errors[0].example.as_deref(), + Some("attempt timed out after 120s") + ); + assert_eq!(report.fault_window_errors[1].class, ErrorClass::Response); + assert_eq!(report.fault_window_errors[1].operations, 1); + } + + /// Findings are hoisted for a human; the share line is context and belongs + /// with the notes, not with the things to act on. + #[test] + fn the_share_line_is_a_note_and_the_findings_are_attention() { + let report = build(&history(180, true)); + + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("outage-not-observed")) + ); + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("of its baseline throughput")) + ); + } +} diff --git a/integration-tests/src/chaos/pinned.rs b/integration-tests/src/chaos/pinned.rs index 95f148f61a..010aa80a17 100644 --- a/integration-tests/src/chaos/pinned.rs +++ b/integration-tests/src/chaos/pinned.rs @@ -63,6 +63,7 @@ use crate::chaos::history::Stream; use crate::chaos::workload::{self, WorkloadContext}; use anyhow::Context; use golem_common::base_model::agent::ParsedAgentId; +use golem_common::model::component::ComponentDto; use golem_common::model::{AgentId, RoutingTable}; use golem_common::{agent_id, data_value}; use golem_test_framework::config::{BenchmarkTestDependencies, TestDependencies}; @@ -116,29 +117,56 @@ pub fn candidate_agent_name(key_prefix: &str, index: u32) -> String { format!("{key_prefix}-{}-{index:04}", Stream::PinnedHttp) } -/// The routing-table `AgentId` for one pinned agent. +/// The routing-table `AgentId` for one agent of `agent_type`. /// /// This has to match how the worker-service builds the id it routes on — /// the component id plus the *string form* of the parsed agent id — or the /// ownership calculation would be answering a different question from the one /// the platform answers. -fn routing_agent_id(ctx: &WorkloadContext, agent: &str) -> AgentId { - let parsed: ParsedAgentId = agent_id!(COUNTER_AGENT, agent.to_string()); +pub fn routing_agent_id(ctx: &WorkloadContext, agent_type: &str, agent: &str) -> AgentId { + routing_agent_id_in(&ctx.counters, agent_type, agent) +} + +/// [`routing_agent_id`] against a named component rather than the counters one. +/// +/// Ownership is per agent id and an agent id contains its component, so an agent +/// type that lives in a different component — S11's waiters are in the promise +/// component — hashes to a different shard than the same name would under +/// `counters`. Looking one up against the wrong component would silently aim a +/// kill at whichever executor happened to own a name nothing uses. +pub fn routing_agent_id_in(component: &ComponentDto, agent_type: &str, agent: &str) -> AgentId { + let parsed: ParsedAgentId = agent_id!(agent_type, agent.to_string()); AgentId { - component_id: ctx.counters.id, + component_id: component.id, agent_id: parsed.to_string(), } } -/// Groups candidate agents by the executor that owns them. -fn owners( +/// Groups agents of `agent_type` by the executor that owns them. +/// +/// Takes the type rather than assuming `Counter` because S10 asks the same +/// question about its schedule targets, and ownership is per agent id: two +/// agent types with the same name are two different agents on two possibly +/// different executors. +pub fn owners_by_pod( ctx: &WorkloadContext, table: &RoutingTable, - candidates: &[String], + agent_type: &str, + agents: &[String], +) -> BTreeMap> { + owners_by_pod_in(&ctx.counters, table, agent_type, agents) +} + +/// [`owners_by_pod`] against a named component. See [`routing_agent_id_in`]. +pub fn owners_by_pod_in( + component: &ComponentDto, + table: &RoutingTable, + agent_type: &str, + agents: &[String], ) -> BTreeMap> { let mut by_pod: BTreeMap> = BTreeMap::new(); - for agent in candidates { - if let Some(pod) = table.lookup(&routing_agent_id(ctx, agent)) { + for agent in agents { + if let Some(pod) = table.lookup(&routing_agent_id_in(component, agent_type, agent)) { by_pod .entry(pod.to_string()) .or_default() @@ -148,6 +176,27 @@ fn owners( by_pod } +/// Groups candidate pinned agents by the executor that owns them. +fn owners( + ctx: &WorkloadContext, + table: &RoutingTable, + candidates: &[String], +) -> BTreeMap> { + owners_by_pod(ctx, table, COUNTER_AGENT, candidates) +} + +/// Host part of an executor address, which is what a Kubernetes `status.podIP` +/// field selector matches. +/// +/// No port at all is not a shape the shard-manager produces, but degrading to +/// the whole string beats panicking mid-run. +pub fn pod_ip_of(pod_address: &str) -> String { + pod_address + .rsplit_once(':') + .map(|(host, _)| host.to_string()) + .unwrap_or_else(|| pod_address.to_string()) +} + /// Chooses an executor and the agents it owns. /// /// Fails rather than falling back to an unpinned run: a scenario that quietly @@ -201,10 +250,7 @@ pub async fn select( } let agents: Vec = owned.into_iter().take(config.agents as usize).collect(); - let pod_ip = pod_address - .rsplit_once(':') - .map(|(host, _)| host.to_string()) - .unwrap_or_else(|| pod_address.clone()); + let pod_ip = pod_ip_of(&pod_address); info!( "S8: pinned {} agents to executor {pod_address} (scanned {pool} candidates across {} executors)", @@ -242,7 +288,7 @@ pub async fn verify_ownership( let mut drifted: Vec = Vec::new(); for agent in &selection.agents { let owner = table - .lookup(&routing_agent_id(ctx, agent)) + .lookup(&routing_agent_id(ctx, COUNTER_AGENT, agent)) .map(|pod| pod.to_string()); if owner.as_deref() != Some(selection.pod_address.as_str()) { drifted.push(format!( @@ -379,16 +425,10 @@ mod tests { /// hold for the `ip:port` form the routing table actually produces. #[test] fn a_pod_address_splits_into_an_ip_the_workflow_can_select_on() { - let split = |address: &str| { - address - .rsplit_once(':') - .map(|(host, _)| host.to_string()) - .unwrap_or_else(|| address.to_string()) - }; - assert_eq!(split("10.0.14.207:9000"), "10.0.14.207"); + assert_eq!(pod_ip_of("10.0.14.207:9000"), "10.0.14.207"); // No port at all is not a shape the shard-manager produces, but // degrading to the whole string beats panicking mid-run. - assert_eq!(split("10.0.14.207"), "10.0.14.207"); + assert_eq!(pod_ip_of("10.0.14.207"), "10.0.14.207"); } /// Candidate names carry the run prefix and the stream, and are zero-padded diff --git a/integration-tests/src/chaos/reachability.rs b/integration-tests/src/chaos/reachability.rs new file mode 100644 index 0000000000..be1765d5e5 --- /dev/null +++ b/integration-tests/src/chaos/reachability.rs @@ -0,0 +1,1233 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! What a partition between worker-service and one executor cost (GOL-370). +//! +//! Three questions, in the order they have to be answered: +//! +//! 1. **Did the fault land?** The agents on the isolated executor must stop +//! being served. If they did not, nothing else in the report means anything, +//! and it says so — see [`ReachabilityViolation::PartitionNotObserved`]. +//! 2. **What did it cost the agents it was not aimed at?** The other executor +//! is reachable throughout and its agents should be untouched. They share a +//! worker-service with the stalled half, and worker-service keeps one +//! process-wide routing table that every stalled caller invalidates, so +//! "untouched" is a claim worth measuring rather than assuming. +//! 3. **Did the isolated agents come back?** Every one of them, and how long +//! after the link was restored. +//! +//! Throughput, not success rate, is what the first two are measured on, and the +//! difference matters. An emitter holds one operation at a time +//! ([`crate::chaos::steady`]), so an agent whose executor is unreachable does +//! not fail repeatedly — it fails *slowly*, once, and offers nothing else for +//! two minutes. A success rate would read that as one failure out of one +//! attempt and call the group 0% degraded. Confirmed operations per second is +//! the number that collapses, so it is the number the report is built on. +//! +//! ### Reading a non-zero isolated cell +//! +//! Operations are placed in the window they were *submitted* in, which is when +//! the platform was asked to do the work. An isolated operation submitted late +//! in the fault window can still be waiting when the link comes back, and then +//! confirms — so the isolated group's during-fault throughput is small rather +//! than exactly zero. At one operation per agent per second against a +//! two-minute client timeout, that is a handful of confirmations against a +//! baseline of hundreds. A cell anywhere near the ceiling means something else. + +use crate::chaos::history::{OperationRecord, Outcome, Stream}; +use crate::chaos::split::{ + FaultWindow, Group, PodSplit, Window, longest_silence_ms, round2, window_end, window_secs, + window_start, +}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// The most findings the report carries. Past this it says how many it dropped +/// rather than growing without bound: 200 agents that all failed to recover is +/// one fact, not 200. +const MAX_FINDINGS: usize = 50; + +/// What a reachability finding is about. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ReachabilityViolation { + /// The isolated executor kept serving its agents through the fault. The + /// partition did not take hold where the run says it did, and every other + /// number here describes an undisturbed cluster. + PartitionNotObserved, + /// The agents on the *reachable* executor lost throughput while the other + /// executor was cut off. They were never partitioned from anything. + ControlDegraded, + /// An isolated agent produced no confirmed operation at all once the link + /// was restored. + NeverRecovered, +} + +impl ReachabilityViolation { + pub fn as_str(self) -> &'static str { + match self { + ReachabilityViolation::PartitionNotObserved => "partition-not-observed", + ReachabilityViolation::ControlDegraded => "control-degraded", + ReachabilityViolation::NeverRecovered => "never-recovered", + } + } +} + +/// One finding. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReachabilityFinding { + pub violation: ReachabilityViolation, + /// The agent it localises to, for the findings that localise to one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent: Option, + pub detail: String, +} + +/// What one group of agents managed in one window. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ThroughputCell { + /// `on-pod` is the group the isolated executor owns; `elsewhere` is the + /// control group. The names are the suite's, shared with every other + /// scenario that divides its agents around one pod — see + /// [`crate::chaos::split`]. + pub group: Group, + pub window: Window, + /// Agents of this group that offered at least one operation in this window. + /// Below the group's size means emitters were stalled across the whole + /// window rather than merely slowed. + pub agents_active: usize, + /// Operations *offered* in this window, and how they eventually ended up. + /// Attributed by submission time, so an operation counted here may not have + /// been answered until a later window. + pub submitted: u64, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Operations *answered* in this window, whenever they were offered. + /// + /// This is the one that says whether the group was being served, and it is + /// deliberately not `confirmed`: while an executor is unreachable the two + /// differ by exactly the work that was accepted and answered only once the + /// partition came down. + pub served: u64, + /// Attempts that hit the client's attempt timeout rather than answering. + /// The pending-then-timeout behaviour the scenario exists to make visible. + pub attempts_timed_out: u64, + pub window_secs: f64, + pub served_per_sec: f64, + /// This cell's rate against the same group's own before-fault rate. `None` + /// for the before-fault cell itself, and for a group that never had a + /// baseline to compare against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_of_baseline_percent: Option, + /// The longest the group was answered nothing at all, anywhere in this + /// window. + /// + /// The number that stops a small non-zero rate being read as residual + /// service. A during-fault cell can show a handful of confirmations that + /// all arrived in the last seconds of the window, once the fault was + /// already coming down. Measured against the window's own edges, so a group + /// that fell silent at the start or stayed silent to the end is caught by + /// it too, and a `quietMs` close to `windowSecs` is a total outage however + /// the rate arithmetic came out. + /// + /// `None` when the window has no fixed bounds to measure against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quiet_ms: Option, + pub latency: LatencyStats, +} + +/// The operations the fault landed in the middle of. +/// +/// The population S3 actually disturbed, and the one number a reader wants +/// first. It cannot be read off the cells: these operations were *submitted* +/// before the cut, so every trace of them — their timeouts, their duration — +/// is attributed to the `before-fault` row, which is the last place anyone +/// looks for the damage. +/// +/// The control group's entry is the comparison that makes it mean something, +/// but read it on **duration**, not on count. How many operations a healthy +/// group has in flight at any instant is its duty cycle — operation time over +/// interval — so a group answering in 45ms on a one-second cadence has about +/// one agent in twenty busy. A stalled group accumulates instead: every emitter +/// ends up holding an operation that will not return, so its count climbs to +/// the size of the group. A real run showed 113 of 113 against 3 of 87, and the +/// gap between 47ms and 182 seconds is the finding, not the gap between 3 and +/// 113. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CaughtInFlight { + pub group: Group, + /// Operations submitted before the fault that were still running when it + /// landed. + pub operations: u64, + /// Distinct agents they belonged to. Equal to the group size when every + /// emitter was mid-operation, which is what one-in-flight-per-agent makes + /// the normal case. + pub agents: usize, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Submission to final outcome, across every attempt. + pub duration: LatencyStats, + /// Attempts that hit the client's attempt timeout rather than answering. + pub attempts_timed_out: u64, + /// The most attempts any one of them needed. + /// + /// Load-bearing rather than trivia. An operation that stalled and then + /// answered on a later attempt was rescued by the caller's retry, not + /// returned by the platform: with retries off it would have ended + /// indeterminate. That distinction is invisible in the outcome alone. + pub max_attempts: u32, + /// How many were still unresolved when the fault was reported healed. + pub outlived_the_fault: u64, +} + +/// The reachability account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReachabilityReport { + /// The executor the partition was aimed at, as the shard-manager names it. + pub isolated_pod: String, + pub isolated_agents: usize, + pub reachable_agents: usize, + /// The thresholds from the suite YAML, recorded so an archived cell can be + /// read years later against the numbers it was judged by rather than + /// against today's config. + pub isolated_ceiling_percent: f64, + pub control_floor_percent: f64, + pub recovery_budget_ms: u64, + pub cells: Vec, + /// What the fault landed in the middle of, per group. Empty for a run that + /// never learned when the fault was. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub caught_in_flight: Vec, + /// Per isolated agent, how long after the link was restored its first + /// confirmed operation landed. + pub recovery: LatencyStats, + pub recovery_over_budget: u64, + /// Isolated agents that never confirmed anything after the heal. + pub agents_never_recovered: Vec, + /// Records whose agent the selection never saw. Zero on a healthy run; + /// non-zero means the split and the workload disagree about who was driven. + pub records_outside_the_split: u64, + pub findings: Vec, + /// Findings past [`MAX_FINDINGS`], dropped rather than carried. Non-zero + /// means `findings` is a sample. + pub findings_omitted: u64, +} + +/// One group's per-window accumulation, before it becomes a cell. +#[derive(Default)] +struct Tally { + agents: BTreeSet, + submitted: u64, + confirmed: u64, + rejected: u64, + indeterminate: u64, + attempts_timed_out: u64, + durations: Vec, + /// When this group actually answered inside this window, sorted later. + /// + /// Keyed on the window a confirmation *landed* in rather than the one its + /// operation was offered in, which is the only way either of the numbers + /// derived from it means what it says. + served_at: Vec>, +} + +impl ReachabilityReport { + /// Builds the account from the operation history. + /// + /// `fault` is what the workflow reported. Without it every record lands in + /// [`Window::Unknown`] and the report carries counts but no verdict, which + /// is the honest outcome for a run that never learned when the fault was: + /// the thresholds are all defined relative to a before-and-during + /// comparison that cannot be made. + pub fn build( + records: &[OperationRecord], + split: &PodSplit, + fault: Option, + isolated_ceiling_percent: f64, + control_floor_percent: f64, + recovery_budget: std::time::Duration, + ) -> Self { + let mut tallies: BTreeMap<(Group, Window), Tally> = BTreeMap::new(); + let mut records_outside_the_split = 0u64; + let mut first_submitted: Option> = None; + let mut last_completed: Option> = None; + + for record in records.iter().filter(|r| r.stream == Stream::Durable) { + let Some(group) = split.group_of(&record.agent) else { + records_outside_the_split += 1; + continue; + }; + let window = Window::of(record.submitted_at, fault); + let tally = tallies.entry((group, window)).or_default(); + + tally.agents.insert(record.agent.clone()); + tally.submitted += 1; + match record.outcome { + Outcome::Confirmed => { + tally.confirmed += 1; + tally.durations.push(record.duration_ms); + } + Outcome::Rejected => tally.rejected += 1, + Outcome::Indeterminate => tally.indeterminate += 1, + } + tally.attempts_timed_out += record.attempts_timed_out(); + + // Answered work is filed under the window it was answered in, which + // is usually but not always the one it was offered in. An operation + // offered while the executor was unreachable and answered once the + // partition came down is not service the fault window delivered, + // and counting it there is how a total outage reads as partial. + if record.outcome == Outcome::Confirmed + && let Some(completed) = record.completed_at + { + tallies + .entry((group, Window::of(completed, fault))) + .or_default() + .served_at + .push(completed); + } + + first_submitted = Some(match first_submitted { + Some(at) if at <= record.submitted_at => at, + _ => record.submitted_at, + }); + if let Some(completed) = record.completed_at { + last_completed = Some(match last_completed { + Some(at) if at >= completed => at, + _ => completed, + }); + } + } + + // Baselines first: every other cell is expressed as a share of its own + // group's before-fault rate, so a lopsided split cannot make one group + // look better than the other. + let mut baseline_rate: BTreeMap = BTreeMap::new(); + let mut cells: Vec = Vec::new(); + for ((group, window), tally) in &tallies { + let secs = window_secs(*window, fault, first_submitted, last_completed); + let served = tally.served_at.len() as u64; + let rate = if secs > 0.0 { + served as f64 / secs + } else { + 0.0 + }; + if *window == Window::BeforeFault { + baseline_rate.insert(*group, rate); + } + let quiet_ms = longest_silence_ms( + &tally.served_at, + window_start(*window, fault, first_submitted), + window_end(*window, fault, last_completed), + ); + cells.push(ThroughputCell { + group: *group, + window: *window, + agents_active: tally.agents.len(), + submitted: tally.submitted, + confirmed: tally.confirmed, + served, + rejected: tally.rejected, + indeterminate: tally.indeterminate, + attempts_timed_out: tally.attempts_timed_out, + window_secs: round2(secs), + served_per_sec: round2(rate), + share_of_baseline_percent: None, + quiet_ms, + latency: LatencyStats::from_durations(tally.durations.clone()), + }); + } + + for cell in &mut cells { + if cell.window == Window::BeforeFault { + continue; + } + if let Some(baseline) = baseline_rate.get(&cell.group).filter(|r| **r > 0.0) { + cell.share_of_baseline_percent = + Some(round2(cell.served_per_sec / baseline * 100.0)); + } + } + cells.sort_by_key(|c| (c.group, c.window)); + + // ── Recovery, per isolated agent ──────────────────────────────────── + let recovered_at = fault.and_then(|w| w.recovered_at); + let mut gaps: Vec = Vec::new(); + let mut over_budget = 0u64; + let mut agents_never_recovered: Vec = Vec::new(); + if let Some(healed) = recovered_at { + for agent in &split.on_pod { + let first = records + .iter() + .filter(|r| { + r.stream == Stream::Durable + && &r.agent == agent + && r.outcome == Outcome::Confirmed + }) + .filter_map(|r| r.completed_at) + .filter(|at| *at >= healed) + .min(); + match first { + Some(at) => { + let gap = (at - healed).num_milliseconds().max(0) as u64; + if gap > recovery_budget.as_millis() as u64 { + over_budget += 1; + } + gaps.push(gap); + } + None => agents_never_recovered.push(agent.clone()), + } + } + } + + // ── What the fault landed in the middle of ────────────────────────── + // + // Computed from the whole history rather than from the cells, because + // the cells cannot answer it: these operations were submitted before + // the cut, so every trace of them sits in the `before-fault` row. + let mut caught_in_flight: Vec = Vec::new(); + if let Some(window) = fault { + let mut by_group: BTreeMap> = BTreeMap::new(); + for record in records.iter().filter(|r| r.stream == Stream::Durable) { + let Some(group) = split.group_of(&record.agent) else { + continue; + }; + let still_running = record + .completed_at + .is_none_or(|at| at >= window.injected_at); + if record.submitted_at < window.injected_at && still_running { + by_group.entry(group).or_default().push(record); + } + } + for (group, caught) in by_group { + let agents: BTreeSet<&str> = caught.iter().map(|r| r.agent.as_str()).collect(); + caught_in_flight.push(CaughtInFlight { + group, + operations: caught.len() as u64, + agents: agents.len(), + confirmed: count_of(&caught, Outcome::Confirmed), + rejected: count_of(&caught, Outcome::Rejected), + indeterminate: count_of(&caught, Outcome::Indeterminate), + duration: LatencyStats::from_durations( + caught.iter().map(|r| r.duration_ms).collect(), + ), + attempts_timed_out: caught.iter().map(|r| r.attempts_timed_out()).sum(), + max_attempts: caught.iter().map(|r| r.attempts).max().unwrap_or(0), + outlived_the_fault: caught + .iter() + .filter(|r| match (r.completed_at, window.recovered_at) { + // Never finished at all, so it certainly outlived it. + (None, _) => true, + (Some(done), Some(healed)) => done >= healed, + // No heal was ever reported; nothing can be said. + (Some(_), None) => false, + }) + .count() as u64, + }); + } + } + + let mut report = ReachabilityReport { + isolated_pod: split.pod_address.clone(), + isolated_agents: split.on_pod.len(), + reachable_agents: split.elsewhere.len(), + isolated_ceiling_percent, + control_floor_percent, + recovery_budget_ms: recovery_budget.as_millis() as u64, + cells, + caught_in_flight, + recovery: LatencyStats::from_durations(gaps), + recovery_over_budget: over_budget, + agents_never_recovered, + records_outside_the_split, + findings: Vec::new(), + findings_omitted: 0, + }; + report.judge(); + report + } + + /// A cell by group and window, if the run produced one. + pub fn cell(&self, group: Group, window: Window) -> Option<&ThroughputCell> { + self.cells + .iter() + .find(|c| c.group == group && c.window == window) + } + + fn judge(&mut self) { + let mut findings: Vec = Vec::new(); + + // Did the fault land? Asked first, because a "no" makes the rest of the + // report a description of a cluster nothing happened to. + if let Some(share) = self + .cell(Group::OnPod, Window::DuringFault) + .and_then(|c| c.share_of_baseline_percent) + && share > self.isolated_ceiling_percent + { + findings.push(ReachabilityFinding { + violation: ReachabilityViolation::PartitionNotObserved, + agent: None, + detail: format!( + "the {} agents on the isolated executor kept {share:.1}% of their baseline \ + throughput during the fault, above the {:.0}% ceiling: the partition did \ + not cut worker-service off from {}. Nothing else in this report describes \ + a disturbed cluster.", + self.isolated_agents, self.isolated_ceiling_percent, self.isolated_pod + ), + }); + } + + // What did it cost the half it was not aimed at? + if let Some(share) = self + .cell(Group::Elsewhere, Window::DuringFault) + .and_then(|c| c.share_of_baseline_percent) + && share < self.control_floor_percent + { + findings.push(ReachabilityFinding { + violation: ReachabilityViolation::ControlDegraded, + agent: None, + detail: format!( + "the {} agents on the reachable executor kept only {share:.1}% of their \ + baseline throughput while {} was cut off, below the {:.0}% floor. They \ + were never partitioned from anything, so this is what serving an \ + unreachable executor cost the rest of the cluster.", + self.reachable_agents, self.isolated_pod, self.control_floor_percent + ), + }); + } + + for agent in &self.agents_never_recovered { + findings.push(ReachabilityFinding { + violation: ReachabilityViolation::NeverRecovered, + agent: Some(agent.clone()), + detail: format!( + "{agent} confirmed no operation at all after the link to {} was restored", + self.isolated_pod + ), + }); + } + + self.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + self.findings = findings; + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines: Vec = self + .findings + .iter() + .map(|f| format!("S3 {}: {}", f.violation.as_str(), f.detail)) + .collect(); + + if self.findings_omitted > 0 { + lines.push(format!( + "S3: {} further reachability finding(s) were dropped from the report", + self.findings_omitted + )); + } + if self.records_outside_the_split > 0 { + lines.push(format!( + "S3: {} operation(s) ran against agents the ownership split never saw, so they \ + are in no group and in no cell — the split and the workload disagree about \ + who was driven", + self.records_outside_the_split + )); + } + // Work the fault caught and the run cannot account for. Not a finding — + // an indeterminate operation is doubt, not damage, and the read-back + // and exactly-once accounts are what resolve it — but the operator has + // to see it next to the clean cells rather than infer it from them. + for caught in &self.caught_in_flight { + let unresolved = caught.indeterminate + caught.rejected; + if unresolved > 0 { + lines.push(format!( + "S3: {unresolved} of the {} operations the cut caught in flight on {} did \ + not confirm ({} indeterminate, {} rejected)", + caught.operations, + caught.group.as_str(), + caught.indeterminate, + caught.rejected + )); + } + } + + // Over budget is not a finding. How long a partition heal may take + // before an agent is served again is a judgement, and the driver is not + // the one to make it. + if self.recovery_over_budget > 0 { + lines.push(format!( + "S3: {} of {} isolated agents took longer than the {}ms recovery budget to \ + confirm anything after the heal (p99 {}ms, worst {}ms)", + self.recovery_over_budget, + self.isolated_agents, + self.recovery_budget_ms, + self.recovery.p99_ms, + self.recovery.max_ms + )); + } + lines + } + + /// The lines a reader needs in order to interpret the run, which are not + /// themselves problems. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S3: {} agents on the isolated executor {}, {} elsewhere", + self.isolated_agents, self.isolated_pod, self.reachable_agents + )]; + + // The caught population first: it is what the fault actually disturbed, + // and nothing in the cells below points at it. + for caught in &self.caught_in_flight { + lines.push(format!( + "S3 {}: {} operations across {} agents were in flight when the cut landed — \ + p50 {}ms / p99 {}ms / worst {}ms, {} attempt(s) timed out, up to {} attempts \ + each, {} still unresolved at the heal", + caught.group.as_str(), + caught.operations, + caught.agents, + caught.duration.p50_ms, + caught.duration.p99_ms, + caught.duration.max_ms, + caught.attempts_timed_out, + caught.max_attempts, + caught.outlived_the_fault, + )); + } + + for group in [Group::OnPod, Group::Elsewhere] { + for window in [Window::BeforeFault, Window::DuringFault, Window::AfterFault] { + if let Some(cell) = self.cell(group, window) { + lines.push(format!( + "S3 {} {}: {:.2} served/s{}, {} offered{}, {} indeterminate, {} \ + attempt(s) timed out, {} of {} agents active", + group.as_str(), + window.as_str(), + cell.served_per_sec, + cell.share_of_baseline_percent + .map(|s| format!(" ({s:.1}% of baseline)")) + .unwrap_or_default(), + cell.submitted, + // Silence is the reading that stops a small rate being + // mistaken for residual service. + cell.quiet_ms + .filter(|_| cell.window_secs > 0.0) + .map(|q| { + format!( + ", answered nothing for {:.1}s of a {:.1}s window", + q as f64 / 1000.0, + cell.window_secs + ) + }) + .unwrap_or_default(), + cell.indeterminate, + cell.attempts_timed_out, + cell.agents_active, + if group == Group::OnPod { + self.isolated_agents + } else { + self.reachable_agents + }, + )); + } + } + } + + if self.recovery.count > 0 { + lines.push(format!( + "S3: isolated agents were served again p50 {}ms / p99 {}ms / worst {}ms after \ + the heal", + self.recovery.p50_ms, self.recovery.p99_ms, self.recovery.max_ms + )); + } + lines + } +} + +/// Operations of one outcome. +fn count_of(records: &[&OperationRecord], outcome: Outcome) -> u64 { + records.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::errors::ErrorClass; + use crate::chaos::history::{AttemptRecord, Phase}; + use chrono::TimeDelta; + use std::time::Duration; + use test_r::test; + + const ISOLATED: &str = "chaos-s3-durable-0000"; + const CONTROL: &str = "chaos-s3-durable-0001"; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-24T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![ISOLATED.to_string()], + elsewhere: vec![CONTROL.to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(180)), + } + } + + /// One operation, submitted `offset` seconds from the moment the partition + /// was injected. Negative offsets are the baseline. + fn op(agent: &str, offset_secs: i64, outcome: Outcome) -> OperationRecord { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + OperationRecord { + op_id: 0, + stream: Stream::Durable, + phase: Phase::Baseline, + agent: agent.to_string(), + method: "increment".to_string(), + idempotency_key: format!("{agent}-{offset_secs}"), + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(20)), + attempts: 1, + outcome, + duration_ms: 20, + returned_value: Some(1), + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 20, + returned_value: Some(1), + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + /// An operation that hung until the client gave up, twice: the shape every + /// isolated invocation takes while the link is cut. + fn stalled(agent: &str, offset_secs: i64) -> OperationRecord { + let mut record = op(agent, offset_secs, Outcome::Indeterminate); + record.duration_ms = 245_000; + record.completed_at = Some(record.submitted_at + TimeDelta::seconds(245)); + record.returned_value = None; + record.attempts = 2; + record.error_class = Some(ErrorClass::Transport); + record.attempt_log = (1..=2) + .map(|attempt| AttemptRecord { + attempt, + started_at: record.submitted_at, + duration_ms: 120_000, + returned_value: None, + succeeded: false, + error_class: Some(ErrorClass::Transport), + error: Some("attempt timed out after 120s".to_string()), + }) + .collect(); + record + } + + /// A baseline both groups share, then a fault the isolated group is cut off + /// by and the control group sails through. `during_isolated` is how many + /// operations the isolated group still managed. + fn history(during_isolated: usize, during_control: usize) -> Vec { + let mut records = Vec::new(); + // 300s of baseline, one operation per second per agent. + for second in 1..=300 { + for agent in [ISOLATED, CONTROL] { + records.push(op(agent, -second, Outcome::Confirmed)); + } + } + // 180s of fault. + for i in 0..during_isolated { + records.push(op(ISOLATED, i as i64, Outcome::Confirmed)); + } + for i in 0..during_control { + records.push(op(CONTROL, i as i64, Outcome::Confirmed)); + } + // 240s of recovery, both groups back to cadence. + for second in 181..=420 { + for agent in [ISOLATED, CONTROL] { + records.push(op(agent, second, Outcome::Confirmed)); + } + } + records + } + + fn build(records: &[OperationRecord]) -> ReachabilityReport { + ReachabilityReport::build( + records, + &split(), + Some(fault()), + 25.0, + 75.0, + Duration::from_secs(60), + ) + } + + /// The healthy shape: the cut half stops, the other half does not, and the + /// report says so without raising anything. + #[test] + fn a_partition_that_lands_and_costs_the_control_group_nothing_has_no_findings() { + let report = build(&history(2, 180)); + + assert!( + report.findings.is_empty(), + "expected no findings, got {:?}", + report.findings + ); + let isolated = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + assert!( + isolated.share_of_baseline_percent.unwrap() < 25.0, + "the isolated group should have collapsed, got {isolated:?}" + ); + let control = report.cell(Group::Elsewhere, Window::DuringFault).unwrap(); + assert!( + control.share_of_baseline_percent.unwrap() >= 75.0, + "the control group should have held, got {control:?}" + ); + } + + /// The inconclusive case, and the most important one in this module: a + /// partition that never took hold produces a report full of healthy numbers, + /// and it has to read as "this run tested nothing" rather than as a pass. + #[test] + fn an_isolated_group_that_kept_working_says_the_partition_never_landed() { + let report = build(&history(180, 180)); + + assert_eq!( + report + .findings + .iter() + .map(|f| f.violation) + .collect::>(), + vec![ReachabilityViolation::PartitionNotObserved] + ); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("Nothing else in this report")), + "the operator has to be told the rest of the report is meaningless" + ); + } + + /// The finding S3 exists to hunt: agents that were never partitioned from + /// anything losing throughput because their worker-service was busy waiting + /// on a pod they do not use. + #[test] + fn a_control_group_that_degraded_alongside_the_isolated_one_is_a_finding() { + // Half the control group's baseline rate, well under the 75% floor. + let report = build(&history(2, 90)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == ReachabilityViolation::ControlDegraded), + "expected collateral damage to be reported, got {:?}", + report.findings + ); + } + + /// Throughput, not raw counts. The baseline is 300s long and the fault + /// window 180s, so comparing totals would call an untouched group degraded + /// by 40% on window length alone. + #[test] + fn a_group_that_held_its_rate_is_not_penalised_for_a_shorter_fault_window() { + // 180 operations across a 180s window is exactly the baseline rate. + let report = build(&history(2, 180)); + let control = report.cell(Group::Elsewhere, Window::DuringFault).unwrap(); + + assert_eq!(control.submitted, 180); + assert!( + (control.share_of_baseline_percent.unwrap() - 100.0).abs() < 1.0, + "an unchanged rate should read as ~100% of baseline, got {:?}", + control.share_of_baseline_percent + ); + } + + /// The pending-then-timeout behaviour the ticket asks to see in the history. + #[test] + fn attempts_that_hit_the_client_timeout_are_counted_per_cell() { + let mut records = history(0, 180); + records.push(stalled(ISOLATED, 10)); + let report = build(&records); + + let isolated = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + assert_eq!(isolated.attempts_timed_out, 2); + assert_eq!(isolated.indeterminate, 1); + assert_eq!(isolated.confirmed, 0); + } + + /// Recovery is measured from the heal, not from when the operation was + /// submitted: an agent whose call was already in flight when the link came + /// back was served promptly, and the number has to say so. + #[test] + fn recovery_is_measured_from_the_heal() { + let mut records = history(0, 180); + // Submitted mid-fault, answered five seconds after the heal. + let mut late = op(ISOLATED, 100, Outcome::Confirmed); + late.completed_at = Some(t0() + TimeDelta::seconds(185)); + records.retain(|r| !(r.agent == ISOLATED && r.submitted_at > t0())); + records.push(late); + + let report = build(&records); + assert_eq!(report.recovery.count, 1); + assert_eq!(report.recovery.max_ms, 5_000); + assert!(report.agents_never_recovered.is_empty()); + } + + /// An isolated agent that never answered again is the strongest finding this + /// report can make: the link is back and nothing else is wrong. + #[test] + fn an_isolated_agent_that_never_came_back_is_a_finding() { + let records: Vec = history(0, 180) + .into_iter() + .filter(|r| !(r.agent == ISOLATED && r.submitted_at >= t0())) + .collect(); + + let report = build(&records); + assert_eq!(report.agents_never_recovered, vec![ISOLATED.to_string()]); + assert!( + report + .findings + .iter() + .any(|f| f.violation == ReachabilityViolation::NeverRecovered + && f.agent.as_deref() == Some(ISOLATED)) + ); + } + + /// Without the fault window there is no before-and-during to compare, so + /// every threshold in this report is undefined. It must count what it saw + /// and claim nothing — a verdict here would be invented. + #[test] + fn a_run_that_never_learned_when_the_fault_was_reports_counts_but_no_verdict() { + let report = ReachabilityReport::build( + &history(180, 180), + &split(), + None, + 25.0, + 75.0, + Duration::from_secs(60), + ); + + assert!(report.findings.is_empty()); + assert!(report.cells.iter().all(|c| c.window == Window::Unknown)); + assert!(report.cells.iter().all(|c| c.served_per_sec == 0.0)); + assert_eq!(report.recovery.count, 0); + // And nothing is silently blamed on the agents themselves. + assert!(report.agents_never_recovered.is_empty()); + } + + /// An agent the selection never saw is reported, not folded into a group. + /// It means the split and the workload disagree about who was driven, and + /// silently counting it as a control would corrupt the one comparison the + /// whole scenario rests on. + #[test] + fn operations_against_an_unknown_agent_are_reported_rather_than_grouped() { + let mut records = history(2, 180); + records.push(op("chaos-s3-durable-9999", -10, Outcome::Confirmed)); + + let report = build(&records); + assert_eq!(report.records_outside_the_split, 1); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("the ownership split never saw")) + ); + } + + /// Two hundred agents that all failed to recover is one fact, not two + /// hundred, and an artifact carrying all of them is unreadable. + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let many: Vec = (0..MAX_FINDINGS + 10) + .map(|i| format!("chaos-s3-durable-{i:04}")) + .collect(); + let mut split = split(); + split.on_pod = many.clone(); + + let report = ReachabilityReport::build( + &[], + &split, + Some(fault()), + 25.0, + 75.0, + Duration::from_secs(60), + ); + + assert_eq!(report.agents_never_recovered.len(), MAX_FINDINGS + 10); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 10); + } + + /// Over budget is context, not a finding: how long worker-service's retry + /// loop may take to notice a healed link is a judgement, and the driver is + /// not the one to make it. + #[test] + fn a_slow_recovery_is_an_attention_line_rather_than_a_finding() { + let mut records = history(0, 180); + let mut slow = op(ISOLATED, 100, Outcome::Confirmed); + slow.completed_at = Some(t0() + TimeDelta::seconds(300)); + records.retain(|r| !(r.agent == ISOLATED && r.submitted_at > t0())); + records.push(slow); + + let report = build(&records); + assert_eq!(report.recovery_over_budget, 1); + assert!(report.findings.is_empty()); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("recovery budget")) + ); + } + + /// An operation the cut landed in the middle of: submitted just before it, + /// still running when it landed, answered only once the link returned. + fn caught(agent: &str, timed_out_attempts: u32) -> OperationRecord { + let mut record = op(agent, -1, Outcome::Confirmed); + record.duration_ms = 178_000; + record.completed_at = Some(record.submitted_at + TimeDelta::milliseconds(178_000)); + record.attempts = timed_out_attempts + 1; + record.attempt_log = (1..=timed_out_attempts) + .map(|attempt| AttemptRecord { + attempt, + started_at: record.submitted_at, + duration_ms: 120_000, + returned_value: None, + succeeded: false, + error_class: Some(ErrorClass::Transport), + error: Some("attempt timed out after 120s".to_string()), + }) + .chain(std::iter::once(AttemptRecord { + attempt: timed_out_attempts + 1, + started_at: record.submitted_at, + duration_ms: 56_000, + returned_value: Some(1), + succeeded: true, + error_class: None, + error: None, + })) + .collect(); + record + } + + /// The population the fault actually disturbed, which no cell can show: + /// these were submitted before the cut, so their timeouts and their + /// duration are attributed to the `before-fault` row. + #[test] + fn the_operations_the_cut_caught_are_reported_apart_from_the_cells() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + records.push(caught(ISOLATED, 1)); + + let report = build(&records); + let caught = report + .caught_in_flight + .iter() + .find(|c| c.group == Group::OnPod) + .expect("the isolated group had an operation in flight"); + + assert_eq!(caught.operations, 1); + assert_eq!(caught.agents, 1); + assert_eq!(caught.confirmed, 1); + assert_eq!(caught.duration.max_ms, 178_000); + assert_eq!(caught.attempts_timed_out, 1); + // The retry is what landed it. With retries off it would have been + // indeterminate, and the outcome alone cannot say so. + assert_eq!(caught.max_attempts, 2); + // It answered before the heal was stamped, so it did not outlive it. + assert_eq!(caught.outlived_the_fault, 0); + + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("were in flight when the cut landed")), + "the caught population has to be in front of the reader" + ); + } + + /// An operation still unanswered when the link came back is a different + /// statement from one that resolved inside the window, and the report has + /// to keep them apart. + #[test] + fn an_operation_still_running_at_the_heal_is_counted_as_outliving_it() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + let mut late = caught(ISOLATED, 1); + late.completed_at = Some(t0() + TimeDelta::seconds(200)); + late.duration_ms = 201_000; + records.push(late); + + let report = build(&records); + let caught = report + .caught_in_flight + .iter() + .find(|c| c.group == Group::OnPod) + .unwrap(); + assert_eq!(caught.outlived_the_fault, 1); + } + + /// The number that stops a small during-fault rate reading as residual + /// service. A real run showed 1.1% of baseline from operations that all + /// arrived in the last four seconds of a 182-second window, once the fault + /// was already coming down. + #[test] + fn a_group_silent_until_the_heal_reports_how_long_it_answered_nothing() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + // A late tail, exactly as the heal window produces. + for offset in 176..180 { + records.push(op(ISOLATED, offset, Outcome::Confirmed)); + } + + let report = build(&records); + let cell = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + + assert_eq!(cell.submitted, 4); + assert_eq!(cell.quiet_ms, Some(176_020)); + assert!( + cell.quiet_ms.unwrap() as f64 / 1000.0 > cell.window_secs * 0.9, + "silence has to dominate the window, not trail it" + ); + // The same group's baseline answers once a second, so its longest + // silence is one interval. That contrast is the whole reading: 176s of + // silence is not a slower version of this, it is a different state. + let baseline = report.cell(Group::OnPod, Window::BeforeFault).unwrap(); + assert_eq!(baseline.quiet_ms, Some(1_000)); + + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("answered nothing for 176.0s of a 180.0s window")), + "notes were {:?}", + report.note_lines() + ); + } + + /// The regression the first S16 run turned up, in the module that shares + /// the construction. + /// + /// The workload keeps offering work to an unreachable executor all through + /// the partition, so anything derived from submission times looks busy no + /// matter what the platform is doing. Every one of these is offered during + /// the fault and answered only after the heal: the fault window served the + /// isolated group nothing, and the cell has to say so. + #[test] + fn work_offered_to_an_isolated_group_and_answered_after_the_heal_is_not_service() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + for offset in 0..120 { + let mut record = op(ISOLATED, offset, Outcome::Confirmed); + record.completed_at = Some(t0() + TimeDelta::seconds(181)); + record.duration_ms = (181 - offset) as u64 * 1_000; + records.push(record); + } + + let report = build(&records); + let cell = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + + assert_eq!(cell.submitted, 120, "they were offered during the fault"); + assert_eq!(cell.confirmed, 120, "and they did all eventually confirm"); + assert_eq!(cell.served, 0, "but none of it was served during the fault"); + assert_eq!(cell.served_per_sec, 0.0); + assert_eq!(cell.quiet_ms, Some(180_000), "silent for the whole window"); + } + + /// Work the cut caught and the run cannot account for. Not a finding, but + /// it must sit next to the clean cells rather than be inferred from them. + #[test] + fn caught_work_that_never_confirmed_is_raised_to_the_operator() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + let mut lost = caught(ISOLATED, 2); + lost.outcome = Outcome::Indeterminate; + records.push(lost); + + let report = build(&records); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("caught in flight") && l.contains("did not confirm")), + "attention was {:?}", + report.attention_lines() + ); + // Still not a finding: doubt is not damage, and the exactly-once and + // read-back accounts are what resolve it. + assert!(report.findings.is_empty()); + } + + /// A run with no fault window cannot say what was in flight when the cut + /// landed, because it does not know when that was. + #[test] + fn a_run_without_a_fault_window_claims_nothing_was_caught() { + let report = ReachabilityReport::build( + &history(2, 180), + &split(), + None, + 25.0, + 75.0, + Duration::from_secs(60), + ); + assert!(report.caught_in_flight.is_empty()); + assert!(report.cells.iter().all(|c| c.quiet_ms.is_none())); + } +} diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 0937b0d0f2..5cc3c99518 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -25,8 +25,13 @@ //! same scenario ran anywhere else. use crate::chaos::pinned::PinnedSelection; +use crate::chaos::scheduled::ScheduledSelection; +use crate::chaos::split::PodSplit; use crate::chaos::summary::{ChaosSummary, TerminationReason}; -use crate::chaos::{FaultConfig, PinnedConfig, RetryPolicy, WorkloadConfig}; +use crate::chaos::{ + DeleteConfig, FaultConfig, IsolationConfig, PinnedConfig, PromiseConfig, RetryPolicy, + RevertConfig, RollbackConfig, ScheduledConfig, StorageConfig, WorkloadConfig, +}; use chrono::{DateTime, Utc}; use golem_test_framework::benchmark::RunMetadata; use serde::{Deserialize, Serialize}; @@ -34,7 +39,14 @@ use std::path::Path; /// Bumped when the on-disk shape changes incompatibly. Archived results outlive /// the tooling that reads them, so the shape has to say which shape it is. -pub const RESULT_SCHEMA_VERSION: u32 = 2; +/// +/// 3: the storage scenarios stopped sharing one verdict. `storage` carries an +/// `expect` block where it used to carry `outageQuietFloorPercent`, and the +/// storage-fault account carries the same block plus +/// `leastServingStreamPercent`. A version 2 result does not deserialise into +/// the version 3 types, which is what the bump is for; the report generator +/// reads both, because the runs already in the bucket are worth rendering. +pub const RESULT_SCHEMA_VERSION: u32 = 3; /// A phase's wall-clock extent. These are the numbers the workflow pins Grafana /// time ranges to, so they are recorded in UTC with no ambiguity. @@ -130,6 +142,54 @@ pub struct ChaosResult { /// to own. Present only for scenarios that pin the target. #[serde(default, skip_serializing_if = "Option::is_none")] pub pinned_selection: Option, + /// The scheduled-registration workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheduled: Option, + /// How the schedule targets divided around the executor the fault was aimed + /// at. Present only for S10, and load-bearing for reading its percentiles: + /// without it there is no way to tell the affected population from the + /// control group. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheduled_selection: Option, + /// The suspended-waiter workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub promise: Option, + /// How the promise waiters divided around the executor the fault was aimed + /// at. Present only for S11, and load-bearing for the same reason as + /// `scheduledSelection`: without it there is no way to tell the affected + /// population from the control group. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub promise_selection: Option, + /// The reachability workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub isolation: Option, + /// How the agents divided around the executor the partition cut off. + /// Present only for S3. Load-bearing for the same reason as + /// `promiseSelection`, and for one more: S3's whole verdict is a comparison + /// between the two groups, so a report without this cannot be re-checked. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub isolation_selection: Option, + /// The revert workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revert: Option, + /// How the revert agents divided around the executor the kill was aimed at. + /// Present only for S7. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revert_selection: Option, + /// The deletion workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete: Option, + /// How the agent slots divided around the executor the kill was aimed at. + /// Present only for S6. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete_selection: Option, + /// The component rollback the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rollback: Option, + /// The storage the run was configured to take away, and the thresholds its + /// account was judged by. Present only for S16. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, pub retry_policy: RetryPolicy, pub scope: RunScope, pub summary: ChaosSummary, @@ -185,6 +245,7 @@ mod tests { target: "shard-manager".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, workload: Some(WorkloadConfig { @@ -197,6 +258,18 @@ mod tests { }), pinned: None, pinned_selection: None, + scheduled: None, + scheduled_selection: None, + promise: None, + promise_selection: None, + isolation: None, + isolation_selection: None, + revert: None, + revert_selection: None, + delete: None, + delete_selection: None, + rollback: None, + storage: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "env-1".to_string(), @@ -209,6 +282,542 @@ mod tests { } } + /// The S11 shape. Same contract as the S10 test below, for the same reason: + /// `ci-scripts/chaos-investigation-report.py` in golem-cloud reads these by + /// name, and the two repositories cannot be changed atomically. + #[test] + fn an_s11_result_carries_the_promise_wakeup_fields_the_investigation_report_reads() { + use crate::chaos::history::{WaiterWakeupLog, WakeupRecord}; + use crate::chaos::split::{FaultWindow, PodSplit}; + use crate::chaos::wakeups::WakeupReport; + + let now = Utc::now(); + let waiter = "chaos-s11-promise-waiter-0000".to_string(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![waiter.clone()], + elsewhere: Vec::new(), + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S11".to_string(); + result.promise = Some(crate::chaos::PromiseConfig { + waiters: 200, + dwell_millis: 5000, + wakeup_budget_secs: 60, + }); + result.promise_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_promise_wakeups(WakeupReport::build( + &[], + &[WaiterWakeupLog { + agent: waiter.clone(), + wakes: Some(1), + wakeups: vec![WakeupRecord { + token: format!("{waiter}-00000001"), + armed_at: now, + woken_at: now + chrono::Duration::seconds(6), + }], + error: None, + }], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + std::time::Duration::from_secs(5), + std::time::Duration::from_secs(60), + 0, + )); + + let json = serde_json::to_value(&result).unwrap(); + let wakeups = &json["summary"]["promiseWakeups"]; + for key in [ + "wakeupBudgetMs", + "dwellMs", + "completionsConfirmed", + "completionsIndeterminate", + "completionsRejected", + "wakeupsRecorded", + "wokeOnce", + "indeterminateThatWoke", + "inconclusive", + "unverifiable", + "unknownTokens", + "waitersUnreadable", + "waitersTruncated", + "waitersStoodDown", + "waitersWedged", + "delay", + "findings", + "findingsOmitted", + ] { + assert!( + !wakeups[key].is_null(), + "summary.promiseWakeups.{key} is what the investigation report reads" + ); + } + assert_eq!(json["promise"]["wakeupBudgetSecs"], 60); + assert_eq!(json["promiseSelection"]["podIp"], "10.0.1.1"); + + // And it still round-trips, so an archived S11 result stays readable. + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S11"); + assert!(parsed.summary.promise_wakeups.is_some()); + assert!(parsed.promise_selection.is_some()); + } + + /// The S9 shape. Same contract as the others: the investigation report in + /// golem-cloud reads these fields by name. + #[test] + fn an_s9_result_carries_the_rollback_fields_the_investigation_report_reads() { + use crate::chaos::rollback::{ControlPlaneAttempts, RollbackReport, VersionCensus}; + + let now = Utc::now(); + let mut forward = std::collections::BTreeMap::new(); + forward.insert("chaos-s9-durable-0000".to_string(), Some(2u32)); + let mut back = std::collections::BTreeMap::new(); + back.insert("chaos-s9-durable-0000".to_string(), Some(1u32)); + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S9".to_string(); + result.rollback = Some(crate::chaos::RollbackConfig { + settle_secs: 90, + rolled_forward_floor_percent: 90.0, + control_retries: 2, + control_retry_delay_secs: 5, + kill_delay_secs: 2, + }); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)).with_rollback( + RollbackReport { + forward_revision: 2, + rollback_revision: 3, + forward_version: 2, + rollback_version: 1, + rolled_forward: VersionCensus::build("before-rollback", 2, &forward), + rolled_back: Some(VersionCensus::build("after-recovery", 1, &back)), + control: ControlPlaneAttempts { + requested: 200, + accepted_first_try: 198, + accepted_after_retry: 2, + refused: 0, + max_retries: 2, + }, + rolled_forward_floor_percent: 90.0, + }, + ); + + let json = serde_json::to_value(&result).unwrap(); + let rollback = &json["summary"]["rollback"]; + for key in [ + "forwardRevision", + "rollbackRevision", + "forwardVersion", + "rollbackVersion", + "rolledForward", + "rolledBack", + "control", + "rolledForwardFloorPercent", + ] { + assert!( + !rollback[key].is_null(), + "summary.rollback.{key} is what the investigation report reads" + ); + } + for key in [ + "requested", + "acceptedFirstTry", + "acceptedAfterRetry", + "refused", + ] { + assert!(!rollback["control"][key].is_null(), "control.{key}"); + } + assert_eq!(json["rollback"]["rolledForwardFloorPercent"], 90.0); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S9"); + assert!(parsed.summary.rollback.is_some()); + } + + /// The S6 shape. Same contract as the S3, S7, S10 and S11 tests: the + /// investigation report in golem-cloud reads these fields by name. + #[test] + fn an_s6_result_carries_the_resurrection_fields_the_investigation_report_reads() { + use crate::chaos::deletions::DeleteRound; + use crate::chaos::resurrection::ResurrectionReport; + use crate::chaos::split::{FaultWindow, PodSplit}; + + let now = Utc::now(); + let agent = "chaos-s6-delete-0000".to_string(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![agent.clone()], + elsewhere: Vec::new(), + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S6".to_string(); + result.delete = Some(crate::chaos::DeleteConfig { + agents: 200, + increments_per_round: 3, + interval_millis: 500, + recovery_budget_secs: 60, + }); + result.delete_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_resurrection(ResurrectionReport::build( + &[DeleteRound { + agent, + round: 0, + before_delete: Some(3), + outcome: crate::chaos::history::Outcome::Confirmed, + rejected_as_not_found: false, + submitted_at: now, + completed_at: Some(now), + observed_after: Some(0), + }], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + 3, + )); + + let json = serde_json::to_value(&result).unwrap(); + let resurrection = &json["summary"]["resurrection"]; + for key in [ + "incrementsPerRound", + "roundsRecorded", + "deletesConfirmed", + "deletesIndeterminate", + "deletesRejected", + "deletedExactly", + "indeterminateThatDeleted", + "indeterminateThatDidNot", + "unjudgeable", + "unprobed", + "cells", + "caughtByTheKill", + "findings", + "findingsOmitted", + ] { + assert!( + !resurrection[key].is_null(), + "summary.resurrection.{key} is what the investigation report reads" + ); + } + assert_eq!(json["delete"]["incrementsPerRound"], 3); + assert_eq!(json["deleteSelection"]["podIp"], "10.0.1.1"); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S6"); + assert!(parsed.summary.resurrection.is_some()); + assert!(parsed.delete_selection.is_some()); + } + + /// The S7 shape. Same contract as the S3, S10 and S11 tests: the + /// investigation report in golem-cloud reads these fields by name. + #[test] + fn an_s7_result_carries_the_truncation_fields_the_investigation_report_reads() { + use crate::chaos::reverts::RevertRound; + use crate::chaos::split::{FaultWindow, PodSplit}; + use crate::chaos::truncation::TruncationReport; + + let now = Utc::now(); + let agent = "chaos-s7-revert-0000".to_string(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![agent.clone()], + elsewhere: Vec::new(), + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S7".to_string(); + result.revert = Some(crate::chaos::RevertConfig { + agents: 200, + increments_per_round: 4, + revert_invocations: 2, + interval_millis: 500, + recovery_budget_secs: 60, + }); + result.revert_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_truncation(TruncationReport::build( + &[RevertRound { + agent, + round: 0, + before_revert: Some(10), + asked_to_revert: 2, + outcome: crate::chaos::history::Outcome::Confirmed, + submitted_at: now, + completed_at: Some(now), + observed_after: Some(8), + }], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + 4, + 2, + )); + + let json = serde_json::to_value(&result).unwrap(); + let truncation = &json["summary"]["truncation"]; + for key in [ + "incrementsPerRound", + "revertInvocations", + "roundsRecorded", + "revertsConfirmed", + "revertsIndeterminate", + "revertsRejected", + "appliedExactly", + "indeterminateThatApplied", + "indeterminateThatDidNot", + "unjudgeable", + "unprobed", + "cells", + "caughtByTheKill", + "findings", + "findingsOmitted", + ] { + assert!( + !truncation[key].is_null(), + "summary.truncation.{key} is what the investigation report reads" + ); + } + assert_eq!(json["revert"]["revertInvocations"], 2); + assert_eq!(json["revertSelection"]["podIp"], "10.0.1.1"); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S7"); + assert!(parsed.summary.truncation.is_some()); + assert!(parsed.revert_selection.is_some()); + } + + /// The S3 shape. Same contract as the S10 and S11 tests, for the same + /// reason: `ci-scripts/chaos-investigation-report.py` in golem-cloud reads + /// these by name, and the two repositories cannot be changed atomically. + #[test] + fn an_s3_result_carries_the_reachability_fields_the_investigation_report_reads() { + use crate::chaos::reachability::ReachabilityReport; + use crate::chaos::split::{FaultWindow, PodSplit}; + + let now = Utc::now(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec!["chaos-s3-durable-0000".to_string()], + elsewhere: vec!["chaos-s3-durable-0001".to_string()], + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S3".to_string(); + result.isolation = Some(crate::chaos::IsolationConfig { + agents: 200, + interval_millis: 1000, + isolated_ceiling_percent: 25.0, + control_floor_percent: 75.0, + recovery_budget_secs: 60, + }); + result.isolation_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_reachability(ReachabilityReport::build( + &[], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: Some(now + chrono::Duration::seconds(180)), + }), + 25.0, + 75.0, + std::time::Duration::from_secs(60), + )); + + let json = serde_json::to_value(&result).unwrap(); + let reachability = &json["summary"]["reachability"]; + for key in [ + "isolatedPod", + "isolatedAgents", + "reachableAgents", + "isolatedCeilingPercent", + "controlFloorPercent", + "recoveryBudgetMs", + "cells", + "recovery", + "recoveryOverBudget", + "agentsNeverRecovered", + "recordsOutsideTheSplit", + "findings", + "findingsOmitted", + ] { + assert!( + !reachability[key].is_null(), + "summary.reachability.{key} is what the investigation report reads" + ); + } + assert_eq!(json["isolation"]["controlFloorPercent"], 75.0); + assert_eq!(json["isolationSelection"]["podIp"], "10.0.1.1"); + + // And it still round-trips, so an archived S3 result stays readable. + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S3"); + assert!(parsed.summary.reachability.is_some()); + assert!(parsed.isolation_selection.is_some()); + } + + /// The S10 shape, whose report is read by a script in another repository. + /// + /// `ci-scripts/chaos-investigation-report.py` in golem-cloud renders these + /// fields by name. The two repositories cannot be changed atomically, so a + /// rename here would silently empty a section of the investigation report + /// rather than fail anything. Naming the keys in a test is what makes that + /// break loudly and locally. + #[test] + fn an_s10_result_carries_the_schedule_fire_fields_the_investigation_report_reads() { + use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; + use crate::chaos::history::{FireRecord, Stream, TargetFireLog}; + + let now = Utc::now(); + let due = now + chrono::Duration::seconds(10); + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S10".to_string(); + result.scheduled = Some(crate::chaos::ScheduledConfig { + targets: 100, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 60, + }); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_schedule_fires(ScheduleFireReport::build( + &[], + &[TargetFireLog { + agent: "chaos-s10-scheduled-target-0000".to_string(), + polls: Some(1), + fires: vec![FireRecord { + token: "chaos-s10-scheduled-target-0000-00000001".to_string(), + scheduled_at: due, + observed_at: due + chrono::Duration::seconds(4), + }], + error: None, + }], + std::time::Duration::from_secs(10), + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + &std::collections::BTreeSet::from(["chaos-s10-scheduled-target-0000".to_string()]), + std::time::Duration::from_secs(60), + )); + + let json = serde_json::to_value(&result).unwrap(); + let fires = &json["summary"]["scheduleFires"]; + for key in [ + "leaseBudgetMs", + "registrationsConfirmed", + "registrationsIndeterminate", + "firesRecorded", + "firedOnce", + "indeterminateThatFired", + "inconclusive", + "unverifiable", + "unknownTokens", + "targetsUnreadable", + "targetsTruncated", + "delay", + "findings", + "findingsOmitted", + "overdueOnArrival", + "overdueDelay", + ] { + assert!( + !fires[key].is_null(), + "summary.scheduleFires.{key} is what the investigation report reads" + ); + } + let cell = &fires["delay"][0]; + assert_eq!(cell["group"], "on-killed-executor"); + assert_eq!(cell["window"], "during-fault"); + assert_eq!(cell["delay"]["p99Ms"], 4000); + assert_eq!(cell["overBudget"], 0); + assert_eq!(cell["minDelayMs"], 4000); + assert_eq!(json["scheduled"]["leaseBudgetSecs"], 60); + + // And it still round-trips, so an archived S10 result stays readable. + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S10"); + assert!(parsed.summary.schedule_fires.is_some()); + assert!(!Stream::Scheduled.to_string().is_empty()); + } + + /// The CI annotation branches on `summary.attention` being non-empty, so + /// the two lists have to stay two lists across the repo boundary. Folding + /// context back into `attention` would make the annotation fire on every + /// healthy run, which is how it came to mean nothing the first time. + #[test] + fn a_result_separates_findings_from_context_for_the_ci_annotation() { + use crate::chaos::summary::Note; + + let mut result = sample_result(TerminationReason::Completed); + result.summary.absorb([ + Note::context("routing at start: 1024/1024 shards (settled before measuring)"), + Note::attention("4 scheduled targets filled their fire log and dropped entries"), + ]); + + let json = serde_json::to_value(&result).unwrap(); + assert_eq!( + json["summary"]["attention"].as_array().unwrap().len(), + 1, + "summary.attention is what --attention-count counts" + ); + assert_eq!( + json["summary"]["notes"].as_array().unwrap().len(), + 1, + "summary.notes is what the report renders as run context" + ); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.summary.attention.len(), 1); + assert_eq!(parsed.summary.notes.len(), 1); + } + + /// A tag can be moved; a digest identifies the build a run actually tested. + /// The workflow emits both, and the runbook tells a reader to match them + /// against the deployment manifests, so both key names are load-bearing. + #[test] + fn run_metadata_carries_the_image_digest_beside_the_tag() { + use golem_test_framework::benchmark::RunMetadata; + + let mut result = sample_result(TerminationReason::Completed); + result.run_metadata = Some(RunMetadata { + worker_executor_image_tag: Some("v1.5.10-dev.2".to_string()), + worker_executor_image_digest: Some("sha256:60eac87a".to_string()), + ..Default::default() + }); + + let json = serde_json::to_value(&result).unwrap(); + assert_eq!( + json["runMetadata"]["workerExecutorImageTag"], + "v1.5.10-dev.2" + ); + assert_eq!( + json["runMetadata"]["workerExecutorImageDigest"], + "sha256:60eac87a" + ); + } + #[test] fn result_round_trips_through_json() { let result = sample_result(TerminationReason::Completed); @@ -388,6 +997,7 @@ mod sample_artifact { target: "shard-manager".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, workload: Some(WorkloadConfig { @@ -400,6 +1010,18 @@ mod sample_artifact { }), pinned: None, pinned_selection: None, + scheduled: None, + scheduled_selection: None, + promise: None, + promise_selection: None, + isolation: None, + isolation_selection: None, + revert: None, + revert_selection: None, + delete: None, + delete_selection: None, + rollback: None, + storage: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), @@ -418,4 +1040,223 @@ mod sample_artifact { result.save(&path).unwrap(); println!("wrote sample result to {path}"); } + + /// The same, for S10, whose report has a section of its own. + /// + /// A separate artifact rather than a field added to the one above: the two + /// scenarios do not share a workload shape, and a sample that carried both + /// a mixed workload and a scheduled one would not resemble anything the + /// driver ever writes. + #[test] + fn write_sample_s10_result_artifact() { + use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; + use crate::chaos::history::{FireRecord, TargetFireLog}; + use crate::chaos::scheduled::ScheduledSelection; + + let Ok(path) = std::env::var("CHAOS_SAMPLE_RESULT_S10") else { + return; + }; + let at = |s: i64| Utc.timestamp_opt(1_800_000_000 + s, 0).unwrap(); + let killed = "chaos-s10-scheduled-target-0000"; + let survivor = "chaos-s10-scheduled-target-0001"; + + // Four registrations per target. On the killed executor one of them + // never fires, which is the finding the section exists to render. + let mut records = Vec::new(); + let mut killed_fires = Vec::new(); + let mut survivor_fires = Vec::new(); + for i in 0..8u64 { + let target = if i % 2 == 0 { killed } else { survivor }; + // Spread so some actions fall due before the kill and some during it, + // which is what gives the report both a control row and the row it is + // actually about. + let submitted = at(285 + i as i64 * 4); + let token = format!("{target}-{:08}", i / 2); + records.push(OperationRecord { + op_id: i, + stream: Stream::Scheduled, + phase: if submitted < at(300) { + Phase::Baseline + } else { + Phase::Fault + }, + agent: target.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: token.clone(), + submitted_at: submitted, + completed_at: Some(submitted), + attempts: 1, + outcome: Outcome::Confirmed, + duration_ms: 8 + i, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + }); + + let due = submitted + chrono::Duration::seconds(10); + // One registration stalls on the client's attempt timeout and only + // lands long after its action was due — the shape the first real run + // produced 26 times over. + if i == 1 { + let stalled = records.last_mut().unwrap(); + stalled.completed_at = Some(submitted + chrono::Duration::seconds(125)); + stalled.duration_ms = 125_000; + stalled.attempts = 2; + } + if target == killed { + // The last one is the action the kill swallowed. + if i < 6 { + killed_fires.push(FireRecord { + token, + scheduled_at: due, + // Late by a shard reassignment. + observed_at: due + chrono::Duration::milliseconds(41_500), + }); + } + } else { + // The stalled registration's action fires the moment it lands, + // which the raw arithmetic calls 115s late. + let late = if i == 1 { 115_200 } else { 120 }; + survivor_fires.push(FireRecord { + token, + scheduled_at: due, + observed_at: due + chrono::Duration::milliseconds(late), + }); + } + } + + let logs = vec![ + TargetFireLog { + agent: killed.to_string(), + polls: Some(killed_fires.len() as u64), + fires: killed_fires, + error: None, + }, + TargetFireLog { + agent: survivor.to_string(), + polls: Some(survivor_fires.len() as u64), + fires: survivor_fires, + error: None, + }, + ]; + + let on_pod = std::collections::BTreeSet::from([killed.to_string()]); + let report = ScheduleFireReport::build( + &records, + &logs, + std::time::Duration::from_secs(10), + Some(FaultWindow { + injected_at: at(300), + recovered_at: Some(at(420)), + }), + &on_pod, + std::time::Duration::from_secs(60), + ); + + let readback: Vec = logs + .iter() + .map(|log| { + let scoped: Vec<&OperationRecord> = + records.iter().filter(|r| r.agent == log.agent).collect(); + AgentReadback::evaluate( + Stream::Scheduled, + &log.agent, + &scoped, + Ok(log.polls.unwrap_or(0)), + ) + }) + .collect(); + + let result = ChaosResult { + schema_version: RESULT_SCHEMA_VERSION, + scenario_code: "S10".to_string(), + scenario_name: "executor-crash-during-scheduled-fire".to_string(), + completed: false, + termination_reason: TerminationReason::ScheduledFireViolated { + findings: report.findings.len() as u64, + first: report + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + }, + started_at: at(0), + ended_at: Some(at(900)), + phases: Phases { + baseline: Some({ + let mut w = PhaseWindow::started(at(0)); + w.end(at(300)); + w + }), + fault: Some({ + let mut w = PhaseWindow::started(at(300)); + w.end(at(420)); + w + }), + recovery: Some({ + let mut w = PhaseWindow::started(at(420)); + w.end(at(720)); + w + }), + }, + fault_injected_at: Some(at(300)), + fault_recovered_at: Some(at(420)), + fault_id: Some("chaos-s10-12345".to_string()), + fault_target_observed: Some("worker-executor-abc123".to_string()), + fault: FaultConfig { + kind: "pod-kill".to_string(), + target: "worker-executor".to_string(), + mode: "one".to_string(), + target_count: None, + manifest: None, + duration_secs: 60, + }, + workload: None, + pinned: None, + pinned_selection: None, + scheduled: Some(crate::chaos::ScheduledConfig { + targets: 2, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 60, + }), + scheduled_selection: Some(ScheduledSelection { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![killed.to_string()], + elsewhere: vec![survivor.to_string()], + targets_per_pod: [ + ("10.0.1.1:9000".to_string(), 1), + ("10.0.1.2:9000".to_string(), 1), + ] + .into_iter() + .collect(), + number_of_shards: 1024, + }), + promise: None, + promise_selection: None, + isolation: None, + isolation_selection: None, + revert: None, + revert_selection: None, + delete: None, + delete_selection: None, + rollback: None, + storage: None, + retry_policy: RetryPolicy::default(), + scope: RunScope { + environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), + component_ids: vec!["0192f000-0000-7000-8000-000000000002".to_string()], + agent_id_prefix: "chaos-s10".to_string(), + idempotency_key_prefix: "chaos-s10-".to_string(), + }, + summary: ChaosSummary::build(&records, readback, Vec::new(), Some(at(300))) + .with_schedule_fires(report), + run_metadata: None, + }; + result.save(&path).unwrap(); + println!("wrote sample S10 result to {path}"); + } } diff --git a/integration-tests/src/chaos/resurrection.rs b/integration-tests/src/chaos/resurrection.rs new file mode 100644 index 0000000000..f5209448fc --- /dev/null +++ b/integration-tests/src/chaos/resurrection.rs @@ -0,0 +1,828 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Did every deleted agent stay deleted (GOL-372)? +//! +//! The same two-value oracle [`crate::chaos::truncation`] uses, one step +//! further. There a round asked the platform to forget some of an agent's work; +//! here it asks it to forget the agent. Invoking a deleted id creates a **new** +//! agent, so the counter a deletion leaves behind has exactly two legal values: +//! +//! * `0` — the deletion took, and the id is a fresh agent counting from nothing +//! * `V` — the deletion did not take, and the old agent is still there +//! +//! Anything else means an agent came back carrying part of a state it was +//! supposed to have lost. +//! +//! ### Which of the two is the finding +//! +//! Neither, on its own. Both are legitimate outcomes of a delete the driver +//! never heard back about — a lost response leaves the question genuinely open. +//! What makes one a defect is the platform's own answer next to it: +//! +//! * confirmed, and the agent is still worth `V` → **resurrection**. The +//! platform said the agent was gone and it is not. +//! * refused, and the agent is gone → the opposite, and just as wrong. +//! +//! ### Why this is worth a scenario at all +//! +//! Because the happy path is already defended and the crash path is not +//! obviously so. `Worker::start_deleting` in the executor exists specifically to +//! stop a background status flush from "resurrecting the cached status" after +//! the durable removal — its own comment. Deleting is four steps (interrupt, +//! mark, remove from the worker service, remove from the active set) and only +//! the third is durable, so a pod that dies between the mark and the removal +//! leaves an agent marked for deletion that was never removed. Whoever picks up +//! its shard next decides what that means. + +use crate::chaos::deletions::{COUNTER_OF_A_NEW_AGENT, DeleteRound}; +use crate::chaos::history::Outcome; +use crate::chaos::split::{FaultWindow, Group, PodSplit, Window}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// The most findings the report carries. +const MAX_FINDINGS: usize = 50; + +/// What went wrong with one deletion. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ResurrectionViolation { + /// The platform confirmed the deletion and the agent came back carrying the + /// value it had before. The failure this scenario is named for. + ResurrectedWithState, + /// The agent came back worth neither nothing nor what it had been: part of + /// a state it was supposed to have lost survived. + PartialState, + /// The platform refused the deletion and the agent is gone anyway, for a + /// reason other than not finding it. + /// + /// A refusal that says the agent was not there is **not** this: see + /// [`ResurrectionReport::deleted_despite_not_found`]. + RefusedButDeleted, +} + +impl ResurrectionViolation { + pub fn as_str(self) -> &'static str { + match self { + ResurrectionViolation::ResurrectedWithState => "resurrected-with-state", + ResurrectionViolation::PartialState => "partial-state", + ResurrectionViolation::RefusedButDeleted => "refused-but-deleted", + } + } +} + +/// One violation, against one named round. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResurrectionFinding { + pub violation: ResurrectionViolation, + pub agent: String, + pub round: u32, + pub window: Window, + /// What the agent was worth before the delete, and what the slot reported + /// afterwards. Carried so a finding reads without the history. + pub before: u64, + pub observed: u64, + pub detail: String, +} + +/// Rounds and their verdicts for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResurrectionCell { + pub group: Group, + pub window: Window, + pub rounds: u64, + /// Deletions that took: the slot came back as a new agent. + pub deleted: u64, + /// Deletions that did not. Legitimate only when the platform never + /// confirmed them. + pub survived: u64, + pub violations: u64, + pub unjudgeable: u64, + pub unprobed: u64, +} + +/// Deletions the kill landed in the middle of. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeletesCaught { + pub group: Group, + /// Deletes submitted before the kill that had not answered when it landed. + /// The population the scenario is actually about. + pub deletes: u64, + pub agents: usize, + pub confirmed: u64, + pub indeterminate: u64, + pub rejected: u64, +} + +/// The resurrection account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResurrectionReport { + /// What each round was configured to do, so an archived finding reads + /// without the suite YAML. + pub increments_per_round: u32, + pub rounds_recorded: u64, + pub deletes_confirmed: u64, + pub deletes_indeterminate: u64, + pub deletes_rejected: u64, + /// Confirmed deletes whose slot came back as a new agent. + pub deleted_exactly: u64, + /// Deletes the driver never heard back about that took anyway. + pub indeterminate_that_deleted: u64, + /// Deletes the driver never heard back about that did not. + pub indeterminate_that_did_not: u64, + /// Deletions the platform reported as `AGENT_NOT_FOUND` whose agent was + /// nonetheless gone afterwards. + /// + /// Not a violation, and the distinction matters. Deleting is not + /// idempotent — `delete_worker_internal` opens with a metadata lookup and + /// returns not-found when there is nothing there — and worker-service's + /// routing layer retries a call whose executor became unreachable. So a + /// delete that succeeded, on an executor that then died, is retried against + /// the new owner and comes back as not-found. The work happened; the answer + /// is misleading. That is worth an operator's attention and is not the + /// platform resurrecting anything. + pub deleted_despite_not_found: u64, + pub unjudgeable: u64, + pub unprobed: u64, + pub cells: Vec, + pub caught_by_the_kill: Vec, + pub findings: Vec, + pub findings_omitted: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Verdict { + Deleted, + /// Gone, but reported to the caller as "no such agent". + DeletedDespiteNotFound, + Survived, + Violation(ResurrectionViolation), + Unjudgeable, + Unprobed, +} + +/// Judges one round against the two values it is allowed to have landed on. +fn judge(round: &DeleteRound) -> Verdict { + let Some(before) = round.before_delete else { + return Verdict::Unjudgeable; + }; + let Some(observed) = round.observed_after else { + return Verdict::Unprobed; + }; + + if observed == COUNTER_OF_A_NEW_AGENT { + // Gone. Legitimate unless the platform said it refused — and even then, + // one refusal means the opposite of what it looks like. See below. + if round.outcome == Outcome::Rejected { + if round.rejected_as_not_found { + return Verdict::DeletedDespiteNotFound; + } + return Verdict::Violation(ResurrectionViolation::RefusedButDeleted); + } + return Verdict::Deleted; + } + if observed == before { + // Still there. Legitimate unless the platform said it was gone. + if round.outcome == Outcome::Confirmed { + return Verdict::Violation(ResurrectionViolation::ResurrectedWithState); + } + return Verdict::Survived; + } + Verdict::Violation(ResurrectionViolation::PartialState) +} + +impl ResurrectionReport { + /// Builds the account from the rounds the workload recorded. + pub fn build( + rounds: &[DeleteRound], + split: &PodSplit, + fault: Option, + increments_per_round: u32, + ) -> Self { + let mut cells: BTreeMap<(Group, Window), ResurrectionCell> = BTreeMap::new(); + let mut findings: Vec = Vec::new(); + let mut report = ResurrectionReport { + increments_per_round, + rounds_recorded: rounds.len() as u64, + deletes_confirmed: 0, + deletes_indeterminate: 0, + deletes_rejected: 0, + deleted_exactly: 0, + indeterminate_that_deleted: 0, + indeterminate_that_did_not: 0, + deleted_despite_not_found: 0, + unjudgeable: 0, + unprobed: 0, + cells: Vec::new(), + caught_by_the_kill: Vec::new(), + findings: Vec::new(), + findings_omitted: 0, + }; + + for round in rounds { + let group = split.group_of(&round.agent).unwrap_or(Group::Elsewhere); + let window = Window::of(round.submitted_at, fault); + let cell = cells + .entry((group, window)) + .or_insert_with(|| ResurrectionCell { + group, + window, + rounds: 0, + deleted: 0, + survived: 0, + violations: 0, + unjudgeable: 0, + unprobed: 0, + }); + cell.rounds += 1; + + match round.outcome { + Outcome::Confirmed => report.deletes_confirmed += 1, + Outcome::Indeterminate => report.deletes_indeterminate += 1, + Outcome::Rejected => report.deletes_rejected += 1, + } + + match judge(round) { + Verdict::Deleted => { + cell.deleted += 1; + if round.outcome == Outcome::Confirmed { + report.deleted_exactly += 1; + } else { + report.indeterminate_that_deleted += 1; + } + } + Verdict::DeletedDespiteNotFound => { + cell.deleted += 1; + report.deleted_despite_not_found += 1; + } + Verdict::Survived => { + cell.survived += 1; + if round.outcome != Outcome::Confirmed { + report.indeterminate_that_did_not += 1; + } + } + Verdict::Unjudgeable => { + cell.unjudgeable += 1; + report.unjudgeable += 1; + } + Verdict::Unprobed => { + cell.unprobed += 1; + report.unprobed += 1; + } + Verdict::Violation(violation) => { + cell.violations += 1; + let before = round.before_delete.unwrap_or_default(); + let observed = round.observed_after.unwrap_or_default(); + findings.push(ResurrectionFinding { + violation, + agent: round.agent.clone(), + round: round.round, + window, + before, + observed, + detail: detail_for(violation, before, observed), + }); + } + } + } + + if let Some(window) = fault { + let mut by_group: BTreeMap> = BTreeMap::new(); + for round in rounds { + let Some(group) = split.group_of(&round.agent) else { + continue; + }; + let unresolved = round.completed_at.is_none_or(|at| at >= window.injected_at); + if round.submitted_at < window.injected_at && unresolved { + by_group.entry(group).or_default().push(round); + } + } + for (group, caught) in by_group { + let agents: BTreeSet<&str> = caught.iter().map(|r| r.agent.as_str()).collect(); + report.caught_by_the_kill.push(DeletesCaught { + group, + deletes: caught.len() as u64, + agents: agents.len(), + confirmed: count(&caught, Outcome::Confirmed), + indeterminate: count(&caught, Outcome::Indeterminate), + rejected: count(&caught, Outcome::Rejected), + }); + } + } + + report.cells = cells.into_values().collect(); + report.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + report.findings = findings; + report + } + + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() || self.findings_omitted > 0 + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines: Vec = self + .findings + .iter() + .map(|f| { + format!( + "S6 {}: {} round {} — {}", + f.violation.as_str(), + f.agent, + f.round, + f.detail + ) + }) + .collect(); + if self.findings_omitted > 0 { + lines.push(format!( + "S6: {} further resurrection finding(s) were dropped from the report", + self.findings_omitted + )); + } + + if self.deleted_despite_not_found > 0 { + lines.push(format!( + "S6: {} deletion(s) came back as AGENT_NOT_FOUND and had in fact taken effect. \ + Deleting is not idempotent and worker-service retries a call whose executor \ + became unreachable, so a delete that succeeded on a dying pod is reported to \ + the caller as though the agent had never existed. The work happened; the \ + answer did not say so.", + self.deleted_despite_not_found + )); + } + + let caught: u64 = self + .caught_by_the_kill + .iter() + .filter(|c| c.group == Group::OnPod) + .map(|c| c.deletes) + .sum(); + if caught == 0 { + lines.push( + "S6: the kill caught no delete in flight on the targeted executor, so this run \ + says nothing about crashing during a deletion. Every verdict below describes \ + deletes that completed either side of it." + .to_string(), + ); + } + lines + } + + /// Lines a reader needs in order to interpret the run. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S6: {} rounds of {} increments then a delete; {} confirmed, {} in doubt, {} refused", + self.rounds_recorded, + self.increments_per_round, + self.deletes_confirmed, + self.deletes_indeterminate, + self.deletes_rejected + )]; + lines.push(format!( + "S6: {} confirmed deletes left a slot that came back as a new agent; {} in doubt \ + deleted anyway, {} in doubt did not", + self.deleted_exactly, self.indeterminate_that_deleted, self.indeterminate_that_did_not + )); + if self.unjudgeable > 0 || self.unprobed > 0 { + lines.push(format!( + "S6: {} rounds could not be judged (an increment never answered) and {} were \ + never probed by a following increment", + self.unjudgeable, self.unprobed + )); + } + for caught in &self.caught_by_the_kill { + lines.push(format!( + "S6 {}: {} deletes across {} agents were unresolved when the kill landed — {} \ + confirmed, {} in doubt, {} refused", + caught.group.as_str(), + caught.deletes, + caught.agents, + caught.confirmed, + caught.indeterminate, + caught.rejected + )); + } + for cell in &self.cells { + lines.push(format!( + "S6 {} {}: {} rounds, {} deleted, {} survived, {} violations, {} unjudgeable, \ + {} unprobed", + cell.group.as_str(), + cell.window.as_str(), + cell.rounds, + cell.deleted, + cell.survived, + cell.violations, + cell.unjudgeable, + cell.unprobed + )); + } + lines + } +} + +fn count(rounds: &[&DeleteRound], outcome: Outcome) -> u64 { + rounds.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +fn detail_for(violation: ResurrectionViolation, before: u64, observed: u64) -> String { + match violation { + ResurrectionViolation::ResurrectedWithState => format!( + "the platform confirmed the deletion of an agent worth {before}, and the slot came \ + back worth {observed} — the old agent is still there with its state intact" + ), + ResurrectionViolation::PartialState => format!( + "the agent was worth {before} when it was deleted and the slot came back worth \ + {observed}, which is neither a new agent ({COUNTER_OF_A_NEW_AGENT}) nor the old \ + one ({before}): part of a state it was supposed to have lost survived" + ), + ResurrectionViolation::RefusedButDeleted => format!( + "the platform refused to delete an agent worth {before} and it is gone anyway — the \ + slot came back worth {observed}" + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{DateTime, TimeDelta, Utc}; + use test_r::test; + + const ON_POD: &str = "chaos-s6-delete-0000"; + const CONTROL: &str = "chaos-s6-delete-0001"; + /// What a round builds an agent up to before deleting it. + const BEFORE: u64 = 3; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-25T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![ON_POD.to_string()], + elsewhere: vec![CONTROL.to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(120)), + } + } + + fn round( + agent: &str, + offset_secs: i64, + before: Option, + outcome: Outcome, + observed: Option, + ) -> DeleteRound { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + DeleteRound { + agent: agent.to_string(), + round: 0, + before_delete: before, + outcome, + rejected_as_not_found: false, + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(90)), + observed_after: observed, + } + } + + fn build(rounds: &[DeleteRound]) -> ResurrectionReport { + ResurrectionReport::build(rounds, &split(), Some(fault()), 3) + } + + fn violations(report: &ResurrectionReport) -> Vec { + report.findings.iter().map(|f| f.violation).collect() + } + + /// The healthy shape: the platform said the agent was gone, and the slot + /// came back counting from nothing. + #[test] + fn a_deletion_that_took_is_not_a_finding() { + let report = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(0), + )]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.deleted_exactly, 1); + assert!(!report.has_violations()); + } + + /// The failure this scenario is named for. + #[test] + fn a_confirmed_deletion_whose_agent_came_back_is_a_resurrection() { + // Worth 3, deleted, and the slot reported 4 — the old agent, incremented. + let report = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(BEFORE), + )]); + assert_eq!( + violations(&report), + vec![ResurrectionViolation::ResurrectedWithState] + ); + assert!(report.has_violations(), "this must be able to fail the run"); + let finding = &report.findings[0]; + assert_eq!(finding.before, BEFORE); + assert_eq!(finding.observed, BEFORE); + } + + /// Its opposite: refused, and gone regardless. + #[test] + fn a_refused_deletion_that_happened_anyway_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(BEFORE), Outcome::Rejected, Some(0))]); + assert_eq!( + violations(&report), + vec![ResurrectionViolation::RefusedButDeleted] + ); + } + + /// Neither a new agent nor the old one: some of a state that was supposed + /// to be gone survived. + #[test] + fn a_slot_that_came_back_part_way_is_a_finding() { + // Worth 3, so the delete was allowed to leave 0 or 3. It left 2. + let report = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(2), + )]); + assert_eq!( + violations(&report), + vec![ResurrectionViolation::PartialState] + ); + assert!( + report.findings[0].detail.contains("neither a new agent"), + "{}", + report.findings[0].detail + ); + } + + /// A delete the driver never heard back about may land either way, and + /// neither answer is a defect. + #[test] + fn a_deletion_in_doubt_may_land_either_way_without_being_a_finding() { + let report = build(&[ + round(ON_POD, -10, Some(BEFORE), Outcome::Indeterminate, Some(0)), + round( + CONTROL, + -10, + Some(BEFORE), + Outcome::Indeterminate, + Some(BEFORE), + ), + ]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.indeterminate_that_deleted, 1); + assert_eq!(report.indeterminate_that_did_not, 1); + } + + /// Why `incrementsPerRound` must be at least two, stated exactly. + /// + /// The two legal answers never collide: a fresh agent reports 1 and a + /// survivor reports `before + 1`. What one increment removes is the *gap* + /// between them, and the gap is where a partial state would show up. At + /// `before = 1` the answers are 1 and 2 with nothing in between, so + /// `PartialState` cannot fire whatever the platform does. + #[test] + fn one_increment_leaves_no_room_for_a_partial_state_to_be_seen() { + // At before = 3 there is room, and a slot landing in it is caught. + let seen = build(&[round(ON_POD, -10, Some(3), Outcome::Confirmed, Some(2))]); + assert_eq!(violations(&seen), vec![ResurrectionViolation::PartialState]); + + // At before = 1 every value is one of the two legal answers, so no + // observation can ever produce this finding. `require_delete` refuses + // the configuration rather than shipping a blind third of the oracle. + for observed in [0, 1] { + let report = build(&[round( + ON_POD, + -10, + Some(1), + Outcome::Indeterminate, + Some(observed), + )]); + assert!( + !violations(&report).contains(&ResurrectionViolation::PartialState), + "observed {observed} should be legal at before=1" + ); + } + } + + #[test] + fn a_round_whose_increments_never_answered_is_not_judged() { + let report = build(&[round(ON_POD, -10, None, Outcome::Confirmed, Some(0))]); + assert_eq!(report.unjudgeable, 1); + assert!(report.findings.is_empty()); + } + + #[test] + fn a_round_no_increment_ever_probed_is_counted_as_unprobed() { + let report = build(&[round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, None)]); + assert_eq!(report.unprobed, 1); + assert!(report.findings.is_empty()); + } + + /// The S10 lesson: a kill that caught nothing proves nothing, and the run + /// has to say so rather than read as clean. + #[test] + fn a_kill_that_caught_no_delete_says_the_run_proved_nothing() { + let report = build(&[round( + ON_POD, + -100, + Some(BEFORE), + Outcome::Confirmed, + Some(0), + )]); + assert!(report.caught_by_the_kill.is_empty()); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("caught no delete in flight")), + "attention was {:?}", + report.attention_lines() + ); + } + + /// A delete still unresolved when the pod died is the population the whole + /// scenario is about. + #[test] + fn deletes_unresolved_when_the_pod_died_are_reported_separately() { + let mut caught = round(ON_POD, -1, Some(BEFORE), Outcome::Indeterminate, Some(0)); + caught.completed_at = Some(t0() + TimeDelta::seconds(30)); + let report = build(&[caught]); + + let entry = report + .caught_by_the_kill + .iter() + .find(|c| c.group == Group::OnPod) + .expect("the kill caught a delete"); + assert_eq!(entry.deletes, 1); + assert_eq!(entry.indeterminate, 1); + assert!( + !report + .attention_lines() + .iter() + .any(|l| l.contains("caught no delete in flight")) + ); + } + + #[test] + fn rounds_are_split_by_group_and_window() { + let report = build(&[ + round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, Some(0)), + round(ON_POD, 10, Some(BEFORE), Outcome::Confirmed, Some(0)), + round(CONTROL, 10, Some(BEFORE), Outcome::Confirmed, Some(0)), + ]); + let cell = |g, w| { + report + .cells + .iter() + .find(|c| c.group == g && c.window == w) + .cloned() + }; + assert_eq!(cell(Group::OnPod, Window::BeforeFault).unwrap().rounds, 1); + assert_eq!(cell(Group::OnPod, Window::DuringFault).unwrap().deleted, 1); + assert_eq!( + cell(Group::Elsewhere, Window::DuringFault).unwrap().rounds, + 1 + ); + } + + /// The bug the first S6 run reported 125 times, pinned. + /// + /// A slot's last round has no increment after it and is closed by a plain + /// read instead. A read reports the counter directly; an increment reports + /// the counter it just raised. `observed_after` means the counter the + /// deletion left behind, so the *workload* subtracts one from the increment + /// and the final read stores its value as-is. Get that backwards and every + /// slot's last round reads as a partial state — 125 findings out of 138,275 + /// rounds, all in `after-fault`, all with the same shape. + #[test] + fn a_round_closed_by_the_final_read_is_judged_on_the_same_scale() { + // A deleted agent reads 0. That is the fresh value, not a partial one. + let closed_by_read = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(COUNTER_OF_A_NEW_AGENT), + )]); + assert!( + violations(&closed_by_read).is_empty(), + "a final read of a deleted agent must not read as a partial state: {:?}", + closed_by_read.findings + ); + assert_eq!(closed_by_read.deleted_exactly, 1); + + // And a survivor read reports what it was worth, not one more. + let survivor = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Indeterminate, + Some(BEFORE), + )]); + assert!(violations(&survivor).is_empty(), "{:?}", survivor.findings); + assert_eq!(survivor.indeterminate_that_did_not, 1); + } + + /// The other thing the first run found, and the reason it is not a finding. + /// + /// Deleting is not idempotent, and worker-service retries a call whose + /// executor became unreachable. So a delete that succeeded on a pod that + /// then died is retried against the new owner and comes back + /// `AGENT_NOT_FOUND`. The agent really is gone; only the answer is wrong. + #[test] + fn a_not_found_refusal_whose_agent_is_gone_is_reported_not_failed() { + let mut r = round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Rejected, + Some(COUNTER_OF_A_NEW_AGENT), + ); + r.rejected_as_not_found = true; + let report = build(&[r]); + + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.deleted_despite_not_found, 1); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("AGENT_NOT_FOUND") && l.contains("had in fact taken effect")), + "the operator still has to be told: {:?}", + report.attention_lines() + ); + } + + /// Any *other* refusal with the agent gone is still a finding. The + /// not-found case is an exception with a mechanism behind it, not a blanket + /// excuse for refusals. + #[test] + fn a_refusal_that_is_not_about_finding_the_agent_still_fails_the_run() { + let mut r = round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Rejected, + Some(COUNTER_OF_A_NEW_AGENT), + ); + r.rejected_as_not_found = false; + let report = build(&[r]); + + assert_eq!( + violations(&report), + vec![ResurrectionViolation::RefusedButDeleted] + ); + assert!(report.has_violations()); + } + + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let rounds: Vec = (0..MAX_FINDINGS + 4) + .map(|_| round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, Some(BEFORE))) + .collect(); + let report = build(&rounds); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 4); + assert!(report.has_violations()); + } +} diff --git a/integration-tests/src/chaos/reverts.rs b/integration-tests/src/chaos/reverts.rs new file mode 100644 index 0000000000..e21e15c351 --- /dev/null +++ b/integration-tests/src/chaos/reverts.rs @@ -0,0 +1,334 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Agents that build state up and then ask the platform to take it back +//! (GOL-371). +//! +//! One emitter per agent, running rounds. A round is `increments_per_round` +//! increments followed by one `revert` of the last `revert_invocations` of +//! them. Both numbers are exact, which is what gives this scenario an oracle +//! with no band of doubt in it: the last increment of a round *returns* the +//! counter's value, so the driver knows exactly what the agent was worth before +//! the revert, and afterwards there are exactly two legitimate answers. +//! +//! ### The revert must never be retried +//! +//! Every other operation in this suite retries once, under the original +//! idempotency key, and that retry is load-bearing: it is what exposes +//! duplicate execution. A revert has no such key and is not idempotent. +//! Reverting "the last two invocations" twice takes back four. So a revert +//! whose response was lost but which actually landed would, on retry, revert +//! again — and the result would be indistinguishable from the platform tearing +//! a truncation, which is precisely the finding this scenario exists to make. +//! +//! The retry is therefore switched off for the revert call alone, and left on +//! for the increments around it. This is the only place in the suite that does +//! that, and it is not an oversight anywhere else. +//! +//! ### How a round is judged +//! +//! Not by reading the counter: a read is an invocation, and it would land in +//! the oplog between the increments and the next revert, shifting what "the +//! last N invocations" means. The **next round's first increment** is the probe +//! instead. It returns the new value, so the value the revert left behind is +//! that minus one, and it costs nothing extra. + +use crate::chaos::history::{Outcome, Stream}; +use crate::chaos::workload::{self, WorkloadContext}; +use crate::chaos::{RetryPolicy, RevertConfig}; +use chrono::{DateTime, Utc}; +use golem_common::model::worker::{RevertLastInvocations, RevertWorkerTarget}; +use golem_test_framework::dsl::TestDsl; +use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::task::JoinSet; +use tracing::info; + +/// One round, as the driver observed it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RevertRound { + pub agent: String, + pub round: u32, + /// The counter value the round's last increment returned, so the value the + /// agent was worth immediately before the revert. `None` when an increment + /// in the round did not answer, which leaves the round unjudgeable rather + /// than failed. + pub before_revert: Option, + /// How many invocations this round's revert asked to take back. + pub asked_to_revert: u32, + /// What the revert call itself returned. + pub outcome: Outcome, + pub submitted_at: DateTime, + pub completed_at: Option>, + /// The counter value observed after the revert, taken from the next round's + /// first increment. `None` for the last round of the run, which the final + /// read-back answers instead. + pub observed_after: Option, +} + +/// The counter an operation left behind, given what the *next* increment +/// reported. +/// +/// Same conversion as [`crate::chaos::deletions::counter_left_by`] and named for +/// the same reason: the account judges `observed_after` as a counter, and the +/// increment probe reports one more than that. +pub fn counter_left_by(increment_returned: u64) -> u64 { + increment_returned.saturating_sub(1) +} + +/// A running revert workload. +pub struct RevertHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, + rounds: Arc>>, +} + +impl RevertHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + /// Rounds recorded so far, in no particular order. + pub fn rounds(&self) -> Vec { + self.rounds.lock().map(|r| r.clone()).unwrap_or_default() + } + + /// Signals every emitter to stop and waits for operations in flight to + /// record themselves. + pub async fn stop(mut self) -> Vec { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + let rounds = self.rounds(); + info!( + "Chaos revert workload stopped after {} operations across {} rounds", + self.submitted(), + rounds.len() + ); + rounds + } +} + +/// The agents a run of `count` emitters drives, in index order. +pub fn agent_names(ctx: &WorkloadContext, count: u32) -> Vec { + (0..count) + .map(|index| ctx.agent_name(Stream::Revert, index)) + .collect() +} + +/// Starts one emitter per agent. +pub fn start(ctx: WorkloadContext, config: &RevertConfig) -> RevertHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let rounds: Arc>> = Arc::new(Mutex::new(Vec::new())); + let mut tasks = JoinSet::new(); + + info!( + "Chaos revert workload starting: {} emitters, {} increments then a revert of {} per \ + round, {:?} between rounds", + config.agents, + config.increments_per_round, + config.revert_invocations, + config.interval() + ); + + for index in 0..config.agents { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + let rounds = rounds.clone(); + let config = config.clone(); + + tasks.spawn(async move { + let agent = ctx.agent_name(Stream::Revert, index); + let mut round: u32 = 0; + // Index into `rounds` of the round still waiting to be judged by + // the next increment this agent runs. + let mut pending: Option = None; + + while stop.load(Ordering::Relaxed) == 0 { + // ── Increments ────────────────────────────────────────────── + let mut last_value = None; + let mut all_answered = true; + for step in 0..config.increments_per_round { + if stop.load(Ordering::Relaxed) != 0 { + break; + } + let seq = round as u64 * config.increments_per_round as u64 + step as u64; + submitted.fetch_add(1, Ordering::Relaxed); + let value = increment(&ctx, &agent, seq).await; + + // The first increment of a round is also the probe that + // says what the previous round's revert left behind. + if let Some(slot) = pending.take() + && let Some(observed) = value + && let Ok(mut rounds) = rounds.lock() + && let Some(entry) = rounds.get_mut(slot) + { + entry.observed_after = Some(counter_left_by(observed)); + } + + match value { + Some(v) => last_value = Some(v), + None => all_answered = false, + } + } + if stop.load(Ordering::Relaxed) != 0 { + break; + } + + // ── Revert ────────────────────────────────────────────────── + let submitted_at = Utc::now(); + submitted.fetch_add(1, Ordering::Relaxed); + let outcome = revert_once(&ctx, &agent, round, &config).await; + + if let Ok(mut rounds) = rounds.lock() { + rounds.push(RevertRound { + agent: agent.clone(), + round, + before_revert: all_answered.then_some(last_value).flatten(), + asked_to_revert: config.revert_invocations, + outcome, + submitted_at, + completed_at: Some(Utc::now()), + observed_after: None, + }); + pending = Some(rounds.len() - 1); + } + + round += 1; + if stop.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(config.interval()).await; + } + } + }); + } + + RevertHandle { + stop, + tasks, + submitted, + rounds, + } +} + +/// One increment, returning the counter value it reported. +async fn increment(ctx: &WorkloadContext, agent: &str, seq: u64) -> Option { + let key = ctx.idempotency_key(agent, seq); + workload::increment_counter(ctx, Stream::Revert, agent, key) + .await + .value + .map(u64::from) +} + +/// One revert, with retries switched off. See the module docs. +async fn revert_once( + ctx: &WorkloadContext, + agent: &str, + round: u32, + config: &RevertConfig, +) -> Outcome { + let mut once = ctx.clone(); + once.retry = RetryPolicy { + transport_only: true, + max_retries: 0, + delay_secs: 0, + }; + + let key = format!("{agent}-revert-{round:08}"); + let agent_id = workload::counter_agent_id(&once, agent); + let number_of_invocations = config.revert_invocations; + let ctx2 = once.clone(); + + workload::run_operation( + &once, + Stream::Revert, + agent.to_string(), + "revert", + key, + |_| { + let ctx = ctx2.clone(); + let agent_id = agent_id.clone(); + async move { + ctx.user + .revert( + &agent_id, + RevertWorkerTarget::RevertLastInvocations(RevertLastInvocations { + number_of_invocations: number_of_invocations as u64, + }), + ) + .await?; + Ok(None) + } + }, + ) + .await + .outcome +} + +/// The value a completed run of `rounds` rounds should leave on an agent whose +/// every round landed. +pub fn expected_after(config: &RevertConfig, rounds: u32) -> u64 { + config.net_per_round() as u64 * rounds as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + fn config() -> RevertConfig { + RevertConfig { + agents: 200, + increments_per_round: 4, + revert_invocations: 2, + interval_millis: 500, + recovery_budget_secs: 60, + } + } + + /// The arithmetic the whole oracle rests on: a round that lands is worth + /// exactly its increments less what the revert took back. + #[test] + fn a_completed_round_is_worth_its_increments_less_the_revert() { + assert_eq!(config().net_per_round(), 2); + assert_eq!(expected_after(&config(), 0), 0); + assert_eq!(expected_after(&config(), 10), 20); + } + + /// The increment probe and the final read have to land on the same scale. + /// See [`crate::chaos::deletions::counter_left_by`] for the run that proved + /// what happens when they do not. + #[test] + fn the_two_probes_agree_on_what_a_revert_left_behind() { + assert_eq!(counter_left_by(9), 8); + assert_eq!(counter_left_by(1), 0); + assert_eq!(counter_left_by(0), 0); + } + + /// A revert that takes back everything it added is legal and leaves the + /// agent where it started. Nothing in the arithmetic may go negative. + #[test] + fn a_round_that_reverts_everything_it_added_is_worth_nothing() { + let config = RevertConfig { + increments_per_round: 3, + revert_invocations: 3, + ..config() + }; + assert_eq!(config.net_per_round(), 0); + assert_eq!(expected_after(&config, 100), 0); + } +} diff --git a/integration-tests/src/chaos/rollback.rs b/integration-tests/src/chaos/rollback.rs new file mode 100644 index 0000000000..49752eb2a3 --- /dev/null +++ b/integration-tests/src/chaos/rollback.rs @@ -0,0 +1,399 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Did every agent come back to the build it was rolled back to (GOL-369)? +//! +//! S5 asks whether agents reach a *new* build when an executor dies mid-update. +//! S9 asks the return question, and the return leg is the one that matters +//! operationally: a rollback is what you reach for when the new build is +//! already going wrong, so it happening under a dying executor is exactly the +//! situation you would be in. +//! +//! ### Why the evidence is the running code, not the metadata +//! +//! `Counter::component_version` is compiled into each build — `1` in +//! `agent-counters`, `2` in `agent-counters-v2`, and nothing else differs +//! between them. Component metadata says which revision the platform *believes* +//! an agent is on; invoking `component_version` says what the code actually +//! executing reports. Only the second can distinguish a rollback that landed +//! from one the platform merely recorded. +//! +//! ### Why the forward leg is verified before the backward one +//! +//! If the agents never reached the new build, rolling them back returns them to +//! a build they never left, every check passes, and the run proves nothing. So +//! the forward leg is measured and the rollback is refused outright if too few +//! agents made it. The same instinct as S6's smoke round and S10's +//! how-much-did-the-kill-catch line: a clean report from a scenario that never +//! happened is the worst artifact this suite can produce. +//! +//! ### Why the control-plane retries are counted apart +//! +//! The workload's retry is one attempt, transport-only, under the original +//! idempotency key, and it exists to *expose* duplicate execution. The +//! rollback's per-agent update requests are a different thing entirely: they are +//! control-plane calls aimed at agents whose executor is about to be killed, and +//! a request refused because its owner just died says nothing about the +//! platform's correctness. Counting them together would let control-plane noise +//! read as workload trouble. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// What the agents were running when asked. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VersionCensus { + /// When the census was taken, for the report to label it. + pub at: String, + /// The version every agent was expected to report. + pub expected: u32, + pub agents: usize, + pub on_expected: usize, + /// Agents on some other build, by the version they reported. Non-empty is + /// the finding; the key says which build they are stuck on. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub on_other: BTreeMap, + /// Agents that answered nothing. Neither passed nor failed: an agent that + /// cannot be read says nothing either way about which build it is on. + pub unreadable: usize, +} + +impl VersionCensus { + /// Builds a census from what each agent reported. + pub fn build(at: &str, expected: u32, observed: &BTreeMap>) -> Self { + let mut on_expected = 0; + let mut unreadable = 0; + let mut on_other: BTreeMap = BTreeMap::new(); + for version in observed.values() { + match version { + Some(v) if *v == expected => on_expected += 1, + Some(v) => *on_other.entry(*v).or_default() += 1, + None => unreadable += 1, + } + } + VersionCensus { + at: at.to_string(), + expected, + agents: observed.len(), + on_expected, + on_other, + unreadable, + } + } + + /// The share of agents on the expected build, out of those that answered. + /// + /// Unreadable agents are excluded from both halves rather than counted as + /// failures: they are reported separately, and treating silence as a wrong + /// answer would let a flaky read block a rollback that was fine. + pub fn share_of_answered_percent(&self) -> Option { + let answered = self.agents - self.unreadable; + (answered > 0).then(|| self.on_expected as f64 * 100.0 / answered as f64) + } +} + +/// Rollback requests, counted apart from the workload's own retries. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ControlPlaneAttempts { + pub requested: u64, + pub accepted_first_try: u64, + pub accepted_after_retry: u64, + /// Requests that never got through, even after the configured retries. Each + /// one is an agent nobody asked to come back, so it explains a stale agent + /// without excusing one. + pub refused: u64, + pub max_retries: u32, +} + +impl ControlPlaneAttempts { + pub fn accepted(&self) -> u64 { + self.accepted_first_try + self.accepted_after_retry + } +} + +/// The rollback account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RollbackReport { + /// The revision the agents were moved forward to, and the one they were + /// rolled back to. The second carries the original build's code. + pub forward_revision: u64, + pub rollback_revision: u64, + /// What the running code reports on each of those builds. + pub forward_version: u32, + pub rollback_version: u32, + /// The forward leg, measured before the rollback was attempted. + pub rolled_forward: VersionCensus, + /// After recovery. `None` if the run aborted before it got there. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rolled_back: Option, + pub control: ControlPlaneAttempts, + /// The floor the forward leg had to clear for the rollback to be worth + /// attempting, from the suite YAML. + pub rolled_forward_floor_percent: f64, +} + +impl RollbackReport { + /// Whether enough agents reached the new build for a rollback to mean + /// anything. + pub fn forward_leg_landed(&self) -> bool { + self.rolled_forward + .share_of_answered_percent() + .is_some_and(|share| share >= self.rolled_forward_floor_percent) + } + + /// Agents still on the build they were supposed to leave, after recovery. + pub fn stuck_on_the_new_build(&self) -> usize { + self.rolled_back + .as_ref() + .and_then(|census| census.on_other.get(&self.forward_version).copied()) + .unwrap_or(0) + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines = Vec::new(); + + if !self.forward_leg_landed() { + lines.push(format!( + "S9: only {} of {} agents reached revision {} before the rollback ({}% of those \ + that answered, against a {:.0}% floor). Rolling agents back to a build they \ + never left proves nothing, so this run does not test rollback.", + self.rolled_forward.on_expected, + self.rolled_forward.agents, + self.forward_revision, + self.rolled_forward + .share_of_answered_percent() + .map(|s| format!("{s:.1}")) + .unwrap_or_else(|| "n/a".to_string()), + self.rolled_forward_floor_percent + )); + } + + let stuck = self.stuck_on_the_new_build(); + if stuck > 0 { + lines.push(format!( + "S9: {stuck} agent(s) still report component version {} after recovery, not the \ + {} they were rolled back to", + self.forward_version, self.rollback_version + )); + } + + if self.control.refused > 0 { + lines.push(format!( + "S9: {} of {} rollback requests were refused even after {} control-plane \ + retries. Those agents were never asked to come back, which explains a stale \ + agent without excusing one.", + self.control.refused, self.control.requested, self.control.max_retries + )); + } + + if let Some(census) = &self.rolled_back + && census.unreadable > 0 + { + lines.push(format!( + "S9: {} agent(s) could not be read after recovery, so the run cannot say which \ + build they are on", + census.unreadable + )); + } + lines + } + + /// Lines a reader needs in order to interpret the run. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S9: revision {} carries build v{}, revision {} carries build v{} again", + self.forward_revision, + self.forward_version, + self.rollback_revision, + self.rollback_version + )]; + lines.push(format!( + "S9 forward leg: {} of {} agents on version {} before the rollback ({} unreadable)", + self.rolled_forward.on_expected, + self.rolled_forward.agents, + self.forward_version, + self.rolled_forward.unreadable + )); + lines.push(format!( + "S9 rollback requests: {} asked, {} accepted first try, {} after a retry, {} refused \ + (up to {} control-plane retries)", + self.control.requested, + self.control.accepted_first_try, + self.control.accepted_after_retry, + self.control.refused, + self.control.max_retries + )); + if let Some(census) = &self.rolled_back { + lines.push(format!( + "S9 return leg: {} of {} agents on version {} after recovery ({} unreadable, \ + {:?} elsewhere)", + census.on_expected, + census.agents, + census.expected, + census.unreadable, + census.on_other + )); + } + lines + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + fn census( + expected: u32, + on_expected: usize, + on_other: &[(u32, usize)], + unreadable: usize, + ) -> VersionCensus { + let mut observed: BTreeMap> = BTreeMap::new(); + let mut n = 0; + for _ in 0..on_expected { + observed.insert(format!("agent-{n:04}"), Some(expected)); + n += 1; + } + for (version, count) in on_other { + for _ in 0..*count { + observed.insert(format!("agent-{n:04}"), Some(*version)); + n += 1; + } + } + for _ in 0..unreadable { + observed.insert(format!("agent-{n:04}"), None); + n += 1; + } + VersionCensus::build("test", expected, &observed) + } + + fn report(forward: VersionCensus, back: Option) -> RollbackReport { + RollbackReport { + forward_revision: 2, + rollback_revision: 3, + forward_version: 2, + rollback_version: 1, + rolled_forward: forward, + rolled_back: back, + control: ControlPlaneAttempts { + requested: 200, + accepted_first_try: 200, + max_retries: 2, + ..Default::default() + }, + rolled_forward_floor_percent: 90.0, + } + } + + /// An unreadable agent is not a wrong answer. Counting it as one would let a + /// flaky read block a rollback that was perfectly fine. + #[test] + fn silence_is_excluded_from_the_share_rather_than_counted_against_it() { + // 90 on the expected build, 10 silent, none actually wrong. + let c = census(2, 90, &[], 10); + assert_eq!(c.agents, 100); + assert_eq!(c.unreadable, 10); + assert_eq!(c.share_of_answered_percent(), Some(100.0)); + assert!(report(c, None).forward_leg_landed()); + } + + /// The gate the whole scenario rests on: rolling agents back to a build + /// they never left would pass every check downstream. + #[test] + fn a_forward_leg_that_did_not_land_refuses_the_rollback() { + // 50 of 100 answered agents made it, against a 90% floor. + let r = report(census(2, 50, &[(1, 50)], 0), None); + assert!(!r.forward_leg_landed()); + assert!( + r.attention_lines() + .iter() + .any(|l| l.contains("does not test rollback")), + "the operator has to be told the run proved nothing: {:?}", + r.attention_lines() + ); + } + + /// A census with nothing readable cannot clear the gate, rather than + /// clearing it vacuously on an empty average. + #[test] + fn a_census_nobody_answered_does_not_clear_the_gate() { + let c = census(2, 0, &[], 40); + assert_eq!(c.share_of_answered_percent(), None); + assert!(!report(c, None).forward_leg_landed()); + } + + /// The finding: agents still on the build they were rolled back from. + #[test] + fn agents_still_on_the_old_build_after_recovery_are_raised() { + let r = report(census(2, 200, &[], 0), Some(census(1, 197, &[(2, 3)], 0))); + assert_eq!(r.stuck_on_the_new_build(), 3); + assert!( + r.attention_lines() + .iter() + .any(|l| l.contains("still report component version 2")), + "{:?}", + r.attention_lines() + ); + } + + /// A clean return raises nothing. + #[test] + fn a_rollback_that_landed_everywhere_raises_nothing() { + let r = report(census(2, 200, &[], 0), Some(census(1, 200, &[], 0))); + assert_eq!(r.stuck_on_the_new_build(), 0); + assert!(r.attention_lines().is_empty(), "{:?}", r.attention_lines()); + } + + /// A refused control-plane request explains a stale agent without excusing + /// one, so it is raised even when the return leg otherwise looks clean. + #[test] + fn refused_rollback_requests_are_raised_even_on_a_clean_return() { + let mut r = report(census(2, 200, &[], 0), Some(census(1, 200, &[], 0))); + r.control = ControlPlaneAttempts { + requested: 200, + accepted_first_try: 190, + accepted_after_retry: 6, + refused: 4, + max_retries: 2, + }; + assert_eq!(r.control.accepted(), 196); + assert!( + r.attention_lines() + .iter() + .any(|l| l.contains("were refused even after")), + "{:?}", + r.attention_lines() + ); + } + + /// Retries are counted apart from first-try acceptances, because the two + /// say different things about how the control plane behaved under a kill. + #[test] + fn first_try_and_retried_acceptances_are_counted_apart() { + let control = ControlPlaneAttempts { + requested: 10, + accepted_first_try: 7, + accepted_after_retry: 2, + refused: 1, + max_retries: 2, + }; + assert_eq!(control.accepted(), 9); + assert_eq!(control.accepted() + control.refused, control.requested); + } +} diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index 4ced61c0fe..88459b1076 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -14,9 +14,12 @@ //! Chaos scenario implementations. //! -//! One module per scenario code. Each one owns its phase choreography — which -//! is the part that differs, and the part worth reading — while everything -//! around it lives here: where artifacts go, how a signal failure becomes a +//! One module per scenario code, except where two codes are the same +//! choreography under different settings: `storage_fault` runs S14, S16, S17, +//! S18 and S22, which differ in which store the fault is aimed at, what the +//! fault does to it, and for how long. Each +//! module owns its phase choreography — which is the part that differs, and the +//! part worth reading — while everything around it lives here: where artifacts go, how a signal failure becomes a //! termination reason, how a routing table is sampled, and how a result is //! assembled. //! @@ -26,19 +29,27 @@ //! produce a plausible-looking report from a wasted maintenance window. pub mod s1; +pub mod s10; +pub mod s11; pub mod s12; pub mod s13; +pub mod s3; pub mod s5; +pub mod s6; +pub mod s7; pub mod s8; +pub mod s9; +pub mod storage_fault; use crate::chaos::ScenarioConfig; use crate::chaos::history::{OperationHistory, OperationRecord, Stream}; use crate::chaos::ownership::OwnershipSample; use crate::chaos::pinned::PinnedSelection; use crate::chaos::result::{ChaosResult, Phases, RESULT_SCHEMA_VERSION, RunScope}; +use crate::chaos::scheduled::ScheduledSelection; use crate::chaos::signal::SignalError; use crate::chaos::summary::{ - AgentReadback, ChaosSummary, ExactlyOnceReport, RoutingSnapshot, TerminationReason, + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, RoutingSnapshot, TerminationReason, }; use crate::chaos::workload::{self, WorkloadContext}; use chrono::{DateTime, Utc}; @@ -76,6 +87,21 @@ pub struct ScenarioOutcome { pub termination_reason: TerminationReason, /// Present only for scenarios that pin the fault to one executor. pub pinned_selection: Option, + /// Present only for S10, which divides its targets around the executor the + /// fault was aimed at rather than driving only the ones it owns. + pub scheduled_selection: Option, + /// Present only for S11, which divides its waiters around the executor the + /// fault was aimed at the same way S10 divides its targets. + pub promise_selection: Option, + /// Present only for S3, which divides its agents around the executor the + /// partition cuts off rather than around one that dies. + pub isolation_selection: Option, + /// Present only for S7, which divides the agents whose state is being + /// reverted around the executor the kill is aimed at. + pub revert_selection: Option, + /// Present only for S6, which divides the agent slots being deleted around + /// the executor the kill is aimed at. + pub delete_selection: Option, } /// Assembles the archived result. @@ -99,6 +125,18 @@ pub fn build_result(config: &ScenarioConfig, outcome: ScenarioOutcome) -> ChaosR workload: config.workload.clone(), pinned: config.pinned.clone(), pinned_selection: outcome.pinned_selection, + scheduled: config.scheduled.clone(), + scheduled_selection: outcome.scheduled_selection, + promise: config.promise.clone(), + promise_selection: outcome.promise_selection, + isolation: config.isolation.clone(), + isolation_selection: outcome.isolation_selection, + revert: config.revert.clone(), + revert_selection: outcome.revert_selection, + delete: config.delete.clone(), + delete_selection: outcome.delete_selection, + rollback: config.rollback.clone(), + storage: config.storage.clone(), retry_policy: config.retry_policy.clone(), scope: outcome.scope, summary: outcome.summary, @@ -201,11 +239,14 @@ const ROUTING_POLL_SECS: u64 = 3; /// Blocks until the routing table covers every shard, or the timeout lapses. /// /// Returns the line to record, so the result says which of the two happened -/// rather than leaving a reader to infer it from timings. +/// rather than leaving a reader to infer it from timings. A settled table is +/// context — it is what every healthy run reports. An unsettled one is a +/// finding, because the baseline then measures convergence rather than the +/// platform. pub async fn wait_for_settled_routing( deps: &BenchmarkTestDependencies, snapshots: &mut Vec, -) -> String { +) -> Note { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(ROUTING_SETTLE_TIMEOUT_SECS); // Assigned on every path through the loop below before it is read. @@ -224,7 +265,7 @@ pub async fn wait_for_settled_routing( if assigned == total && executors > 0 { snapshots.push(snapshot_routing(deps, "settled-before-start").await); info!("Chaos: {last} — settled"); - return format!("{last} (settled before measuring)"); + return Note::context(format!("{last} (settled before measuring)")); } info!("Chaos: {last} — waiting for the table to cover every shard"); } @@ -237,11 +278,11 @@ pub async fn wait_for_settled_routing( if std::time::Instant::now() >= deadline { snapshots.push(snapshot_routing(deps, "unsettled-before-start").await); warn!("Chaos: {last} — proceeding anyway after {ROUTING_SETTLE_TIMEOUT_SECS}s"); - return format!( + return Note::attention(format!( "WARNING: measured against an unsettled cluster — {last}. \ Baseline numbers may reflect routing convergence rather than the \ platform." - ); + )); } tokio::time::sleep(std::time::Duration::from_secs(ROUTING_POLL_SECS)).await; } diff --git a/integration-tests/src/chaos/scenarios/s1.rs b/integration-tests/src/chaos/scenarios/s1.rs index 1b055901a3..be382bcd9f 100644 --- a/integration-tests/src/chaos/scenarios/s1.rs +++ b/integration-tests/src/chaos/scenarios/s1.rs @@ -76,7 +76,9 @@ use crate::chaos::scenarios::{ wait_for_settled_routing, warm_up, write_outputs, }; use crate::chaos::signal::{BaselineReady, FaultSignals, ScaleEvent}; -use crate::chaos::summary::{AgentReadback, ChaosSummary, ExactlyOnceReport, TerminationReason}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; use crate::chaos::{ScenarioCode, ScenarioConfig}; use chrono::Utc; @@ -162,7 +164,7 @@ pub async fn run( let mut fault_id = None; let mut fault_target_observed = None; let mut inconclusive: Option = None; - let mut attention_extra: Vec = Vec::new(); + let mut attention_extra: Vec = Vec::new(); // Sample the assignment continuously for the whole run, alongside the // labelled phase-boundary samples. See ASSIGNMENT_SAMPLE_INTERVAL. @@ -216,7 +218,7 @@ pub async fn run( if let Some(detail) = inconclusive.clone() { summary.attention.push(detail); } - summary.attention.extend(attention_extra.clone()); + summary.absorb(attention_extra.clone()); if let Some(report) = $exactly_once { summary = summary.with_exactly_once(report); } @@ -233,6 +235,11 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; @@ -557,17 +564,17 @@ pub async fn run( holds {smallest} against a balanced {balanced}" ); if executors < expected { - attention_extra.push(format!( + attention_extra.push(Note::attention(format!( "executors were scaled back to {expected} during the fault, but only \ {executors} hold shards after settling — the cluster did not take the \ restored executor back" - )); + ))); } else if smallest * 2 < balanced { - attention_extra.push(format!( + attention_extra.push(Note::attention(format!( "after settling the least-loaded executor holds {smallest} shards against a \ balanced {balanced}: the cluster took the executor back but has not \ rebalanced onto it" - )); + ))); } } diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs new file mode 100644 index 0000000000..7008272c0d --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -0,0 +1,725 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S10 — executor crash during scheduled-action fire (GOL-378). +//! +//! Every other scenario in this suite kills an executor while the driver is +//! holding a connection to it. S10 kills one while it is holding a *promise*: +//! work the platform accepted, acknowledged, and undertook to run later. Nobody +//! is waiting on the other end of a socket for it, which is exactly why it is +//! worth testing separately. A dropped invocation shows up as a failed request. +//! A dropped scheduled action shows up as nothing at all. +//! +//! ## The two windows, and which one a kill reliably lands in +//! +//! An executor's scheduler claims the actions that are due for the shards it +//! owns, leases each one, enqueues the invocation, and acknowledges. That +//! claim-to-acknowledge window is milliseconds wide: the action itself is a +//! near-no-op, and once the invocation is enqueued it is durable in the target +//! agent's oplog and covered by ordinary worker recovery rather than by the +//! lease. Killing an executor inside it is possible, not aimable, and its +//! signature is a fire delayed by roughly the lease TTL. The run reports when +//! that happens instead of claiming to have arranged it. +//! +//! The window a kill does land in, every time, is the wide one. At any instant +//! several hundred actions are registered and not yet due, and the shards they +//! belong to are owned by an executor that is about to stop existing. Nothing +//! claims them until the shards move, so the delay this scenario measures is +//! dominated by shard reassignment rather than by lease expiry. Both end in the +//! same question: does an accepted action still run, exactly once, once the +//! cluster has finished rearranging itself. +//! +//! ## Why the target agent records tokens rather than counting +//! +//! S12 already counts scheduled fires and compares the total against what the +//! driver registered. That is enough to notice that *something* went wrong +//! across a whole population, and useless for saying what. Here every +//! registration carries its own idempotency key into the scheduled action, and +//! the target agent records it when the action runs. Pairing tokens turns both +//! failures into statements about one named action: this registration, accepted +//! at this time, due at this time, never ran. See [`crate::chaos::fires`]. +//! +//! ## Why the kill is aimed, and why every target is still driven +//! +//! The driver picks the executor owning the largest share of targets and names +//! it in the readiness signal, the same way S8 does. Without that, `mode: one` +//! would pick a pod at random and the run could not say which actions were +//! supposed to be disturbed. +//! +//! Unlike S8, though, the targets it does *not* own keep running too. They are +//! the control group: on a two-executor cluster roughly half the population is +//! never touched, and reporting one percentile over both would let a lease +//! recovery that took its full TTL hide behind the half that was never +//! disturbed. +//! +//! ## What the run measures the kill against +//! +//! The driver cannot see a claim, and does not pretend to. Every target +//! re-registers on a fixed cadence at a fixed lead, so actions arrive into +//! every part of the cycle continuously and there is always a population +//! registered but not yet due. What the run then records is how large that +//! population actually was at the instant the pod died, and how much of it was +//! on the pod — measured from the history, not assumed from the cadence, +//! because a platform that had slowed down would have registered fewer than the +//! arithmetic says. A kill that caught none of it is reported as a warning: a +//! clean account of a mechanism that was never disturbed is not evidence. +//! +//! ## What fails the run +//! +//! Only the three token-level violations in [`ScheduleFireReport`]. Fire delay +//! is reported against the configured lease budget as SLO evidence, not +//! asserted: how much a lease recovery may cost is a judgement, and the number +//! that matters is in the result either way. +//! +//! ## Reading a fire delay: registration first, scheduler second +//! +//! A large fire delay does not necessarily mean the scheduler was slow. The +//! registering invocation mints its action's due time *before* the call goes +//! out, so a registration that stalls in the client describes an action that was +//! already overdue when the platform first heard about it. Those fire instantly +//! and correctly, and would still arrive in the percentiles as minutes late. +//! +//! [`crate::chaos::fires`] holds them out of the scheduler-delay cells and +//! reports them as `overdue_on_arrival` instead. Two things follow for anyone +//! reading a result: +//! +//! * Check the registration latencies before concluding anything about the +//! scheduler. A client-side stall and a lease recovery look identical in a +//! delay percentile and have nothing to do with each other. +//! * Read the `overdue_on_arrival` count together with its worst case, never +//! alone. The classification catches any registration slower than the +//! configured lead, so a stall shrinking from minutes to seconds moves entries +//! *into* the bucket rather than out of it: the count can rise while the +//! platform gets strictly better. +//! +//! Run-by-run findings live in the S10 runbook in golem-cloud, not here. + +use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; +use crate::chaos::history::{ + OperationHistory, OperationRecord, Outcome, Phase, Stream, TargetFireLog, +}; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, readback_for, signal_termination, + snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::scheduled::{self, ScheduledSelection}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::summary::{AgentReadback, ChaosSummary, Note, TerminationReason}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig, ScheduledConfig}; +use chrono::{DateTime, TimeDelta, Utc}; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::collections::BTreeSet; +use std::time::Duration; +use tracing::{info, warn}; + +/// Extra quiet after the last registration's action is due, before the fire +/// logs are read. +/// +/// The rest of the settle is derived from the configuration rather than fixed: +/// the final registration falls due one `lead` after the workload stops, and if +/// its executor died holding the claim, the recovery costs up to one lease +/// budget on top. Reading before that elapsed would report actions as lost that +/// were merely late, which is the one mistake this scenario cannot afford. +const SETTLE_MARGIN: Duration = Duration::from_secs(30); + +/// How many targets to sample after the baseline to prove actions are firing at +/// all. +/// +/// A handful, because this is a smoke test rather than a measurement: if the +/// scheduling path is broken, every target is equally broken, and the point is +/// to fail before spending the fault window on a run that would report a clean +/// account of nothing. +const FIRE_PROOF_SAMPLE: usize = 5; + +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let scheduled_config = config.require_scheduled()?; + let history = OperationHistory::new(ScenarioCode::S10.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S10); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let targets: Vec = (0..scheduled_config.targets) + .map(|index| ctx.schedule_target_name(index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $fires:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if let Some(report) = $fires { + summary = summary.with_schedule_fires(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: selection.clone(), + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + info!( + "S10: warming {} emitters and targets before the baseline", + targets.len() + ); + let warmed = scheduled::warm(&ctx, &targets).await; + info!("S10: warmed {warmed} agents, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim the fault ─────────────────────────────────────────────────────── + // Before the baseline, because a run that cannot be aimed should not spend + // a maintenance window proving it. + let chosen = match scheduled::select(&ctx, deps, &targets).await { + Ok(chosen) => chosen, + Err(e) => { + warn!("S10: could not aim the fault at an executor: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None + ); + } + }; + selection = Some(chosen.clone()); + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S10: baseline phase, {} targets registering every {:?} at a {:?} lead, for {:?}", + targets.len(), + scheduled_config.interval(), + scheduled_config.lead(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = scheduled::start(ctx.clone(), &targets, scheduled_config); + tokio::time::sleep(config.phases.baseline()).await; + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S10: baseline registered nothing, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no scheduled registration succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None + ); + } + + // Registering is not firing. A platform that accepted every registration and + // scheduled none of them would otherwise reach read-back and report a + // flawless account of a mechanism that never ran. + let sampled = sample_fire_count(&ctx, &targets).await; + if sampled == 0 { + warn!("S10: {baseline_operations} registrations accepted and no action has fired"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::StreamNeverSucceeded { + stream: Stream::Scheduled.to_string(), + }, + &records, + Vec::new(), + None + ); + } + info!( + "S10: baseline complete ({baseline_operations} registrations, {sampled} fires across a \ + sample of {} targets)", + FIRE_PROOF_SAMPLE.min(targets.len()) + ); + + // ── Verify ownership, then signal ─────────────────────────────────────── + if let Err(e) = scheduled::verify_ownership(&ctx, deps, &chosen).await { + warn!("S10: target ownership no longer holds, refusing to inject: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None + ); + } + + info!( + "S10: signalling readiness with fault target {} ({} of {} targets on it)", + chosen.pod_address, + chosen.on_pod.len(), + targets.len() + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S10.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: chosen.pod_address.clone(), + pod_ip: chosen.pod_ip.clone(), + owned_agents: chosen.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S10: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None); + } + }; + info!( + "S10: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + // How much work was actually in the window the scenario is about. Measured + // from the history rather than assumed from the cadence, because a platform + // that had slowed down would have registered fewer than the arithmetic says. + let on_pod: BTreeSet = chosen.on_pod.iter().cloned().collect(); + let pending = pending_at_injection( + &history.snapshot(), + injected.injected_at, + scheduled_config.lead(), + &on_pod, + ); + attention_extra.push(pending.note()); + info!("S10: {}", pending.describe()); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S10: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None); + } + }; + info!( + "S10: fault cleared at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "S10: recovery phase, registering for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + let skipped = handle.skipped(); + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + if skipped > 0 { + attention_extra.push(Note::attention(format!( + "S10 skipped {skipped} registration ticks because targets still had their budget \ + of {} in flight — the offered rate was clamped by the platform, so the phase \ + counts understate what the run intended to submit", + scheduled::MAX_IN_FLIGHT_PER_TARGET + ))); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Account ───────────────────────────────────────────────────────────── + let settle = settle_before_readback(scheduled_config); + info!("S10: letting the last actions fall due and fire, {settle:?} before read-back"); + tokio::time::sleep(settle).await; + + let records = history.snapshot(); + let logs = scheduled::read_logs(&ctx, &targets).await; + // Archived alongside the operations, not just reduced into the report. The + // first S10 run needed a correction to its delay percentiles that could not + // be applied afterwards, because only the reduced numbers had been kept. + history.record_fire_logs(logs.clone()); + + let report = ScheduleFireReport::build( + &records, + &logs, + scheduled_config.lead(), + fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }), + &on_pod, + scheduled_config.lease_budget(), + ); + info!( + "S10: scheduled-fire account — {} registrations accepted, {} fired once, {} never \ + fired, {} inconclusive, {} unverifiable, {} findings", + report.registrations_confirmed, + report.fired_once, + report + .findings + .iter() + .filter(|f| f.violation == crate::chaos::fires::FireViolation::NeverFired) + .count(), + report.inconclusive, + report.unverifiable, + report.findings.len() + ); + if let Some(p99) = report.fault_window_p99_ms() { + info!( + "S10: fire delay p99 during the fault, on the killed executor's targets: {p99}ms \ + against a {}ms lease budget", + report.lease_budget_ms + ); + } + + // The count-based read-back as well, on the same read. It cannot localise + // anything the token pairing does not, but it is the view every other + // scenario reports and a disagreement between the two would itself be worth + // knowing about. + let readback = readback_from_polls(&records, &logs); + + let reason = if report.has_violations() { + TerminationReason::ScheduledFireViolated { + findings: report.findings.len() as u64, + first: report + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + } + } else if report.fired_once == 0 { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Scheduled.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, readback, Some(report)); +} + +/// How long to wait after the workload stops before reading the fire logs. +fn settle_before_readback(config: &ScheduledConfig) -> Duration { + config.lead() + config.lease_budget() + SETTLE_MARGIN +} + +/// Actions that were registered but not yet due when the executor died. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PendingAtInjection { + pub total: u64, + pub on_killed_executor: u64, +} + +impl PendingAtInjection { + /// Whether the kill landed anywhere near the mechanism under test. + /// + /// A run that caught nothing pending proves nothing however clean the rest + /// of its numbers look, which is the one thing here a human has to act on. + pub fn needs_attention(&self) -> bool { + self.total == 0 + } + + /// The same line as [`Self::describe`], classified. + pub fn note(&self) -> Note { + Note::leveled(self.needs_attention(), self.describe()) + } + + /// The line an operator needs in order to know whether the kill landed + /// anywhere near the mechanism under test. + pub fn describe(&self) -> String { + if self.total == 0 { + return "WARNING: no scheduled action was between registration and its due time \ + when the executor died, so this run says nothing about lease recovery" + .to_string(); + } + format!( + "S10 killed the executor with {} scheduled actions registered and not yet due, {} \ + of them on targets it owned", + self.total, self.on_killed_executor + ) + } +} + +/// Counts the actions in the claim window at the moment of injection. +/// +/// Registered before the kill, due after it, and not definitively refused. A +/// refused registration is not work the platform owes anything, so counting it +/// here would overstate what the kill was aimed at. +pub fn pending_at_injection( + records: &[OperationRecord], + injected_at: DateTime, + lead: Duration, + on_killed_executor: &BTreeSet, +) -> PendingAtInjection { + let lead = TimeDelta::from_std(lead).unwrap_or(TimeDelta::zero()); + let mut pending = PendingAtInjection { + total: 0, + on_killed_executor: 0, + }; + + for record in records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.outcome != Outcome::Rejected) + .filter(|r| r.submitted_at <= injected_at && r.submitted_at + lead >= injected_at) + { + pending.total += 1; + if on_killed_executor.contains(&record.agent) { + pending.on_killed_executor += 1; + } + } + pending +} + +/// Per-target read-back from `polls`, the view every other scenario reports. +fn readback_from_polls(records: &[OperationRecord], logs: &[TargetFireLog]) -> Vec { + logs.iter() + .filter_map(|log| { + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.agent == log.agent); + let observed = match (log.polls, &log.error) { + (Some(polls), _) => Ok(polls), + (None, Some(error)) => Err(error.clone()), + (None, None) => Err(format!("target {} reported no poll count", log.agent)), + }; + readback_for(Stream::Scheduled, &log.agent, scoped, observed) + }) + .collect() +} + +/// Reads the fire count of a few targets, to prove actions are firing at all. +async fn sample_fire_count(ctx: &WorkloadContext, targets: &[String]) -> u64 { + let mut total = 0u64; + for target in targets.iter().take(FIRE_PROOF_SAMPLE) { + match workload::read_polls(ctx, target).await { + Ok(polls) => total += polls, + Err(e) => warn!("S10: could not sample fires on {target}: {e}"), + } + } + total +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::AttemptRecord; + use crate::chaos::summary::NoteLevel; + use test_r::test; + + fn at(offset_secs: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + offset_secs, 0).unwrap() + } + + fn registration(agent: &str, submitted_at: DateTime, outcome: Outcome) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Scheduled, + phase: Phase::Baseline, + agent: agent.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: format!("{agent}-key"), + submitted_at, + completed_at: Some(submitted_at), + attempts: 1, + outcome, + duration_ms: 10, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 10, + returned_value: None, + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + const LEAD: Duration = Duration::from_secs(10); + + /// The population the scenario is about: registered before the kill, due + /// after it. + #[test] + fn only_actions_still_inside_their_lead_count_as_pending() { + let killed = BTreeSet::from(["target-0".to_string()]); + let records = vec![ + // Due at 105, after the kill at 100. + registration("target-0", at(95), Outcome::Confirmed), + // Due at 99, so it had already fired. + registration("target-0", at(89), Outcome::Confirmed), + // Registered after the kill. + registration("target-0", at(101), Outcome::Confirmed), + ]; + let pending = pending_at_injection(&records, at(100), LEAD, &killed); + assert_eq!(pending.total, 1); + assert_eq!(pending.on_killed_executor, 1); + } + + /// The split is what makes the percentile readable, so it has to be counted + /// here too rather than inferred later. + #[test] + fn pending_actions_are_split_by_the_executor_that_owned_them() { + let killed = BTreeSet::from(["target-0".to_string()]); + let records = vec![ + registration("target-0", at(95), Outcome::Confirmed), + registration("target-1", at(95), Outcome::Confirmed), + ]; + let pending = pending_at_injection(&records, at(100), LEAD, &killed); + assert_eq!(pending.total, 2); + assert_eq!(pending.on_killed_executor, 1); + } + + /// A refusal is not work the platform owes anything, so counting it would + /// overstate what the kill was aimed at. + #[test] + fn a_refused_registration_is_not_pending_work() { + let records = vec![registration("target-0", at(95), Outcome::Rejected)]; + let pending = pending_at_injection(&records, at(100), LEAD, &BTreeSet::new()); + assert_eq!(pending.total, 0); + } + + /// The loudest thing this scenario can say: the kill missed the mechanism + /// entirely, so nothing about lease recovery can be read from the run. + #[test] + fn a_kill_that_caught_no_pending_action_says_so_rather_than_reporting_a_clean_run() { + let pending = PendingAtInjection { + total: 0, + on_killed_executor: 0, + }; + assert!(pending.describe().starts_with("WARNING")); + assert!(pending.needs_attention()); + assert_eq!(pending.note().level, NoteLevel::Attention); + } + + /// The same sentence on a run that landed properly is context. It is the + /// first thing a reader wants and it is true of every healthy run, so + /// putting it in `attention` would fire CI's annotation every time. + #[test] + fn a_kill_that_landed_reports_its_count_as_context() { + let pending = PendingAtInjection { + total: 353, + on_killed_executor: 226, + }; + assert!(!pending.needs_attention()); + assert_eq!(pending.note().level, NoteLevel::Context); + assert!(pending.note().message.contains("353")); + } + + /// The last registration falls due one lead after the workload stops, and a + /// recovery costs up to a lease on top. Reading before that would report + /// late actions as lost. + #[test] + fn the_settle_covers_a_full_lead_plus_a_full_lease_recovery() { + let config = ScheduledConfig { + targets: 100, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 45, + }; + assert_eq!( + settle_before_readback(&config), + Duration::from_secs(10 + 45) + SETTLE_MARGIN + ); + } +} diff --git a/integration-tests/src/chaos/scenarios/s11.rs b/integration-tests/src/chaos/scenarios/s11.rs new file mode 100644 index 0000000000..f8ccbd8d1e --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s11.rs @@ -0,0 +1,661 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S11 — executor crash during promise completion (GOL-377). +//! +//! S8 kills an executor with invocations running on it. S10 kills one holding +//! work it promised to do later. S11 kills one holding agents that are *asleep*: +//! each waiter is suspended inside an invocation, parked on a promise, occupying +//! no thread and running no code, and something outside is about to resolve that +//! promise and expect the agent to carry on. +//! +//! ## Why this is its own scenario +//! +//! A suspended waiter is the one durable-execution state with no in-memory +//! representation to lose and no queue entry to drain. Nothing about it looks +//! like work in flight. Resuming it depends on a chain that a pod kill can break +//! in the middle: the completion is written to storage under the promise's key, +//! then the worker that owns the promise is activated so it notices. Those are +//! two steps, and only the first of them is durable. +//! +//! So the failure this scenario exists to catch is specific and quiet. A +//! completion is accepted — the caller gets a success — the write lands, and the +//! activation goes to an executor that is already dying. The promise is resolved +//! forever after, and the agent waiting on it is never told. Nothing errors, +//! nothing retries, and no count anywhere goes down. +//! +//! ## What the run measures the kill against +//! +//! Each of `waiters` agents holds exactly one promise at a time, so the number of +//! agents standing suspended when the pod dies is a known constant rather than a +//! sample. The driver records how many of them were actually parked across the +//! injection, and on which executor, measured from the history rather than +//! assumed from the cadence — a platform that had slowed down would have armed +//! fewer than the arithmetic says. A kill that caught nothing parked is reported +//! as a warning: a clean account of a mechanism that was never disturbed is not +//! evidence. +//! +//! Like S8 and S10 the kill is aimed, and like S10 every waiter keeps running. +//! The ones on other executors are the control group, which is what stops a +//! recovery that took its whole budget from hiding behind the half of the +//! population that was never touched. +//! +//! ## Two independent answers, and why both are kept +//! +//! Every round is observed twice. The driver holds the `wait` invocation open, so +//! it sees the wakeup as a returning call; the waiter writes the wakeup into its +//! own durable log, which the run reads afterwards. +//! +//! The two disagree exactly when it matters. Killing the executor takes the +//! driver's connection with it, so for the rounds this scenario is about, the +//! client's view is a broken pipe and nothing more. The agent's log is what +//! answers, and [`crate::chaos::wakeups`] is built around it. +//! +//! ## What fails the run +//! +//! Only the three token-level violations in [`WakeupReport`]. Wakeup delay is +//! reported against the configured budget as SLO evidence, not asserted: how much +//! a shard reassignment plus a worker recovery may cost is a judgement, and the +//! number is in the result either way. +//! +//! ## The waiter that answers nothing +//! +//! An unreadable agent normally means the run cannot say. For a suspended waiter +//! it can mean the opposite, because the read queues behind the invocation the +//! waiter is parked in. A waiter that stopped producing rounds during the run and +//! then answered no read is a worker still parked on a promise resolved minutes +//! ago — the defect itself, observed from two directions. The report separates +//! that from an agent that merely timed out. See [`crate::chaos::wakeups`]. + +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, signal_termination, + snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ChaosSummary, Note, TerminationReason}; +use crate::chaos::waiters; +use crate::chaos::wakeups::WakeupReport; +use crate::chaos::workload::{PhaseMarker, WorkloadContext}; +use crate::chaos::{PromiseConfig, ScenarioCode, ScenarioConfig}; +use chrono::{DateTime, Utc}; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::collections::BTreeSet; +use std::time::Duration; +use tracing::{info, warn}; + +/// Extra quiet after the workload stops, before the wakeup logs are read. +/// +/// The rest of the settle is derived from the configuration: the last round's +/// completion goes out one dwell after the workload stops accepting new rounds, +/// and if its waiter's executor died holding it, the resume costs up to one +/// wakeup budget on top. Reading before that elapsed would report wakeups as +/// lost that were merely late, which is the one mistake this scenario cannot +/// afford. +const SETTLE_MARGIN: Duration = Duration::from_secs(30); + +/// How many waiters to sample after the baseline to prove wakeups happen at all. +/// +/// A handful, because this is a smoke test rather than a measurement: if the +/// completion path is broken, every waiter is equally broken, and the point is to +/// fail before spending the fault window on a run that would report a clean +/// account of nothing. +const WAKE_PROOF_SAMPLE: usize = 5; + +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let promise_config = config.require_promise()?; + let history = OperationHistory::new(ScenarioCode::S11.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S11); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + // The promise component, not the counters one: S11's agents live there, + // and a reader narrowing traces by component needs the one that was + // actually driven. + component_ids: vec![manifest.promise_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let waiter_names: Vec = (0..promise_config.waiters) + .map(|index| ctx.waiter_name(index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $wakeups:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + Vec::new(), + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if let Some(report) = $wakeups { + summary = summary.with_promise_wakeups(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: selection.clone(), + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + info!( + "S11: warming {} waiters before the baseline", + waiter_names.len() + ); + let warmed = waiters::warm(&ctx, &waiter_names).await; + info!("S11: warmed {warmed} waiters, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Prove one whole round works ───────────────────────────────────────── + // Before aiming, before the baseline, before anything that costs the window. + // Arming and completing can both succeed while the parking in between is + // refused, and that combination looks entirely healthy in the operation + // totals — so the totals are not what gets asked. + if let Err(e) = waiters::smoke_test(&ctx, promise_config.dwell()).await { + warn!("S11: a single promise round does not work against this cluster: {e}"); + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: format!("promise round smoke test failed: {e}"), + }, + &records, + None + ); + } + info!("S11: smoke round armed, parked, completed and woke"); + + // ── Aim the fault ─────────────────────────────────────────────────────── + // Before the baseline, because a run that cannot be aimed should not spend a + // maintenance window proving it. + let chosen = match split::select(split::waiter_subject(&ctx), deps, &waiter_names).await { + Ok(chosen) => chosen, + Err(e) => { + warn!("S11: could not aim the fault at an executor: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + }; + selection = Some(chosen.clone()); + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S11: baseline phase, {} waiters parking for {:?} a round, for {:?}", + waiter_names.len(), + promise_config.dwell(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = waiters::start(ctx.clone(), &waiter_names, promise_config); + tokio::time::sleep(config.phases.baseline()).await; + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S11: baseline completed no promise round, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no promise round succeeded during the baseline phase".to_string(), + }, + &records, + None + ); + } + + // Completing is not waking. A platform that accepted every completion and + // resumed nobody would otherwise reach read-back and report a flawless + // account of a mechanism that never ran. + let sampled = sample_wakes(&ctx, &waiter_names).await; + if sampled == 0 { + warn!("S11: {baseline_operations} operations accepted and no waiter has woken"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::StreamNeverSucceeded { + stream: Stream::PromiseWait.to_string(), + }, + &records, + None + ); + } + info!( + "S11: baseline complete ({baseline_operations} operations, {sampled} wakeups across a \ + sample of {} waiters)", + WAKE_PROOF_SAMPLE.min(waiter_names.len()) + ); + + // ── Verify ownership, then signal ─────────────────────────────────────── + if let Err(e) = split::verify_ownership(split::waiter_subject(&ctx), deps, &chosen).await { + warn!("S11: waiter ownership no longer holds, refusing to inject: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + + info!( + "S11: signalling readiness with fault target {} ({} of {} waiters on it)", + chosen.pod_address, + chosen.on_pod.len(), + waiter_names.len() + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S11.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: chosen.pod_address.clone(), + pod_ip: chosen.pod_ip.clone(), + owned_agents: chosen.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S11: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, None); + } + }; + info!( + "S11: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let on_pod: BTreeSet = chosen.on_pod.iter().cloned().collect(); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S11: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, None); + } + }; + info!( + "S11: fault cleared at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "S11: recovery phase, running rounds for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + let stood_down = handle.stalled(); + let rounds = handle.rounds(); + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Account ───────────────────────────────────────────────────────────── + let settle = settle_before_readback(promise_config); + info!("S11: letting the last completions land, {settle:?} before read-back"); + tokio::time::sleep(settle).await; + + let records = history.snapshot(); + + // Only now, with every `wait` record landed, can the parked population be + // counted — see [`parked_at_injection`]. + if let Some(injected_at) = fault_injected_at { + let parked = parked_at_injection(&records, injected_at, &on_pod); + attention_extra.push(parked.note()); + info!("S11: {}", parked.describe()); + } + + let logs = waiters::read_logs(&ctx, &waiter_names).await; + // Archived alongside the operations, not just reduced into the report: the + // reduced numbers cannot be recomputed later, and a correction to how a + // delay is classified has to be applicable to a run that has already + // happened. + history.record_wakeup_logs(logs.clone()); + + let report = WakeupReport::build( + &records, + &logs, + &chosen, + fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }), + promise_config.dwell(), + promise_config.wakeup_budget(), + stood_down, + ); + info!( + "S11: promise-wakeup account — {rounds} rounds started, {} completions accepted, {} \ + woke once, {} never woke, {} inconclusive, {} unverifiable, {} findings", + report.completions_confirmed, + report.woke_once, + report + .findings + .iter() + .filter(|f| f.violation == crate::chaos::wakeups::WakeupViolation::NeverWoke) + .count(), + report.inconclusive, + report.unverifiable, + report.findings.len() + ); + if let Some(p99) = report.fault_window_p99_ms() { + info!( + "S11: wakeup delay p99 during the fault, on the killed executor's waiters: {p99}ms \ + against a {}ms budget", + report.wakeup_budget_ms + ); + } + + let reason = if report.has_violations() { + TerminationReason::PromiseWakeupViolated { + findings: report.violations(), + first: report + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + } + } else if report.woke_once == 0 { + TerminationReason::StreamNeverSucceeded { + stream: Stream::PromiseWait.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, Some(report)); +} + +/// How long to wait after the workload stops before reading the wakeup logs. +fn settle_before_readback(config: &PromiseConfig) -> Duration { + config.dwell() + config.wakeup_budget() + SETTLE_MARGIN +} + +/// Waiters that were suspended on a promise when the executor died. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParkedAtInjection { + pub total: u64, + pub on_killed_executor: u64, +} + +impl ParkedAtInjection { + /// Whether the kill landed anywhere near the mechanism under test. + /// + /// A run that caught nothing parked proves nothing however clean the rest of + /// its numbers look, which is the one thing here a human has to act on. So + /// does a run that caught waiters but none of them *on the pod it killed*: + /// the affected group would be empty and every number in the report would + /// describe an undisturbed cluster. + pub fn needs_attention(&self) -> bool { + self.total == 0 || self.on_killed_executor == 0 + } + + /// The same line as [`Self::describe`], classified. + pub fn note(&self) -> Note { + Note::leveled(self.needs_attention(), self.describe()) + } + + pub fn describe(&self) -> String { + if self.total == 0 { + return "WARNING: no waiter was suspended on a promise when the executor died, so \ + this run says nothing about promise-completion recovery" + .to_string(); + } + if self.on_killed_executor == 0 { + return format!( + "WARNING: {} waiters were suspended when the executor died but none of them \ + were on it, so the affected group is empty and this run says nothing about \ + promise-completion recovery", + self.total + ); + } + format!( + "S11 killed the executor with {} waiters suspended on promises, {} of them on \ + waiters it owned", + self.total, self.on_killed_executor + ) + } +} + +/// Counts the waiters parked across the moment of injection. +/// +/// A `wait` invocation that had been submitted before the kill and had not +/// returned by then. Not derived from the round arithmetic, because a platform +/// that had slowed down would have armed fewer rounds than the cadence says, and +/// the whole point of this number is to say whether the kill landed in anything. +/// +/// **This must be computed from the finished history, not from a snapshot taken +/// at injection time.** [`crate::chaos::workload::run_operation`] appends a +/// record only once the operation has completed, so a snapshot taken at the +/// moment of the kill contains none of the invocations that were open across it +/// — which is precisely the population being counted. The first S11 run reported +/// 5 waiters parked when 200 were, and read as a run that had tested nothing. +pub fn parked_at_injection( + records: &[OperationRecord], + injected_at: DateTime, + on_killed_executor: &BTreeSet, +) -> ParkedAtInjection { + let mut parked = ParkedAtInjection { + total: 0, + on_killed_executor: 0, + }; + + for record in records + .iter() + .filter(|r| r.stream == Stream::PromiseWait && r.method == "wait") + .filter(|r| r.outcome != Outcome::Rejected) + .filter(|r| r.submitted_at <= injected_at) + .filter(|r| r.completed_at.is_none_or(|done| done > injected_at)) + { + parked.total += 1; + if on_killed_executor.contains(&record.agent) { + parked.on_killed_executor += 1; + } + } + + parked +} + +/// Total wakeups across a small sample of waiters. +async fn sample_wakes(ctx: &WorkloadContext, waiters_list: &[String]) -> u64 { + let sample: Vec = waiters_list + .iter() + .take(WAKE_PROOF_SAMPLE) + .cloned() + .collect(); + waiters::read_logs(ctx, &sample) + .await + .iter() + .filter_map(|log| log.wakes) + .sum() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{Outcome, Phase, Stream}; + use test_r::test; + + fn at(secs: i64) -> DateTime { + DateTime::from_timestamp(secs, 0).unwrap() + } + + fn wait_record(agent: &str, submitted: i64, completed: Option) -> OperationRecord { + OperationRecord { + op_id: 1, + stream: Stream::PromiseWait, + phase: Phase::Baseline, + agent: agent.to_string(), + method: "wait".to_string(), + idempotency_key: format!("{agent}-wait"), + submitted_at: at(submitted), + completed_at: completed.map(at), + attempts: 1, + outcome: Outcome::Confirmed, + duration_ms: 0, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + } + } + + #[test] + fn a_waiter_still_parked_at_the_kill_is_counted() { + let records = vec![wait_record("w-1", 90, Some(150))]; + let on_pod: BTreeSet = ["w-1".to_string()].into_iter().collect(); + assert_eq!( + parked_at_injection(&records, at(100), &on_pod), + ParkedAtInjection { + total: 1, + on_killed_executor: 1, + } + ); + } + + /// A round that had already woken before the kill was not disturbed by it, + /// and counting it would overstate what the fault landed in. + #[test] + fn a_waiter_that_woke_before_the_kill_is_not_counted() { + let records = vec![wait_record("w-1", 80, Some(90))]; + let on_pod = BTreeSet::new(); + assert_eq!(parked_at_injection(&records, at(100), &on_pod).total, 0); + } + + /// A `wait` that never returned at all is the most interesting case there + /// is, so it must not fall out of the count for want of a completion time. + #[test] + fn a_wait_that_never_returned_is_still_counted_as_parked() { + let records = vec![wait_record("w-1", 90, None)]; + let on_pod = BTreeSet::new(); + assert_eq!(parked_at_injection(&records, at(100), &on_pod).total, 1); + } + + /// A kill that caught waiters but none of its *own* is exactly as + /// uninformative as one that caught none at all — the affected group is + /// empty either way — and the first completed S11 run reported this shape as + /// ordinary context because the rule only looked at the total. + #[test] + fn a_kill_that_caught_none_of_its_own_waiters_needs_attention() { + let parked = ParkedAtInjection { + total: 200, + on_killed_executor: 0, + }; + assert!(parked.needs_attention()); + assert!(parked.describe().contains("none of them")); + } + + #[test] + fn a_kill_that_caught_nothing_parked_needs_attention() { + let parked = ParkedAtInjection { + total: 0, + on_killed_executor: 0, + }; + assert!(parked.needs_attention()); + assert!(parked.describe().contains("says nothing")); + } +} diff --git a/integration-tests/src/chaos/scenarios/s12.rs b/integration-tests/src/chaos/scenarios/s12.rs index d69779c6f4..537aa3375b 100644 --- a/integration-tests/src/chaos/scenarios/s12.rs +++ b/integration-tests/src/chaos/scenarios/s12.rs @@ -132,6 +132,11 @@ pub async fn run( ), termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s13.rs b/integration-tests/src/chaos/scenarios/s13.rs index 65d65c40d6..f76d4b59a3 100644 --- a/integration-tests/src/chaos/scenarios/s13.rs +++ b/integration-tests/src/chaos/scenarios/s13.rs @@ -69,7 +69,8 @@ use crate::chaos::scenarios::{ }; use crate::chaos::signal::{BaselineReady, FaultSignals, RestartEvent}; use crate::chaos::summary::{ - AgentReadback, ChaosSummary, ExactlyOnceReport, TerminationReason, stream_that_never_succeeded, + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, + stream_that_never_succeeded, }; use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; use crate::chaos::{ScenarioCode, ScenarioConfig}; @@ -138,7 +139,7 @@ pub async fn run( let mut fault_recovered_at = None; let mut fault_id = None; let mut fault_target_observed = None; - let mut attention_extra: Vec = Vec::new(); + let mut attention_extra: Vec = Vec::new(); // Sample the assignment continuously for the whole run. Five rebalances in // five minutes cannot be read from phase boundaries. @@ -185,7 +186,7 @@ pub async fn run( fault_injected_at, ); summary.ownership = samples; - summary.attention.extend(attention_extra.clone()); + summary.absorb(attention_extra.clone()); if let Some(report) = $exactly_once { summary = summary.with_exactly_once(report); } @@ -202,6 +203,11 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; @@ -295,7 +301,10 @@ pub async fn run( } let restarts = signals.read_restart_events(); - attention_extra.push(describe_restarts(&restarts)); + attention_extra.push(Note::leveled( + restarts.is_empty(), + describe_restarts(&restarts), + )); for event in &restarts { info!( "S13: restart {} at {} ({})", diff --git a/integration-tests/src/chaos/scenarios/s3.rs b/integration-tests/src/chaos/scenarios/s3.rs new file mode 100644 index 0000000000..63351408a6 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s3.rs @@ -0,0 +1,455 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S3 — executor / worker-service network partition (GOL-370). +//! +//! S1 cuts the shard-manager off from an executor, and the cluster reacts: the +//! shard-manager stops hearing health checks and moves the shards. S3 cuts the +//! *other* link, and the cluster does not react at all — which is the point. +//! +//! The executor keeps talking to the shard-manager, so it keeps its shards and +//! stays in the routing table. worker-service therefore keeps being told, quite +//! correctly, that this executor owns those agents, and keeps trying to reach a +//! pod it cannot reach. There is no route around it, because as far as the +//! platform is concerned nothing is wrong. Every other scenario in the suite +//! ends with the platform recovering; this one asks what it does when there is +//! nothing to recover *from*, only something to wait out. +//! +//! ### What worker-service actually does +//! +//! Read out of `golem-worker-service/src/service/worker/routing_logic.rs`. A +//! call to an unreachable pod fails the 10s connect timeout, which is retriable, +//! so `call_worker_executor` invalidates the routing table and tries again — +//! forever. `get_delay` stops extending the backoff after five attempts and the +//! loop then settles at the 3s ceiling rather than giving up. The freshly +//! fetched table names the same unreachable pod every time, because the +//! shard-manager still believes in it. +//! +//! Two consequences, and S3 measures both: +//! +//! * **For the agents on that executor**, an invocation does not fail fast. It +//! hangs, until the caller's own attempt timeout ends it. That is the +//! "acceptance degradation with pending and timeout behaviour" the ticket +//! asks to see, and it is visible in the history as attempts that timed out +//! rather than as refusals. +//! * **For everyone else**, the routing table is one process-wide cache entry +//! per worker-service replica. Every stalled caller invalidating it costs +//! *every* caller a shard-manager round trip. `invalidation_min_delay` bounds +//! that to twice a second, so the cost should be small — but "should be" is +//! what the control group is there to check. +//! +//! ### The choreography +//! +//! 1. **Warm up** every agent, so the partition lands on a live population. +//! 2. **Select** the executor owning the largest share of them, exactly as S10 +//! and S11 pick their target, and name it for the workflow. +//! 3. **Baseline** — one emitter per agent, one operation each at a time. +//! 4. **Fault** — keep driving. Sample the routing table part-way in: S3's +//! premise is that the assignment does *not* move, and a run where it did is +//! a different experiment that has to be read differently. +//! 5. **Heal**, then keep driving so the isolated agents visibly come back. +//! 6. **Read back and probe** — the same completion and exactly-once oracles +//! every other scenario ends with. + +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::probe; +use crate::chaos::reachability::ReachabilityReport; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ReadKind, ScenarioOutcome, WARMUP_SETTLE, build_result, exactly_once_termination, + read_back_agents, read_counters, sample_ownership, signal_termination, snapshot_routing, + wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::steady; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; +use crate::chaos::workload::{PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before reading durable state. +/// Same reasoning as every other scenario: an increment still in flight has to +/// land, and reading early reports a mismatch that says nothing. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(30); + +/// How far into the fault window the assignment sample is taken, as a fraction +/// of it, and the ceiling on that. +/// +/// Unlike S1 this sample is checking that nothing moved, so it is taken late +/// rather than early: a table that still looks untouched two minutes in is a +/// much stronger statement than one that looks untouched immediately. +const DURING_FAULT_SAMPLE_FRACTION: f64 = 0.75; +const DURING_FAULT_SAMPLE_CAP: Duration = Duration::from_secs(150); + +/// Runs S3 end to end. +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let isolation = config.require_isolation()?; + let history = OperationHistory::new(ScenarioCode::S3.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S3); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let agents = steady::agent_names(&ctx, isolation.agents); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $exactly_once:expr, $reachability:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $exactly_once { + summary = summary.with_exactly_once(report); + } + if let Some(report) = $reachability { + summary = summary.with_reachability(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: selection.clone(), + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Construct every agent before measuring, for the reason S1 spells out: a + // cold start looks exactly like a stall from outside, and this scenario's + // entire signal is a comparison of throughput before and during the fault. + // A baseline that was still cold-starting would understate itself and make + // the fault look milder than it was. + // + // Reads, not increments. An increment here would be invisible to the + // operation history and would leave every read-back off by one. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S3: warming up {} counter agents", agents.len()); + let warm: Vec<(Stream, String, ReadKind)> = agents + .iter() + .map(|agent| (Stream::Durable, agent.clone(), ReadKind::Counter)) + .collect(); + let _ = read_back_agents(&ctx, &[], warm).await; + info!( + "S3: warmed {} agents, settling {WARMUP_SETTLE:?}", + agents.len() + ); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim ───────────────────────────────────────────────────────────────── + // + // Chaos Mesh's `mode: one` would pick an executor at random, and a + // partition that cut off the executor owning six agents out of two hundred + // would still produce a confident-looking report. The driver names the pod + // instead; the workflow turns the IP into a pod name. + let subject = split::counter_subject(&ctx); + let split = match split::select(subject, deps, &agents).await { + Ok(split) => split, + Err(e) => { + warn!("S3: cannot aim the partition: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None + ); + } + }; + selection = Some(split.clone()); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S3: baseline phase, running {} emitters for {:?}", + agents.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = steady::start(ctx.clone(), isolation.agents, isolation.interval()); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S3: baseline produced no confirmed operations, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None, + None + ); + } + + // A rebalance between selection and injection would leave the run naming + // the control group as the affected one and vice versa — a report that is + // not merely wrong but confidently inverted. + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("S3: ownership drifted between selection and injection: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None + ); + } + + info!( + "S3: baseline complete ({baseline_operations} confirmed ops), naming {} and signalling \ + readiness", + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S3.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S3: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None, None); + } + }; + info!( + "S3: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + // Evidence for the premise, not a verdict. The partitioned link is not the + // one the shard-manager uses, so the assignment is expected to be identical + // to the baseline. If it moved, the fault was wider than intended and every + // reading below has a second explanation. + let observe_after = config + .phases + .fault() + .mul_f64(DURING_FAULT_SAMPLE_FRACTION) + .min(DURING_FAULT_SAMPLE_CAP); + info!("S3: sampling assignment {observe_after:?} into the fault window"); + tokio::time::sleep(observe_after).await; + ownership.push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S3: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None, None); + } + }; + info!( + "S3: partition healed at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "S3: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S3: letting the platform settle for {SETTLE_BEFORE_READBACK:?} before read-back"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + let records = history.snapshot(); + + let readback = read_back(&ctx, &records, &agents).await; + let before_probe = read_counters(&ctx, &records).await; + let probes = probe::probe_keys(&ctx, &records, Stream::Durable).await; + let after_probe = read_counters(&ctx, &records).await; + + let exactly_once = ExactlyOnceReport::build( + &records, + &probes, + Stream::Durable, + &before_probe, + &after_probe, + ); + info!( + "S3: exactly-once account — {} keys checked, {} with a final result, {} recovered by the \ + probe, {} findings", + exactly_once.keys_checked, + exactly_once.keys_with_final_result, + exactly_once.keys_recovered_by_probe, + exactly_once.findings.len() + ); + + let reachability = ReachabilityReport::build( + &records, + &split, + fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }), + isolation.isolated_ceiling_percent, + isolation.control_floor_percent, + isolation.recovery_budget(), + ); + info!( + "S3: reachability account — {} findings, {} isolated agents never recovered", + reachability.findings.len(), + reachability.agents_never_recovered.len() + ); + + // The assertion is the same one S1 makes, and for the same reason: a key + // that executed twice is the only harm visible from outside the cluster. + // Everything the reachability report says is reported for a human to judge, + // because "worker-service waited rather than failing fast" is a design + // question and not a defect the driver is entitled to rule on. + let reason = exactly_once_termination(&exactly_once).unwrap_or_else(|| { + if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Durable.to_string(), + } + } else { + TerminationReason::Completed + } + }); + + finish!( + reason, + &records, + readback, + Some(exactly_once), + Some(reachability) + ); +} + +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + agents: &[String], +) -> Vec { + let targets = agents + .iter() + .map(|agent| (Stream::Durable, agent.clone(), ReadKind::Counter)) + .collect(); + read_back_agents(ctx, records, targets).await +} diff --git a/integration-tests/src/chaos/scenarios/s5.rs b/integration-tests/src/chaos/scenarios/s5.rs index 08568cdca1..d80e382490 100644 --- a/integration-tests/src/chaos/scenarios/s5.rs +++ b/integration-tests/src/chaos/scenarios/s5.rs @@ -76,7 +76,8 @@ use crate::chaos::scenarios::{ }; use crate::chaos::signal::{BaselineReady, FaultSignals}; use crate::chaos::summary::{ - AgentReadback, ChaosSummary, ReadbackVerdict, TerminationReason, stream_that_never_succeeded, + AgentReadback, ChaosSummary, Note, ReadbackVerdict, TerminationReason, + stream_that_never_succeeded, }; use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; use crate::chaos::{ScenarioCode, ScenarioConfig}; @@ -150,7 +151,7 @@ pub async fn run( let mut fault_recovered_at = None; let mut fault_id = None; let mut fault_target_observed = None; - let mut attention_extra: Vec = Vec::new(); + let mut attention_extra: Vec = Vec::new(); macro_rules! finish { ($reason:expr, $records:expr, $readback:expr) => {{ @@ -160,7 +161,7 @@ pub async fn run( routing_snapshots.clone(), fault_injected_at, ); - summary.attention.extend(attention_extra.clone()); + summary.absorb(attention_extra.clone()); let result = build_result( config, ScenarioOutcome { @@ -174,6 +175,11 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; @@ -254,10 +260,13 @@ pub async fn run( ); let requested = request_updates(&ctx, workload_config, target_revision).await; - attention_extra.push(format!( - "update to revision {target_revision} requested for {requested} of {} durable agents \ - at {update_started_at}", - workload_config.durable_agents + attention_extra.push(Note::leveled( + requested < workload_config.durable_agents as usize, + format!( + "update to revision {target_revision} requested for {requested} of {} durable \ + agents at {update_started_at}", + workload_config.durable_agents + ), )); // ── Signal: ready for the fault ───────────────────────────────────────── @@ -290,9 +299,9 @@ pub async fn run( "S5: fault {} ({} on {}) reported active at {}, {into_update}ms into the update", injected.fault_id, injected.kind, injected.target, injected.injected_at ); - attention_extra.push(format!( + attention_extra.push(Note::context(format!( "the executor kill landed {into_update}ms into the update" - )); + ))); fault_injected_at = Some(injected.injected_at); fault_id = Some(injected.fault_id.clone()); fault_target_observed = Some(injected.target.clone()); @@ -342,12 +351,16 @@ pub async fn run( .map(|(agent, _)| agent) .collect(); let unreadable = versions.values().filter(|v| v.is_none()).count(); - attention_extra.push(format!( - "after recovery {} of {} durable agents report component version {}; {} could not be read", - versions.len() - stale.len(), - versions.len(), - EXPECTED_VERSION_AFTER_UPDATE, - unreadable + attention_extra.push(Note::leveled( + !stale.is_empty() || unreadable > 0, + format!( + "after recovery {} of {} durable agents report component version {}; {} could not \ + be read", + versions.len() - stale.len(), + versions.len(), + EXPECTED_VERSION_AFTER_UPDATE, + unreadable + ), )); // ── Verdict ───────────────────────────────────────────────────────────── diff --git a/integration-tests/src/chaos/scenarios/s6.rs b/integration-tests/src/chaos/scenarios/s6.rs new file mode 100644 index 0000000000..7afbb8f14e --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s6.rs @@ -0,0 +1,475 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S6 — executor pod kill during agent deletion (GOL-372). +//! +//! S7 asks the platform to forget some of an agent's work. S6 asks it to forget +//! the agent, and then kills the executor while it is doing so. +//! +//! ### Why this one can assert +//! +//! Invoking a deleted agent id creates a **new** agent, and a new counter starts +//! from nothing. So a round — increments to a known `V`, then a delete — has +//! exactly two legal answers when the slot is next used: +//! +//! * `1`, a fresh agent, meaning the deletion took +//! * `V + 1`, meaning the old agent is still there +//! +//! Neither is a defect on its own; a delete whose response was lost leaves the +//! question genuinely open. What makes one a defect is the platform's own answer +//! beside it. Confirmed, and the agent is still worth `V`, is the resurrection +//! this scenario is named for. See [`crate::chaos::resurrection`]. +//! +//! ### What is actually being killed into +//! +//! Deleting is four steps in `delete_worker_internal`: interrupt the running +//! worker, `start_deleting`, remove it from the worker service, remove it from +//! the active set. Only the third is durable. +//! +//! The interesting part is that the happy path is **already defended**. +//! `Worker::start_deleting` stops the background status flush and the +//! checkpointer first, specifically so neither can — in the executor's own +//! comment — "resurrect the cached status" after the removal. So the question +//! S6 asks is not whether anyone thought about resurrection, but whether that +//! defence survives the pod dying between the mark and the removal, when +//! whoever picks up the shard next has to decide what a worker marked for +//! deletion but never removed means. +//! +//! ### A smoke round before anything else +//! +//! The whole account rests on "a deleted id comes back as a new agent". If that +//! is not true, every round in the run reports a resurrection and the report is +//! worthless. So one throwaway agent is built, deleted and re-invoked before the +//! baseline starts, and a run whose premise is wrong aborts in seconds instead +//! of spending the maintenance window discovering it. That lesson is S11's. +//! +//! ### The choreography +//! +//! Like S8, S10, S11, S3 and S7 the driver names the pod: it picks the executor +//! owning the largest share of its agent slots and keeps driving the rest as a +//! control group. As in S7 the last thing before read-back is one read per slot, +//! which answers the round no following increment ever probed. + +use crate::chaos::deletions::{self, DeleteRound}; +use crate::chaos::history::{OperationHistory, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::resurrection::ResurrectionReport; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, sample_ownership, + signal_termination, snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ChaosSummary, Note, TerminationReason}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before the final read. +/// +/// Shorter than the other scenarios' settle: nothing here is queued or +/// scheduled, and `stop` already waits for every operation in flight to record +/// itself. This only covers a slot still coming back from the last delete. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(20); + +/// How far into the fault window the assignment is sampled. +const DURING_FAULT_SAMPLE_FRACTION: f64 = 0.6; +const DURING_FAULT_SAMPLE_CAP: Duration = Duration::from_secs(120); + +/// Runs S6 end to end. +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let delete_config = config.require_delete()?; + let history = OperationHistory::new(ScenarioCode::S6.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S6); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let agents = deletions::agent_names(&ctx, delete_config.agents); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $resurrection:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + Vec::new(), + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $resurrection { + summary = summary.with_resurrection(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: selection.clone(), + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Constructing an agent is itself recorded in its oplog, so doing it inside + // a measured round would leave a slot holding an agent the round did not + // build. Reads here rather than increments, for the same reason every other + // scenario warms with reads: an increment would be invisible to the round + // arithmetic the whole oracle rests on. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S6: warming up {} delete agents", agents.len()); + let mut warmed = 0usize; + for agent in &agents { + if workload::read_counter(&ctx, agent).await.is_ok() { + warmed += 1; + } + } + info!( + "S6: warmed {warmed} of {} agents, settling {WARMUP_SETTLE:?}", + agents.len() + ); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Smoke round ───────────────────────────────────────────────────────── + // + // Everything below assumes a deleted id comes back as a new agent. If that + // is not true, every round reports a resurrection and the artifact is + // worthless — so one throwaway agent proves it before the baseline starts. + // The first S11 run is why this is here: a wrong premise otherwise costs + // the whole maintenance window before the numbers say so. + if let Err(e) = deletions::smoke_round(&ctx, delete_config).await { + warn!("S6: smoke round failed, aborting before the baseline: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + + // ── Aim ───────────────────────────────────────────────────────────────── + let subject = split::delete_subject(&ctx); + let split = match split::select(subject, deps, &agents).await { + Ok(split) => split, + Err(e) => { + warn!("S6: cannot aim the kill: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + }; + selection = Some(split.clone()); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S6: baseline phase, running {} delete emitters for {:?}", + agents.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = deletions::start(ctx.clone(), delete_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S6: baseline produced no confirmed operations, aborting before injection"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Some(build_resurrection(&rounds, &split, None, delete_config)) + ); + } + + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("S6: ownership drifted between selection and injection: {e:#}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Some(build_resurrection(&rounds, &split, None, delete_config)) + ); + } + + info!( + "S6: baseline complete ({baseline_operations} confirmed ops, {} rounds), naming {} and \ + signalling readiness", + handle.rounds().len(), + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S6.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S6: no fault-injected signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_resurrection(&rounds, &split, None, delete_config)) + ); + } + }; + info!( + "S6: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let observe_after = config + .phases + .fault() + .mul_f64(DURING_FAULT_SAMPLE_FRACTION) + .min(DURING_FAULT_SAMPLE_CAP); + tokio::time::sleep(observe_after).await; + ownership.push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S6: no fault-recovered signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_resurrection( + &rounds, + &split, + fault_window(fault_injected_at, None), + delete_config + )) + ); + } + }; + info!( + "S6: executor back at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "S6: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + let mut rounds = handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S6: settling {SETTLE_BEFORE_READBACK:?} before the final read"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + // The one read per agent, which answers the last round it ran. Every other + // round was probed by the increment that followed it; this is the only one + // that has nothing after it. + close_last_rounds(&ctx, &agents, &mut rounds).await; + + let records = history.snapshot(); + let resurrection = build_resurrection( + &rounds, + &split, + fault_window(fault_injected_at, fault_recovered_at), + delete_config, + ); + info!( + "S6: resurrection account — {} rounds, {} deleted exactly, {} findings", + resurrection.rounds_recorded, + resurrection.deleted_exactly, + resurrection.findings.len() + ); + + let reason = if resurrection.has_violations() { + let first = resurrection + .findings + .first() + .map(|f| format!("{} round {}: {}", f.agent, f.round, f.detail)) + .unwrap_or_default(); + TerminationReason::AgentResurrected { + findings: resurrection.findings.len() as u64 + resurrection.findings_omitted, + first, + } + } else if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Delete.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, Some(resurrection)); +} + +fn fault_window( + injected_at: Option>, + recovered_at: Option>, +) -> Option { + injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at, + }) +} + +fn build_resurrection( + rounds: &[DeleteRound], + split: &PodSplit, + fault: Option, + config: &crate::chaos::DeleteConfig, +) -> ResurrectionReport { + ResurrectionReport::build(rounds, split, fault, config.increments_per_round) +} + +/// Reads each agent once and uses the value to judge the last round it ran. +/// +/// A read is an invocation and would shift what "the last N invocations" means +/// for any delete after it — which is exactly why the workload never reads +/// mid-round. Here there is nothing after it, so it is safe, and it recovers a +/// round per agent that would otherwise be unjudgeable. +async fn close_last_rounds(ctx: &WorkloadContext, agents: &[String], rounds: &mut [DeleteRound]) { + let mut last_of: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new(); + for (index, round) in rounds.iter().enumerate() { + if round.observed_after.is_none() { + let slot = last_of.entry(round.agent.as_str()).or_insert(index); + if rounds[*slot].round < round.round { + *slot = index; + } + } + } + let pending: Vec<(String, usize)> = last_of + .into_iter() + .map(|(agent, index)| (agent.to_string(), index)) + .collect(); + + info!( + "S6: closing {} unprobed rounds with a final read", + pending.len() + ); + for (agent, index) in pending { + if !agents.iter().any(|a| a == &agent) { + continue; + } + match workload::read_counter(ctx, &agent).await { + Ok(value) => rounds[index].observed_after = Some(value), + Err(e) => warn!("S6: could not read {agent} to close its last round: {e}"), + } + } +} diff --git a/integration-tests/src/chaos/scenarios/s7.rs b/integration-tests/src/chaos/scenarios/s7.rs new file mode 100644 index 0000000000..db109fded3 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s7.rs @@ -0,0 +1,450 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S7 — executor pod kill during agent state revert (GOL-371). +//! +//! Every other scenario in this suite disturbs work that is trying to happen. +//! S7 disturbs work that is trying to be **undone**: each agent builds its +//! counter up with a run of increments and then asks the platform to take some +//! of them back, over and over, while an executor is killed underneath. +//! +//! ### Why this one can assert +//! +//! Read-backs elsewhere compare a counter against a range, and the width of the +//! range is the operations whose fate the driver could not determine. Here +//! there is no range. The last increment of a round *returns* the counter's +//! value, so the driver knows exactly what the agent was worth immediately +//! before the revert, and it asked for an exact number of invocations back. So +//! afterwards there are two legal values, `V` and `V - N`, and nothing between +//! them. See [`crate::chaos::truncation`] for what each other answer means. +//! +//! That is why S7 is one of the few scenarios whose read-back can fail the run +//! outright rather than being reported for a human to weigh. +//! +//! ### What is actually being killed into +//! +//! The truncation itself cannot tear: `RevertLastInvocations` commits a single +//! `OplogEntry::revert` marking a region deleted. The window worth aiming at is +//! the one around it — reverting takes `lock_stopped_worker`, so the worker is +//! stopped, the entry is committed, and only then is the worker status +//! reattached. An executor that dies between the commit and the reattach has +//! changed durable state and lost the thing that tells anyone about it. +//! +//! ### The choreography +//! +//! Like S8, S10, S11 and S3 the driver names the pod: it picks the executor +//! owning the largest share of its agents and keeps driving the rest as a +//! control group. Unlike them, the last thing it does before read-back is a +//! single read per agent, which answers the one round per agent that no +//! following increment ever probed. + +use crate::chaos::history::{OperationHistory, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::reverts::{self, RevertRound}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, sample_ownership, + signal_termination, snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ChaosSummary, Note, TerminationReason}; +use crate::chaos::truncation::TruncationReport; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before the final read. +/// +/// Shorter than the other scenarios' settle: nothing here is queued or +/// scheduled, and `stop` already waits for every operation in flight to record +/// itself. This only covers a worker still coming back from the last revert. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(20); + +/// How far into the fault window the assignment is sampled. +const DURING_FAULT_SAMPLE_FRACTION: f64 = 0.6; +const DURING_FAULT_SAMPLE_CAP: Duration = Duration::from_secs(120); + +/// Runs S7 end to end. +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let revert_config = config.require_revert()?; + let history = OperationHistory::new(ScenarioCode::S7.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S7); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let agents = reverts::agent_names(&ctx, revert_config.agents); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $truncation:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + Vec::new(), + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $truncation { + summary = summary.with_truncation(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: selection.clone(), + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Constructing an agent is itself recorded in its oplog, so doing it inside + // a measured round would put an entry between the increments and the revert + // that counts them. Reads here rather than increments, for the same reason + // every other scenario warms with reads: an increment would be invisible to + // the round arithmetic the whole oracle rests on. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S7: warming up {} revert agents", agents.len()); + let mut warmed = 0usize; + for agent in &agents { + if workload::read_counter(&ctx, agent).await.is_ok() { + warmed += 1; + } + } + info!( + "S7: warmed {warmed} of {} agents, settling {WARMUP_SETTLE:?}", + agents.len() + ); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim ───────────────────────────────────────────────────────────────── + let subject = split::revert_subject(&ctx); + let split = match split::select(subject, deps, &agents).await { + Ok(split) => split, + Err(e) => { + warn!("S7: cannot aim the kill: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + }; + selection = Some(split.clone()); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S7: baseline phase, running {} revert emitters for {:?}", + agents.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = reverts::start(ctx.clone(), revert_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S7: baseline produced no confirmed operations, aborting before injection"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Some(build_truncation(&rounds, &split, None, revert_config)) + ); + } + + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("S7: ownership drifted between selection and injection: {e:#}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Some(build_truncation(&rounds, &split, None, revert_config)) + ); + } + + info!( + "S7: baseline complete ({baseline_operations} confirmed ops, {} rounds), naming {} and \ + signalling readiness", + handle.rounds().len(), + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S7.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S7: no fault-injected signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_truncation(&rounds, &split, None, revert_config)) + ); + } + }; + info!( + "S7: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let observe_after = config + .phases + .fault() + .mul_f64(DURING_FAULT_SAMPLE_FRACTION) + .min(DURING_FAULT_SAMPLE_CAP); + tokio::time::sleep(observe_after).await; + ownership.push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S7: no fault-recovered signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_truncation( + &rounds, + &split, + fault_window(fault_injected_at, None), + revert_config + )) + ); + } + }; + info!( + "S7: executor back at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "S7: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + let mut rounds = handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S7: settling {SETTLE_BEFORE_READBACK:?} before the final read"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + // The one read per agent, which answers the last round it ran. Every other + // round was probed by the increment that followed it; this is the only one + // that has nothing after it. + close_last_rounds(&ctx, &agents, &mut rounds).await; + + let records = history.snapshot(); + let truncation = build_truncation( + &rounds, + &split, + fault_window(fault_injected_at, fault_recovered_at), + revert_config, + ); + info!( + "S7: truncation account — {} rounds, {} applied exactly, {} findings", + truncation.rounds_recorded, + truncation.applied_exactly, + truncation.findings.len() + ); + + let reason = if truncation.has_violations() { + let first = truncation + .findings + .first() + .map(|f| format!("{} round {}: {}", f.agent, f.round, f.detail)) + .unwrap_or_default(); + TerminationReason::RevertTruncationViolated { + findings: truncation.findings.len() as u64 + truncation.findings_omitted, + first, + } + } else if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Revert.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, Some(truncation)); +} + +fn fault_window( + injected_at: Option>, + recovered_at: Option>, +) -> Option { + injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at, + }) +} + +fn build_truncation( + rounds: &[RevertRound], + split: &PodSplit, + fault: Option, + config: &crate::chaos::RevertConfig, +) -> TruncationReport { + TruncationReport::build( + rounds, + split, + fault, + config.increments_per_round, + config.revert_invocations, + ) +} + +/// Reads each agent once and uses the value to judge the last round it ran. +/// +/// A read is an invocation and would shift what "the last N invocations" means +/// for any revert after it — which is exactly why the workload never reads +/// mid-round. Here there is nothing after it, so it is safe, and it recovers a +/// round per agent that would otherwise be unjudgeable. +async fn close_last_rounds(ctx: &WorkloadContext, agents: &[String], rounds: &mut [RevertRound]) { + let mut last_of: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new(); + for (index, round) in rounds.iter().enumerate() { + if round.observed_after.is_none() { + let slot = last_of.entry(round.agent.as_str()).or_insert(index); + if rounds[*slot].round < round.round { + *slot = index; + } + } + } + let pending: Vec<(String, usize)> = last_of + .into_iter() + .map(|(agent, index)| (agent.to_string(), index)) + .collect(); + + info!( + "S7: closing {} unprobed rounds with a final read", + pending.len() + ); + for (agent, index) in pending { + if !agents.iter().any(|a| a == &agent) { + continue; + } + match workload::read_counter(ctx, &agent).await { + Ok(value) => rounds[index].observed_after = Some(value), + Err(e) => warn!("S7: could not read {agent} to close its last round: {e}"), + } + } +} diff --git a/integration-tests/src/chaos/scenarios/s8.rs b/integration-tests/src/chaos/scenarios/s8.rs index b53fde75f1..5fcfc893fa 100644 --- a/integration-tests/src/chaos/scenarios/s8.rs +++ b/integration-tests/src/chaos/scenarios/s8.rs @@ -152,6 +152,11 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: selection.clone(), + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s9.rs b/integration-tests/src/chaos/scenarios/s9.rs new file mode 100644 index 0000000000..676fa66663 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s9.rs @@ -0,0 +1,722 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S9 — executor pod kill during a component rollback (GOL-369). +//! +//! S5 moves agents forward onto a new build and kills an executor while that is +//! happening. S9 moves them forward, waits for that to land, and then moves them +//! **back**, killing an executor during the return leg. +//! +//! The return leg is the one that matters operationally. A rollback is what you +//! reach for when the new build is already going wrong, so a rollback happening +//! under a dying executor is not a contrived situation — it is the situation you +//! would actually be in. +//! +//! ## What a rollback is here +//! +//! A redeploy. The original artifact is uploaded again as a **new** revision and +//! every agent is asked to move to it. That is what rollback means in practice, +//! and it is what makes the evidence unambiguous: `Counter::component_version` +//! is compiled into each build — `1` in `agent-counters`, `2` in +//! `agent-counters-v2`, nothing else different — so an agent that has genuinely +//! returned reports `1` from the code that is running, not from metadata about +//! what the platform believes. +//! +//! ## The constraint that shapes this scenario +//! +//! An automatic update **replays the agent's oplog against the new build** and +//! aborts if any recorded invocation produces a different result. It can only +//! cross a build boundary that no recorded invocation can tell apart. +//! +//! That has a sharp operational consequence, and it is arguably S9's most +//! useful finding: **a behaviour-changing build cannot be rolled back +//! automatically.** If the new build ever returned a different answer for an +//! invocation still in an agent's oplog — which is usually *why* you are +//! rolling back — the automatic update is refused. A rollback in that situation +//! needs a snapshot-based update instead. +//! +//! It also dictates how this scenario may look at its own agents. The first S9 +//! run verified the forward leg by invoking `component_version`, which exists +//! precisely to differ between builds, and thereby wrote an entry into all 200 +//! oplogs that the rollback's replay could never reproduce. Every rollback was +//! refused with `Unexpected oplog entry: expected component_version => 1, got +//! 2`. The forward leg is now read from metadata, which leaves no trace; the +//! running code is asked only at the very end, where nothing depends on it. +//! See `an_automatic_update_rolls_an_agent_back_to_an_earlier_build` in +//! `golem-worker-executor/tests/hot_update.rs` for the minimal reproduction. +//! +//! ## Why the forward leg is verified first +//! +//! If the agents never reached the new build, rolling them back returns them to +//! a build they never left, every check passes, and the run proves nothing. So +//! the forward leg is measured and the rollback is refused outright if too few +//! agents made it. Same instinct as S6's smoke round: a clean report from a +//! scenario that never happened is the worst artifact this suite can produce. +//! +//! ## What fails the run +//! +//! The same two things S5 asserts, reused rather than re-invented because they +//! are the same facts: +//! +//! - an agent whose durable state fell below what the driver was told succeeded, +//! or rose above what it could possibly have asked for; +//! - an agent still reporting the build it was rolled back *from*. +//! +//! An agent that cannot be read at all is reported rather than assumed either +//! way, and control-plane refusals are counted apart from workload retries — +//! see [`crate::chaos::rollback`] for why that separation is load-bearing. + +use crate::chaos::history::{OperationHistory, OperationRecord, Phase, Stream}; +use crate::chaos::prep::{COUNTERS_V2_WASM, COUNTERS_WASM, ChaosPrepManifest}; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::rollback::{ControlPlaneAttempts, RollbackReport, VersionCensus}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, readback_for, signal_termination, + snapshot_routing, wait_for_settled_routing, warm_up, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, Note, ReadbackVerdict, TerminationReason, + stream_that_never_succeeded, +}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before reading durable state. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(30); + +/// How many agents to update concurrently. +/// +/// Update requests are cheap to issue and the point is that many are in flight +/// when the executor dies, so this is wide rather than polite. +const UPDATE_CONCURRENCY: usize = 32; + +/// What the running code reports on each of the two builds. +/// +/// Compiled into the WASM rather than read from metadata, which is the whole +/// reason these numbers can be trusted: metadata says what the platform +/// believes, `component_version` says what is executing. +const VERSION_ON_THE_NEW_BUILD: u32 = 2; +const VERSION_AFTER_ROLLBACK: u32 = 1; + +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let workload_config = config.require_workload()?; + let rollback_config = config.require_rollback()?; + let history = OperationHistory::new(ScenarioCode::S9.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S9); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![ + manifest.counters_component_id.0.to_string(), + manifest.promise_component_id.0.to_string(), + ], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut attention_extra: Vec = Vec::new(); + let mut rollback_report: Option = None; + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if let Some(report) = rollback_report.clone() { + summary = summary.with_rollback(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Same reason as S1: an agent's first invocation costs far more than its + // later ones, and a population still cold-starting is a different thing to + // update than a population that is running. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S9: warming up agents before the baseline"); + let warmed = warm_up(&ctx, workload_config).await; + info!("S9: warmed {warmed} agents, settling {:?}", WARMUP_SETTLE); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S9: baseline phase, running mixed workload for {:?}", + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = workload::start(ctx.clone(), workload_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S9: baseline produced no confirmed operations, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new() + ); + } + + // ── Roll forward ──────────────────────────────────────────────────────── + // + // The workload keeps running throughout. This leg is not the experiment: it + // exists to put the agents somewhere they can be brought back *from*. + info!("S9: rolling forward — updating the counters component to {COUNTERS_V2_WASM}"); + let forward = match ctx + .user + .update_component(&manifest.counters_component_id, COUNTERS_V2_WASM) + .await + { + Ok(updated) => updated, + Err(e) => { + warn!("S9: roll-forward component update failed: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::Aborted { + detail: format!("component update to {COUNTERS_V2_WASM} failed: {e:#}"), + }, + &records, + Vec::new() + ); + } + }; + let forward_revision = forward.revision; + info!( + "S9: component now at revision {forward_revision}, moving {} durable agents onto it", + workload_config.durable_agents + ); + let _ = request_updates(&ctx, workload_config, forward_revision, 0, Duration::ZERO).await; + + // Let the forward leg land before measuring it. Without this the census + // below reads a population still in transit, and the gate would refuse a + // rollback that would have been perfectly good. + info!( + "S9: letting the roll-forward settle for {:?}", + rollback_config.settle() + ); + tokio::time::sleep(rollback_config.settle()).await; + + // Read from metadata, NOT by invoking `component_version`. + // + // This is the correction the first S9 run forced, and it is not a + // downgrade of the evidence — it is the only way to gather it without + // destroying what comes next. An automatic update replays the agent's + // oplog against the new build and aborts if any recorded invocation + // produces a different result. `component_version` exists precisely to + // differ between builds, so invoking it here writes an entry into every + // agent's oplog that the rollback's replay can never reproduce. The first + // run did exactly that and all 200 rollbacks were refused with + // "Unexpected oplog entry: expected component_version => 1, got 2". + // + // The forward leg only has to establish that the agents moved, so that the + // rollback has something to undo. Which revision the platform has them on + // answers that, and leaves no trace. The end state is still judged on the + // running code, at the very end, where nothing depends on it. + let rolled_forward = VersionCensus::build( + "before-rollback", + forward_revision.get() as u32, + &read_revisions(&ctx, workload_config).await, + ); + info!( + "S9: {} of {} agents are on revision {} ({} unreadable)", + rolled_forward.on_expected, + rolled_forward.agents, + forward_revision, + rolled_forward.unreadable + ); + + // ── Roll back ─────────────────────────────────────────────────────────── + // + // A redeploy of the original artifact as a new revision. Uploading it again + // rather than pointing agents back at the old revision is what "rollback" + // means operationally, and it keeps the evidence in the running code. + info!("S9: rolling back — re-uploading {COUNTERS_WASM} as a new revision"); + let back = match ctx + .user + .update_component(&manifest.counters_component_id, COUNTERS_WASM) + .await + { + Ok(updated) => updated, + Err(e) => { + warn!("S9: rollback component update failed: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::Aborted { + detail: format!("rollback upload of {COUNTERS_WASM} failed: {e:#}"), + }, + &records, + Vec::new() + ); + } + }; + let rollback_revision = back.revision; + + let mut report = RollbackReport { + forward_revision: forward_revision.get(), + rollback_revision: rollback_revision.get(), + forward_version: forward_revision.get() as u32, + rollback_version: VERSION_AFTER_ROLLBACK, + rolled_forward, + rolled_back: None, + control: ControlPlaneAttempts::default(), + rolled_forward_floor_percent: rollback_config.rolled_forward_floor_percent, + }; + + // Refuse to spend the maintenance window rolling agents back to a build + // they never left. Every check would pass and the run would prove nothing. + if !report.forward_leg_landed() { + warn!("S9: the roll-forward did not land, refusing to roll back"); + handle.stop().await; + let records = history.snapshot(); + let detail = report + .attention_lines() + .first() + .cloned() + .unwrap_or_else(|| "the roll-forward did not land".to_string()); + rollback_report = Some(report); + finish!( + TerminationReason::FaultTargetUnverified { detail }, + &records, + Vec::new() + ); + } + + info!( + "S9: rollback revision is {rollback_revision}, asking {} durable agents to return", + workload_config.durable_agents + ); + let rollback_started_at = Utc::now(); + report.control = request_updates( + &ctx, + workload_config, + rollback_revision, + rollback_config.control_retries, + rollback_config.control_retry_delay(), + ) + .await; + attention_extra.push(Note::leveled( + report.control.refused > 0, + format!( + "rollback to revision {rollback_revision} accepted for {} of {} durable agents at \ + {rollback_started_at}", + report.control.accepted(), + workload_config.durable_agents + ), + )); + rollback_report = Some(report); + + // ── Signal: ready for the fault ───────────────────────────────────────── + tokio::time::sleep(rollback_config.kill_delay()).await; + info!( + "S9: {:?} into the rollback, signalling readiness for the kill", + rollback_config.kill_delay() + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S9.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + // Agents are mid-rollback across both executors, so killing either one + // interrupts a return in flight. Which one carries no information. + fault_target: None, + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S9: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new()); + } + }; + let into_update = (injected.injected_at - rollback_started_at).num_milliseconds(); + info!( + "S9: fault {} ({} on {}) reported active at {}, {into_update}ms into the rollback", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + attention_extra.push(Note::context(format!( + "the executor kill landed {into_update}ms into the rollback" + ))); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S9: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new()); + } + }; + info!("S9: fault cleared at {}", recovered.recovered_at); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "S9: recovery phase, running for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S9: settling {SETTLE_BEFORE_READBACK:?} before read-back"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + let records = history.snapshot(); + let readback = read_back(&ctx, &records, workload_config).await; + + let rolled_back = VersionCensus::build( + "after-recovery", + VERSION_AFTER_ROLLBACK, + &read_versions(&ctx, workload_config).await, + ); + let stale: Vec = read_versions(&ctx, workload_config) + .await + .into_iter() + .filter(|(_, v)| *v == Some(VERSION_ON_THE_NEW_BUILD)) + .map(|(agent, _)| agent) + .collect(); + attention_extra.push(Note::leveled( + rolled_back.on_expected < rolled_back.agents, + format!( + "after recovery {} of {} durable agents report component version {}; {} could not \ + be read", + rolled_back.on_expected, + rolled_back.agents, + VERSION_AFTER_ROLLBACK, + rolled_back.unreadable + ), + )); + if let Some(report) = rollback_report.as_mut() { + report.rolled_back = Some(rolled_back); + } + + // ── Verdict ───────────────────────────────────────────────────────────── + let reason = if let Some(bad) = readback.iter().find(|r| { + matches!( + r.verdict, + ReadbackVerdict::LostWork | ReadbackVerdict::DuplicateExecution + ) + }) { + TerminationReason::UpdateStateInconsistent { + agent: bad.agent.clone(), + detail: format!( + "{:?}: observed {:?} against an expected range of {}..={}", + bad.verdict, bad.observed, bad.expected_min, bad.expected_max + ), + } + } else if let Some(agent) = stale.first() { + // An agent still answering with the build it was rolled back *from* is + // a real failure. One that could not be read at all is not: an + // unreadable agent is reported above and says nothing either way. + // + // `UpdateNotApplied` rather than a parallel rollback variant, because + // it is the same fact — an update that did not land — and inventing a + // second name for it would only make the two harder to search for. + TerminationReason::UpdateNotApplied { + agent: agent.clone(), + observed: Some(VERSION_ON_THE_NEW_BUILD), + expected: VERSION_AFTER_ROLLBACK, + } + } else if let Some(stream) = stream_that_never_succeeded(&ChaosSummary::build( + &records, + readback.clone(), + routing_snapshots.clone(), + fault_injected_at, + )) { + TerminationReason::StreamNeverSucceeded { + stream: stream.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, readback); +} + +/// Asks every durable agent to move to `target_revision`, concurrently. +/// +/// Returns how many requests the platform accepted. A refused request is +/// recorded and the run continues: the scenario is about what happens to the +/// updates that *did* start when the executor died. +/// Asks every durable agent to move to `target_revision`, retrying refusals. +/// +/// The retries here are the **control plane's**, counted apart from the +/// workload's. They answer a different question: a request refused because its +/// agent's executor just died says nothing about the platform's correctness, +/// but an agent nobody successfully asked to come back explains a stale agent +/// later without excusing one. Passing `0` retries makes this the same +/// fire-once call S5 does. +async fn request_updates( + ctx: &WorkloadContext, + config: &crate::chaos::WorkloadConfig, + target_revision: golem_common::model::component::ComponentRevision, + retries: u32, + delay: Duration, +) -> ControlPlaneAttempts { + let mut pending: Vec = (0..config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect(); + + let mut account = ControlPlaneAttempts { + requested: pending.len() as u64, + max_retries: retries, + ..Default::default() + }; + + for attempt in 0..=retries { + if pending.is_empty() { + break; + } + if attempt > 0 && !delay.is_zero() { + tokio::time::sleep(delay).await; + } + + let mut refused = Vec::new(); + for chunk in pending.chunks(UPDATE_CONCURRENCY) { + let mut batch = tokio::task::JoinSet::new(); + for agent in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let id = workload::counter_agent_id(&ctx, &agent); + // `disable_wakeup: false` — the agent should be woken to + // process the update rather than waiting for its next + // invocation, because the kill is timed against the + // rollback starting, not against the next caller happening + // along. + ctx.user + .auto_update_worker(&id, target_revision, false) + .await + .map_err(|e| (agent, e)) + }); + } + while let Some(joined) = batch.join_next().await { + match joined { + Ok(Ok(())) => { + if attempt == 0 { + account.accepted_first_try += 1; + } else { + account.accepted_after_retry += 1; + } + } + Ok(Err((agent, e))) => { + warn!("S9: update request for {agent} refused: {e:#}"); + refused.push(agent); + } + Err(e) => warn!("S9: an update request task panicked: {e}"), + } + } + } + pending = refused; + } + + account.refused = pending.len() as u64; + account +} + +/// Asks the platform which component revision each durable agent is on. +/// +/// Deliberately metadata rather than an invocation. See the comment at the +/// forward-leg census: invoking a method whose result differs between builds +/// writes an oplog entry that the next automatic update's replay cannot +/// reproduce, which aborts that update. Reading metadata leaves no trace. +async fn read_revisions( + ctx: &WorkloadContext, + config: &crate::chaos::WorkloadConfig, +) -> std::collections::BTreeMap> { + let names: Vec = (0..config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect(); + + let mut out = std::collections::BTreeMap::new(); + for chunk in names.chunks(UPDATE_CONCURRENCY) { + let mut batch = tokio::task::JoinSet::new(); + for agent in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let id = workload::counter_agent_id(&ctx, &agent); + let observed = ctx + .user + .get_worker_metadata(&id) + .await + .ok() + .map(|m| m.component_revision.get() as u32); + (agent, observed) + }); + } + while let Some(joined) = batch.join_next().await { + if let Ok((agent, observed)) = joined { + out.insert(agent, observed); + } + } + } + out +} + +/// Asks every durable agent which build it is running. +/// +/// `None` means the agent could not be read, which is reported rather than +/// counted against the update. +async fn read_versions( + ctx: &WorkloadContext, + config: &crate::chaos::WorkloadConfig, +) -> std::collections::BTreeMap> { + let names: Vec = (0..config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect(); + + let mut out = std::collections::BTreeMap::new(); + for chunk in names.chunks(UPDATE_CONCURRENCY) { + let mut batch = tokio::task::JoinSet::new(); + for agent in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let observed = workload::read_component_version(&ctx, &agent).await.ok(); + (agent, observed) + }); + } + while let Some(joined) = batch.join_next().await { + if let Ok((agent, observed)) = joined { + out.insert(agent, observed); + } + } + } + out +} + +/// Durable and scheduled state, compared against what the driver submitted. +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + config: &crate::chaos::WorkloadConfig, +) -> Vec { + let mut readback = Vec::new(); + + for index in 0..config.durable_agents { + let agent = ctx.agent_name(Stream::Durable, index); + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Durable && r.agent == agent); + if scoped.clone().next().is_none() { + continue; + } + let observed = workload::read_counter(ctx, &agent).await; + readback.extend(readback_for(Stream::Durable, &agent, scoped, observed)); + } + + for index in 0..config.scheduled_agents { + let target = ctx.schedule_target_name(index); + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.agent == target); + if scoped.clone().next().is_none() { + continue; + } + let observed = workload::read_polls(ctx, &target).await; + readback.extend(readback_for(Stream::Scheduled, &target, scoped, observed)); + } + + readback +} diff --git a/integration-tests/src/chaos/scenarios/storage_fault.rs b/integration-tests/src/chaos/scenarios/storage_fault.rs new file mode 100644 index 0000000000..8db2e7bde9 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/storage_fault.rs @@ -0,0 +1,932 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Storage fault: the shared choreography behind S14, S15, S16, S17, S18 and +//! S22. +//! +//! All six codes run this module. They differ in which store the fault is +//! aimed at, what it does to that store and for how long, all of which are +//! suite settings, and in what a reader should expect of the result: +//! +//! * **S16** (GOL-379) cuts the key-value cluster for the length of an AWS +//! storage failover, about a minute. The key-value retry budget covers that, +//! so the claim is that the platform absorbs it: operations stall and then +//! complete, and no executor is lost. +//! * **S22** (GOL-499) cuts the same cluster for longer than that budget. The +//! budget is then exhausted, the recovery-index write in `status_flusher` +//! panics on purpose, and executors are replaced. The claim is not survival +//! but that the exit is the intended one and nothing is lost across it. +//! * **S14** (GOL-376) cuts the *other* Aurora cluster, the one carrying the +//! oplog, again for the length of a failover. golem-dev gives the indexed +//! retry 200 attempts with a 10s cap, so the budget is not the question here +//! the way it is in the other two, and the claim is again absorption. +//! * **S18** (GOL-384) cuts neither Aurora cluster but the Redis cache in front +//! of the key-value layer, and holds it past the caller's patience rather +//! than past any platform budget. There is no budget to exhaust: golem-dev +//! configures that client to retry forever. The claim is that a stall which +//! outlasts the caller does not turn one operation into two. +//! * **S17** (GOL-375) leaves that same cache reachable and makes it slow. The +//! only scenario here that breaks nothing, and a different question rather +//! than a milder version of S18's: not whether the platform survives losing +//! its worker-status store, but whether it degrades or breaks when that store +//! gets slower. Going quiet would be a failure here rather than the expected +//! result. +//! * **S15** (GOL-374) slows the key-value cluster instead, so it is to S16 +//! what S17 is to S18 and the mirror image of S17 at the same time. The +//! claim is the same one — degradation rather than breakage — but on the +//! opposite set of streams, and it carries a risk the Redis delay does not: +//! this side's connection pools are bounded and its failure path is a panic. +//! See the suite entry for the arithmetic. +//! +//! The driver is the same in all six because the difference is one of +//! expectation, not of choreography. Nothing below asserts on which outcome +//! happened: the account it produces answers all five questions, and the +//! oracles that fail the build — the scheduled-fire account and the +//! exactly-once account — are the ones every one of them shares. +//! +//! The first scenarios in the suite that break something the platform depends +//! on rather than something the platform *is*. Every fault before these removed +//! a golem process or a link between two of them, and in each case some part of +//! the cluster stayed healthy and could be compared against. Here the executors +//! keep running, keep their shards, keep answering the shard-manager, and +//! simply cannot reach a database underneath them. +//! +//! ## What the fault takes away +//! +//! Three different pieces of the platform, depending on the code. The two +//! Aurora cuts are close to mirror images of each other; the Redis cut is a +//! third thing again. +//! +//! Reading the golem-dev executor deployment, the **key-value** cluster that +//! S16 and S22 cut carries four things: +//! +//! * promises, +//! * the running-workers set, +//! * user-defined key-value data, +//! * and the scheduler, in its own schema on the same cluster. +//! +//! The oplog lives on a different Aurora cluster and the worker-status hot +//! cache lives in Redis, and that partition touches neither. So the platform +//! keeps the ability to *record* what it did and loses the ability to know +//! *what it is doing*. +//! +//! The **indexed** cluster that S14 cuts carries the oplog and nothing else. +//! Promises still resolve, the scheduler still claims and acknowledges on time, +//! the running-workers set is still writable. What goes is the ability to +//! commit anything durable at all, which is the opposite arrangement: the +//! platform knows exactly what it is doing and cannot record any of it. +//! +//! The **Redis cache** that S18 and S17 aim at is not a cluster at all but the +//! front half of the key-value layer. `NamespaceRoutedKeyValueStorage` sends the `Worker`, +//! `AgentStatus` and `AgentStatusCheckpoint` namespaces to it and everything +//! else to Postgres, so S18 removes exactly the part S16 leaves standing and +//! S17 slows the same part instead. Both Aurora clusters stay reachable +//! throughout either. +//! +//! S15 aims at the back half of that same layer, which is the key-value cluster +//! again, and the split decides its streams too. `durable` waits on the +//! `RunningWorkers` recovery index, which `AgentStatusFlusher::on_status_changed` +//! updates synchronously whenever an agent crosses between tracked and +//! untracked; `promise` reads and writes promise keys; the scheduler registers +//! into its own schema on the same cluster. `ephemeral` reaches none of it, and +//! is the one stream that should not move. +//! +//! No stream is a control group under the Aurora *cuts*. A durable increment +//! needs the running-workers set before it can start, the worker-status cache +//! to resolve its mode, and the oplog before it can finish, so `durable` +//! degrades under both of those and must not be read as untouched. The three +//! scenarios aimed at one half of the key-value layer are the exception, and +//! deliberately so: there the streams that keep working — or keep their pace — +//! are evidence rather than noise, which is why they are judged by their own +//! expectations. See [`crate::chaos::OutageExpectation`]. +//! +//! ## The control is the baseline, not another pod +//! +//! S1 and S3 keep executors on the healthy side of the cut and read the verdict +//! off the disagreement between the two groups. There is no healthy side here: +//! all three executors share one store. So the comparison runs along time +//! instead, and every stream is measured against its own before-fault rate. See +//! [`crate::chaos::outage`] for what that costs and what it still answers. +//! +//! The first question it has to answer is whether the outage landed at all. A +//! partition that failed to take hold produces a report full of healthy numbers +//! and no error anywhere, which is the worst artifact this suite can produce, +//! so [`OutageViolation::OutageNotObserved`] is a named finding rather than +//! something a reader is left to infer from the cells. +//! +//! ## Why S18 is held longer than the others +//! +//! The first three scenarios are sized against a platform budget: S16 and S14 +//! stay inside one, S22 deliberately runs past it. S18 has no budget to run +//! past. golem-dev sets +//! `GOLEM__KEY_VALUE_STORAGE__CONFIG__CACHE__CONFIG__RETRIES__MAX_ATTEMPTS` to +//! `0`, which `RedisPool::configured` hands to fred as a `ReconnectPolicy` with +//! unlimited attempts, and it passes no performance or connection config, so +//! fred's defaults apply: no command timeout and an unbounded command buffer. +//! A cache write during the cut therefore never fails. It waits, for as long as +//! the cut lasts. +//! +//! That makes the `unwrap_or_else(|err| panic!(...))` on the cache path in +//! `WorkerService` unreachable through this fault, and it makes a short cut +//! uninformative: at 60s every caller is still inside its 120s attempt timeout, +//! so the run would only show operations taking a minute longer and completing. +//! The window is set past that timeout instead, so callers give up and retry +//! under the same idempotency key while the original write is still sitting in +//! fred's buffer. Whether those two land as one operation or two is the +//! question the scenario exists to answer, and the exactly-once account is +//! where it shows. +//! +//! ## What is expected to stall, and what is not +//! +//! Less than the whole platform, and this is the part worth reading the result +//! carefully for. `AgentStatusFlusher` took the status blob off the commit path +//! deliberately: a status change only marks the agent dirty, and a background +//! sweeper coalesces the writes. So a durable agent that is already resident +//! commits to the oplog without touching Redis synchronously, and the blob it +//! cannot flush is derivable from the oplog anyway, which is what makes the +//! staleness safe rather than merely tolerated. +//! +//! What does cross Redis synchronously is a lifecycle boundary — suspend, evict, +//! reattach — and a `get_agent_mode` miss. Run 33130077355 settled which +//! streams that amounts to: `ephemeral` alone. Its agents are created and torn +//! down per operation, so every one crosses a boundary, and it was silent for +//! 99.997% of the window while `durable`, `scheduled` and `promise` held +//! 99.94–100.06% of their baselines. +//! +//! `promise` was expected to be the second and is not. The mixed workload's +//! promise stream is `get_promise+complete` in one round trip against a durable +//! agent and never suspends; the suspending variant is `promise-wait`, which +//! S11 drives and these scenarios do not enable. +//! +//! That partial shape is why S18 does not share the other three scenarios' +//! verdict. `shareOfBaselinePercent` sits far higher here — 77.83% against +//! S14's 22.39% — without the fault being weaker, and the run-wide quiet figure +//! the other three are judged on reads 0.05%, because `durable` never stopped. +//! Judged by that rule the run reported a partition that had plainly landed as +//! one that never happened. See [`crate::chaos::OutageExpectation`] for the +//! split, and for why the streams that keep working are asserted on rather than +//! merely exempted. +//! +//! ## Why the scheduled stream is driven separately +//! +//! The mixed workload's scheduled stream registers through `schedule_poll_at`, +//! which increments a counter and records nothing else. That is enough to ask +//! "did every registration eventually fire" and useless for asking "how late". +//! Scheduler lag is one of the two things GOL-379 asks these runs to record, +//! and the only place a due time survives is the target's own fire log, so +//! these scenarios drive the token-carrying registration loop from +//! [`crate::chaos::scheduled`] instead and leave `scheduledAgents` at zero in +//! the mixed workload. Setting both is refused at load time — see +//! `ScenarioConfig::require_storage`. +//! +//! S14 is where that lag reads most directly. Its cut leaves the scheduler on +//! the healthy cluster, so claims and acknowledgements keep their timing and +//! what the delays measure is purely how long the fired invocation could not +//! commit. Under S16 and S22 the scheduler is inside the outage and the two +//! costs are not separable. +//! +//! One consequence worth stating plainly for anyone reading the result: the +//! fire account's `group` axis is degenerate here. None of these scenarios +//! names a pod to kill, so every target is reported as `elsewhere` and only the +//! `window` axis carries information. The delays to read are the `during-fault` +//! and `after-fault` cells against `before-fault`. +//! +//! ## What fails the run +//! +//! The same narrow set S3 fails on, and for the same reason: this suite reports +//! rather than judges, and the bar for failing outright is "the run produced +//! nothing worth interpreting". +//! +//! * A token-level scheduled-fire violation. An accepted action that never ran, +//! ran twice, or ran after being refused is a statement about one named +//! registration with no band of doubt around it. +//! * A key that the exactly-once probe shows executed twice. +//! * A workload that never confirmed anything at all. +//! +//! Everything the storage account finds is loud without being fatal: it lands +//! in `attention`, which CI annotates. An outage that did not land and a stream +//! that never came back are both things a human has to look at, and neither is +//! improved by turning the job red. +//! +//! Run-by-run findings live in the per-scenario runbooks in golem-cloud, not +//! here. + +use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::outage::StorageFaultReport; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::probe; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, exactly_once_termination, + read_counters, readback_for, signal_termination, snapshot_routing, wait_for_settled_routing, + write_outputs, +}; +use crate::chaos::scheduled; +use crate::chaos::signal::{BaselineReady, FaultSignals}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig, ScheduledConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::collections::BTreeSet; +use std::time::Duration; +use tracing::{info, warn}; + +/// Extra quiet after the last scheduled action is due, before anything is read +/// back. +/// +/// The rest of the settle is derived from the configuration, the same way S10 +/// derives it: the final registration falls due one `lead` after the workload +/// stops, and an action the outage delayed can cost up to one lease budget on +/// top. Reading before that has elapsed would report actions as lost that were +/// merely late, which is the one mistake this scenario cannot afford — a +/// storage outage is precisely the thing that makes work late. +const SETTLE_MARGIN: Duration = Duration::from_secs(30); + +/// How many targets to sample after the baseline to prove actions are firing at +/// all. +/// +/// A smoke test rather than a measurement: if the scheduling path is broken +/// every target is equally broken, and the point is to fail before spending the +/// fault window on a run that would report a clean account of nothing. +const FIRE_PROOF_SAMPLE: usize = 5; + +pub async fn run( + code: ScenarioCode, + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let workload_config = config.require_workload()?; + let scheduled_config = config.require_scheduled()?; + let storage_config = config.require_storage()?; + let history = OperationHistory::new(code.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(code); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![ + manifest.counters_component_id.0.to_string(), + manifest.promise_component_id.0.to_string(), + ], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let targets: Vec = (0..scheduled_config.targets) + .map(|index| ctx.schedule_target_name(index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut attention_extra: Vec = Vec::new(); + + // Every early return below goes through `finish`, so an abort produces the + // same artifact shape as a completed run, with fewer phases filled in. + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $fires:expr, $outage:expr, $exactly:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if let Some(report) = $fires { + summary = summary.with_schedule_fires(report); + } + if let Some(report) = $outage { + summary = summary.with_storage_fault(report); + } + if let Some(report) = $exactly { + summary = summary.with_exactly_once(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + info!( + "{code}: warming {} schedule emitters and targets before the baseline", + targets.len() + ); + let warmed = scheduled::warm(&ctx, &targets).await; + info!("{code}: warmed {warmed} agents, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "{code}: baseline phase, mixed workload at {} ops/s plus {} schedule targets, for {:?}", + workload_config.rate_per_sec, + targets.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let mixed = workload::start(ctx.clone(), workload_config); + let schedules = scheduled::start(ctx.clone(), &targets, scheduled_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + // Both handles have to be stopped on every exit path, and forgetting one + // leaves its emitters submitting into a history that has already been + // snapshotted. + macro_rules! stop_workloads { + () => {{ + mixed.stop().await; + schedules.stop().await; + }}; + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + // Taking a database away from a workload that never worked would + // measure nothing. Stop before touching the cluster. + warn!("{code}: baseline produced no confirmed operations, aborting before injection"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + + // Registering is not firing. The scheduler is one of the three mechanisms + // this scenario is about, and a platform that accepted every registration + // and ran none of them would otherwise reach read-back and report a + // flawless account of a mechanism that never worked. + // + // Only when the run drives the stream at all. A scenario that registers + // nothing has no fire to observe, so holding it to this gate aborts it + // during the baseline and before the fault is ever injected. That is what + // happened to S15A's first run, and it cost a cluster run to learn. + let drives_scheduled = config.drives_stream(Stream::Scheduled); + let sampled = if drives_scheduled { + sample_fire_count(code, &ctx, &targets).await + } else { + 0 + }; + if drives_scheduled && sampled == 0 { + warn!( + "{code}: {baseline_operations} operations confirmed and no scheduled action has fired" + ); + stop_workloads!(); + let records = history.snapshot(); + finish!( + TerminationReason::StreamNeverSucceeded { + stream: Stream::Scheduled.to_string(), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + info!( + "{code}: baseline complete ({baseline_operations} confirmed ops, {sampled} fires across a \ + sample of {} targets), signalling readiness", + FIRE_PROOF_SAMPLE.min(targets.len()) + ); + + // ── Signal: ready for the fault ───────────────────────────────────────── + signals.write_baseline_ready(&BaselineReady { + scenario_code: code.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + // Nothing to aim. The partition is between every executor and one + // endpoint outside the cluster, so there is no pod for the driver to + // choose and no ownership to verify. + fault_target: None, + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("{code}: no fault-injected signal arrived: {e}"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Vec::new(), + None, + None, + None + ); + } + }; + info!( + "{code}: fault {} ({} on {}) reported active at {}, {} is now unreachable from the executors", + injected.fault_id, + injected.kind, + injected.target, + injected.injected_at, + storage_config.endpoint + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("{code}: no fault-recovered signal arrived: {e}"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Vec::new(), + None, + None, + None + ); + } + }; + info!( + "{code}: fault cleared at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "{code}: recovery phase, running for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + + let skipped = schedules.skipped(); + // Stopping waits for in-flight operations to record themselves rather than + // cancelling them: an operation cancelled mid-flight is one the history + // cannot classify, and during a storage outage those are exactly the + // interesting ones. + stop_workloads!(); + + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + if skipped > 0 { + attention_extra.push(Note::attention(format!( + "{code} skipped {skipped} registration ticks because targets still had their budget \ + of {} in flight — the offered rate was clamped by the platform, so the phase counts \ + understate what the run intended to submit", + scheduled::MAX_IN_FLIGHT_PER_TARGET + ))); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Read-back ─────────────────────────────────────────────────────────── + let settle = settle_before_readback(scheduled_config); + info!("{code}: letting the last actions fall due and fire, {settle:?} before read-back"); + tokio::time::sleep(settle).await; + + let records = history.snapshot(); + let logs = scheduled::read_logs(&ctx, &targets).await; + // Archived alongside the operations, not just reduced into the report: the + // reduction is the part a later ticket is most likely to want to redo. + history.record_fire_logs(logs.clone()); + + let fault_window = fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }); + + let fires = ScheduleFireReport::build( + &records, + &logs, + scheduled_config.lead(), + fault_window, + // Empty on purpose: nothing was killed, so every target belongs to the + // report's `elsewhere` group and only its window axis carries meaning. + &BTreeSet::new(), + scheduled_config.lease_budget(), + ); + info!( + "{code}: scheduled-fire account — {} registrations accepted, {} fired once, {} \ + inconclusive, {} unverifiable, {} findings", + fires.registrations_confirmed, + fires.fired_once, + fires.inconclusive, + fires.unverifiable, + fires.findings.len() + ); + + let outage = StorageFaultReport::build( + &records, + fault_window, + &storage_config.endpoint, + storage_config.expect.clone(), + storage_config.recovery_budget(), + ); + info!( + "{code}: storage account — the streams expected to stop were silent for at least {:?}% of \ + the fault window (floor {}%) and the streams expected to carry on held at least {:?}% of \ + their baseline while {} was unreachable, {} findings", + outage.quietest_stream_percent, + outage.expect.quiet_floor_percent(), + outage.least_serving_stream_percent, + outage.endpoint, + outage.findings.len() + ); + for finding in &outage.findings { + warn!("{code}: {}: {}", finding.violation, finding.detail); + } + + let readback = read_back(&ctx, &records, workload_config, &logs).await; + + // The idempotency half of the account. A key that timed out while the + // database was gone may or may not have executed; re-invoking it under the + // same key says which, because a platform that stored the result replays it + // and one that did not runs the work again. + let before_probe = read_counters(&ctx, &records).await; + let probes = probe::probe_keys(&ctx, &records, Stream::Durable).await; + let after_probe = read_counters(&ctx, &records).await; + let exactly_once = ExactlyOnceReport::build( + &records, + &probes, + Stream::Durable, + &before_probe, + &after_probe, + ); + info!( + "{code}: exactly-once account — {} keys checked, {} with a final result, {} recovered by \ + the probe, {} findings", + exactly_once.keys_checked, + exactly_once.keys_with_final_result, + exactly_once.keys_recovered_by_probe, + exactly_once.findings.len() + ); + + let reason = termination(&fires, &exactly_once, &records); + + finish!( + reason, + &records, + readback, + Some(fires), + Some(outage), + Some(exactly_once) + ); +} + +/// Why the run stopped. +/// +/// Kept apart from [`run`] so the precedence is testable without a cluster. The +/// order matters: a token-level fire violation and a duplicated key are both +/// statements about one named thing, and either is worth more to a reader than +/// "the workload never worked", which is a statement about the run rather than +/// about the platform. +fn termination( + fires: &ScheduleFireReport, + exactly_once: &ExactlyOnceReport, + records: &[OperationRecord], +) -> TerminationReason { + if fires.has_violations() { + return TerminationReason::ScheduledFireViolated { + findings: fires.findings.len() as u64, + first: fires + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + }; + } + if let Some(reason) = exactly_once_termination(exactly_once) { + return reason; + } + if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + return TerminationReason::StreamNeverSucceeded { + stream: Stream::Durable.to_string(), + }; + } + TerminationReason::Completed +} + +/// How long to wait after the workload stops before reading anything back. +/// +/// Same derivation as S10's, and load-bearing for the same reason: the last +/// registration falls due one `lead` after the loop stops, and a delayed action +/// can cost a lease budget on top of that. +fn settle_before_readback(config: &ScheduledConfig) -> Duration { + config.lead() + config.lease_budget() + SETTLE_MARGIN +} + +/// Reads durable state back for the streams that keep a count. +/// +/// The durable counters come from the mixed workload's own agents; the +/// scheduled counts come from the fire logs that were already read, because +/// re-reading `polls` would be a second round trip for a number the log already +/// carries. +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + config: &crate::chaos::WorkloadConfig, + logs: &[crate::chaos::history::TargetFireLog], +) -> Vec { + let mut readback = Vec::new(); + + for index in 0..config.durable_agents { + let agent = ctx.agent_name(Stream::Durable, index); + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Durable && r.agent == agent); + if scoped.clone().next().is_none() { + continue; + } + let observed = workload::read_counter(ctx, &agent).await; + readback.extend(readback_for(Stream::Durable, &agent, scoped, observed)); + } + + for log in logs { + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.agent == log.agent); + let observed = match (log.polls, &log.error) { + (Some(polls), _) => Ok(polls), + (None, Some(error)) => Err(error.clone()), + (None, None) => Err(format!("target {} reported no poll count", log.agent)), + }; + readback.extend(readback_for( + Stream::Scheduled, + &log.agent, + scoped, + observed, + )); + } + + readback +} + +/// Reads the fire count of a few targets, to prove actions are firing at all. +async fn sample_fire_count(code: ScenarioCode, ctx: &WorkloadContext, targets: &[String]) -> u64 { + let mut total = 0u64; + for target in targets.iter().take(FIRE_PROOF_SAMPLE) { + match workload::read_polls(ctx, target).await { + Ok(polls) => total += polls, + Err(e) => warn!("{code}: could not sample fires on {target}: {e}"), + } + } + total +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::OutageExpectation; + use crate::chaos::history::{AttemptRecord, FireRecord, TargetFireLog}; + use crate::chaos::outage::{OutageFinding, OutageViolation}; + use chrono::{DateTime, TimeDelta}; + use test_r::test; + + const TARGET: &str = "chaos-s16-schedule-target-0000"; + const LEAD: Duration = Duration::from_secs(10); + const BUDGET: Duration = Duration::from_secs(240); + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-25T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn registration(token: &str, outcome: Outcome) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Scheduled, + phase: Phase::Baseline, + agent: TARGET.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: token.to_string(), + submitted_at: t0(), + completed_at: Some(t0() + TimeDelta::milliseconds(20)), + attempts: 1, + outcome, + duration_ms: 20, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: t0(), + duration_ms: 20, + returned_value: None, + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + fn log(fired: &[&str]) -> TargetFireLog { + TargetFireLog { + agent: TARGET.to_string(), + polls: Some(fired.len() as u64), + fires: fired + .iter() + .map(|token| FireRecord { + token: (*token).to_string(), + scheduled_at: t0() + TimeDelta::seconds(10), + observed_at: t0() + TimeDelta::seconds(11), + }) + .collect(), + error: None, + } + } + + fn fires(records: &[OperationRecord], logs: &[TargetFireLog]) -> ScheduleFireReport { + ScheduleFireReport::build(records, logs, LEAD, None, &BTreeSet::new(), BUDGET) + } + + /// The empty account: nothing registered, nothing fired, nothing to say. + fn clean_exactly_once() -> ExactlyOnceReport { + ExactlyOnceReport::build( + &[], + &[], + Stream::Durable, + &std::collections::BTreeMap::new(), + &std::collections::BTreeMap::new(), + ) + } + + /// A registration the platform accepted whose action never ran is a + /// statement about one named token, and it outranks every aggregate. + #[test] + fn a_token_that_never_fired_fails_the_run() { + let records = vec![registration("token-a", Outcome::Confirmed)]; + let reason = termination( + &fires(&records, &[log(&[])]), + &clean_exactly_once(), + &records, + ); + + assert!( + matches!(reason, TerminationReason::ScheduledFireViolated { .. }), + "expected a fire violation, got {reason:?}" + ); + } + + /// A fire violation outranks "the workload never worked", because it says + /// something about the platform and the other says something about the run. + #[test] + fn a_fire_violation_outranks_a_workload_that_never_confirmed_anything() { + // Rejected registrations mean nothing confirmed anywhere, and a fire + // that happened anyway is the sharpest thing this suite can find. + let records = vec![registration("token-a", Outcome::Rejected)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + + assert!( + matches!(reason, TerminationReason::ScheduledFireViolated { .. }), + "expected a fire violation, got {reason:?}" + ); + } + + /// A run where nothing confirmed at all says nothing about resilience, so + /// it fails rather than reporting a clean account of a workload that never + /// worked. + #[test] + fn a_workload_that_never_confirmed_anything_fails_the_run() { + let records = vec![registration("token-a", Outcome::Indeterminate)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + + assert!( + matches!(reason, TerminationReason::StreamNeverSucceeded { .. }), + "expected a never-succeeded reason, got {reason:?}" + ); + } + + /// The healthy shape completes, and the storage account's own findings do + /// not change that: an outage that failed to land is loud in `attention` + /// and deliberately not fatal. + #[test] + fn a_healthy_run_completes() { + let records = vec![registration("token-a", Outcome::Confirmed)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + + assert_eq!(reason, TerminationReason::Completed); + } + + /// The storage account's findings never reach the termination reason. An + /// outage that failed to land is the loudest thing these scenarios can + /// report, and it is deliberately not fatal: turning the job red would say + /// the platform did something wrong, and what actually went wrong is the + /// experiment. + #[test] + fn a_storage_finding_does_not_change_the_termination_reason() { + let mut outage = StorageFaultReport::build( + &[], + None, + "db.example", + OutageExpectation::WholeWorkload { + quiet_floor_percent: 15.0, + }, + Duration::from_secs(120), + ); + outage.findings.push(OutageFinding { + violation: OutageViolation::OutageNotObserved, + stream: None, + detail: "kept working".to_string(), + }); + assert!(outage.has_findings()); + + let records = vec![registration("token-a", Outcome::Confirmed)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + assert_eq!(reason, TerminationReason::Completed); + } +} diff --git a/integration-tests/src/chaos/scheduled.rs b/integration-tests/src/chaos/scheduled.rs new file mode 100644 index 0000000000..d7707f9743 --- /dev/null +++ b/integration-tests/src/chaos/scheduled.rs @@ -0,0 +1,418 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The scheduled-registration workload (GOL-378). +//! +//! The mixed workload's scheduled stream registers a poll and counts the fires. +//! This one registers a `fire` carrying the registration's own idempotency key, +//! which is what lets [`crate::chaos::fires`] pair an individual action against +//! the invocation that asked for it. +//! +//! ## Why the cadence, and why the lead +//! +//! `lead x targets / interval` is the population that matters: actions the +//! platform has accepted and not yet run. At 100 targets registering every two +//! seconds, ten seconds ahead, that is five hundred of them standing at any +//! instant, spread across both executors' shards. A kill lands in the middle of +//! that population by construction, which is what makes the cadence and the +//! lead the two numbers that decide whether the run measures anything. +//! +//! The lead also has to comfortably exceed the workflow's inject-and-verify +//! path — signal poll, `kubectl apply`, waiting for `AllInjected` — or every +//! action registered before the kill would already have fired by the time the +//! pod died. See [`crate::chaos::scenarios::s10`] for why the narrower +//! claim-to-acknowledge window is not something the driver tries to aim at. +//! +//! ## Why the target set is split rather than pinned +//! +//! [`crate::chaos::pinned`] drives *only* the agents its chosen executor owns, +//! because an S8 operation that was not on the dead pod says nothing. Here the +//! opposite is true: the actions on the surviving executor are the control +//! group. Every target is driven, the driver names the executor owning the +//! largest share, and the report splits the two so a lease recovery that took +//! its full TTL cannot hide behind the half of the population that was never +//! disturbed. + +use crate::chaos::ScheduledConfig; +use crate::chaos::history::{FireRecord, Stream, TargetFireLog}; +use crate::chaos::workload::{self, SCHEDULE_EMITTER_AGENT, WorkloadContext}; +use chrono::{DateTime, Utc}; +use golem_common::base_model::agent::ParsedAgentId; +use golem_common::{agent_id, data_value}; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; +use tracing::{info, warn}; + +/// Registrations one target may have in flight at once. +/// +/// Small on purpose. The cadence is what sets the rate; this only keeps a +/// stalled platform from accumulating tasks, and a target that has spent its +/// budget skips its tick and says so rather than queueing behind itself. +pub const MAX_IN_FLIGHT_PER_TARGET: usize = 8; + +/// How many targets are read back at once. Same reasoning as +/// [`crate::chaos::scenarios::read_back_agents`]: reads do not mutate, and +/// walking them one at a time behind a per-read ceiling outlasts the +/// maintenance window. +const READ_CONCURRENCY: usize = 16; + +/// The executor S10 aims at, and how its targets divide around it. +/// +/// An alias rather than a type of its own: S11 splits its waiters exactly the +/// same way, the logic lives in [`crate::chaos::split`], and the name is kept +/// here because `scheduledSelection` is the key the archived result and the +/// golem-cloud report both read. +pub type ScheduledSelection = crate::chaos::split::PodSplit; + +/// Chooses the executor to aim at. See [`crate::chaos::split::select`]. +pub async fn select( + ctx: &WorkloadContext, + deps: &BenchmarkTestDependencies, + targets: &[String], +) -> anyhow::Result { + crate::chaos::split::select(crate::chaos::split::schedule_subject(ctx), deps, targets).await +} + +/// Re-checks the division immediately before injection. See +/// [`crate::chaos::split::verify_ownership`]. +pub async fn verify_ownership( + ctx: &WorkloadContext, + deps: &BenchmarkTestDependencies, + selection: &ScheduledSelection, +) -> anyhow::Result<()> { + crate::chaos::split::verify_ownership( + crate::chaos::split::schedule_subject(ctx), + deps, + selection, + ) + .await +} + +/// A running registration workload. As elsewhere, dropping the handle does not +/// stop it: call [`ScheduledHandle::stop`] so in-flight registrations record +/// themselves instead of being cancelled mid-flight. +pub struct ScheduledHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, + skipped: Arc, +} + +impl ScheduledHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + /// Ticks a target dropped because it already had its budget of + /// registrations in flight. Non-zero means the platform was slow enough to + /// clamp the cadence, which is context for reading everything else. + pub fn skipped(&self) -> u64 { + self.skipped.load(Ordering::Relaxed) + } + + pub async fn stop(mut self) { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + info!( + "Chaos scheduled workload stopped after {} registrations ({} ticks skipped)", + self.submitted(), + self.skipped() + ); + } +} + +/// Starts one registration loop per target. +pub fn start( + ctx: WorkloadContext, + targets: &[String], + config: &ScheduledConfig, +) -> ScheduledHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let skipped = Arc::new(AtomicU64::new(0)); + let mut tasks = JoinSet::new(); + let interval = config.interval(); + let lead = config.lead(); + let count = targets.len().max(1); + + info!( + "Chaos scheduled workload starting: {} targets, one registration every {:?} each \ + ({:.1}/s overall), {:?} ahead", + targets.len(), + interval, + targets.len() as f64 / interval.as_secs_f64(), + lead + ); + + for (index, target) in targets.iter().enumerate() { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + let skipped = skipped.clone(); + let target = target.clone(); + // Spread the loops across one interval so the whole population does not + // register in the same instant, which would make the offered rate a + // sawtooth instead of the constant the phase stats assume. + let stagger = interval.mul_f64(index as f64 / count as f64); + + tasks.spawn(async move { + tokio::time::sleep(stagger).await; + let budget = Arc::new(Semaphore::new(MAX_IN_FLIGHT_PER_TARGET)); + let mut in_flight = JoinSet::new(); + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut seq = 0u64; + + while stop.load(Ordering::Relaxed) == 0 { + ticker.tick().await; + if stop.load(Ordering::Relaxed) != 0 { + break; + } + let Ok(permit) = budget.clone().try_acquire_owned() else { + skipped.fetch_add(1, Ordering::Relaxed); + continue; + }; + submitted.fetch_add(1, Ordering::Relaxed); + let ctx = ctx.clone(); + let target = target.clone(); + let this_seq = seq; + seq += 1; + in_flight.spawn(async move { + let _permit = permit; + register_one(&ctx, index as u32, &target, this_seq, lead).await; + }); + while in_flight.try_join_next().is_some() {} + } + + // Drain rather than cancel: a registration cancelled mid-flight is + // one the history cannot classify, and during a fault those are the + // interesting ones. + while in_flight.join_next().await.is_some() {} + }); + } + + ScheduledHandle { + stop, + tasks, + submitted, + skipped, + } +} + +/// Registers one action, `lead` from now, under a key the fire will carry back. +async fn register_one(ctx: &WorkloadContext, index: u32, target: &str, seq: u64, lead: Duration) { + let emitter = ctx.agent_name(Stream::Scheduled, index); + let key = ctx.idempotency_key(target, seq); + let parsed: ParsedAgentId = agent_id!(SCHEDULE_EMITTER_AGENT, emitter); + + // Computed once, outside the retry, so a retried registration asks for the + // same instant. A due time that moved with each attempt would make the fire + // log's `scheduledMillis` disagree with what the driver believes it asked + // for, and the delay measurement would be against the wrong baseline. + let fire_at = SystemTime::now() + lead; + let since_epoch = fire_at.duration_since(UNIX_EPOCH).unwrap_or_default(); + let (secs, nanos) = (since_epoch.as_secs(), since_epoch.subsec_nanos()); + + let ctx2 = ctx.clone(); + let target2 = target.to_string(); + // Recorded against the target rather than the emitter, because the target is + // where the fire lands and where the read-back looks. + workload::run_operation( + ctx, + Stream::Scheduled, + target.to_string(), + "schedule_fire_at", + key, + |k| { + let ctx = ctx2.clone(); + let parsed = parsed.clone(); + let target = target2.clone(); + async move { + let token = k.value.clone(); + ctx.user + .invoke_and_await_agent_with_key( + &ctx.counters, + &parsed, + &k, + "schedule_fire_at", + data_value!(target, secs, nanos, token), + ) + .await?; + Ok(None) + } + }, + ) + .await; +} + +/// Creates every emitter and target before the baseline starts. +/// +/// Returns how many agents were touched. Residency matters here for the same +/// reason it does in S8: an agent that has to cold-start on first use would put +/// start-up cost inside the baseline the recovery is measured against. +pub async fn warm(ctx: &WorkloadContext, targets: &[String]) -> usize { + let mut warmed = 0usize; + for (offset, chunk) in targets.chunks(READ_CONCURRENCY).enumerate() { + let mut batch = JoinSet::new(); + for (position, target) in chunk.iter().cloned().enumerate() { + let ctx = ctx.clone(); + let index = (offset * READ_CONCURRENCY + position) as u32; + batch.spawn(async move { + let emitter = ctx.agent_name(Stream::Scheduled, index); + let parsed: ParsedAgentId = agent_id!(SCHEDULE_EMITTER_AGENT, emitter.clone()); + if let Err(e) = ctx + .user + .invoke_and_await_agent(&ctx.counters, &parsed, "warm", data_value!()) + .await + { + warn!("S10: could not warm emitter {emitter}: {e:#}"); + } + // Reading the target creates it without mutating it, which is + // exactly the side effect wanted. + if let Err(e) = workload::read_polls(&ctx, &target).await { + warn!("S10: could not warm target {target}: {e}"); + } + }); + } + while batch.join_next().await.is_some() { + warmed += 1; + } + } + warmed +} + +/// Reads every target's fire log back. +/// +/// A failed read is carried as [`TargetFireLog::error`] rather than dropped: an +/// agent that could not be read says nothing either way about its actions, and +/// the account has to be able to tell that apart from an agent that lost them. +pub async fn read_logs(ctx: &WorkloadContext, targets: &[String]) -> Vec { + let total = targets.len(); + let mut logs = Vec::with_capacity(total); + + for chunk in targets.chunks(READ_CONCURRENCY) { + let mut batch = JoinSet::new(); + for target in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let polls = workload::read_polls(&ctx, &target).await.ok(); + let fires = workload::read_fires(&ctx, &target).await; + match fires { + Ok(raw) => TargetFireLog { + agent: target, + polls, + fires: to_fire_records(&raw), + error: None, + }, + Err(e) => TargetFireLog { + agent: target, + polls, + fires: Vec::new(), + error: Some(e), + }, + } + }); + } + + let mut batch_results = Vec::new(); + while let Some(joined) = batch.join_next().await { + match joined { + Ok(log) => batch_results.push(log), + Err(e) => warn!("S10: a fire-log read task panicked: {e}"), + } + } + batch_results.sort_by(|a, b| a.agent.cmp(&b.agent)); + logs.extend(batch_results); + info!("S10: read fire logs for {} of {total} targets", logs.len()); + } + + logs +} + +/// Turns the agent's raw triples into timestamps. +/// +/// An entry whose millis cannot be a timestamp is dropped rather than clamped. +/// Dropping leaves the log shorter than the target's own `polls`, which is what +/// the account reads as "this target cannot testify" — a clamped nonsense +/// timestamp would instead have produced a confident wrong delay. +pub fn to_fire_records(raw: &[(String, u64, u64)]) -> Vec { + let mut out = Vec::with_capacity(raw.len()); + for (token, scheduled_millis, observed_millis) in raw { + match ( + millis_to_time(*scheduled_millis), + millis_to_time(*observed_millis), + ) { + (Some(scheduled_at), Some(observed_at)) => out.push(FireRecord { + token: token.clone(), + scheduled_at, + observed_at, + }), + _ => warn!( + "S10: dropping fire log entry {token} with unreadable timestamps \ + ({scheduled_millis}, {observed_millis})" + ), + } + } + out +} + +fn millis_to_time(millis: u64) -> Option> { + DateTime::from_timestamp_millis(i64::try_from(millis).ok()?) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + /// The happy path of the conversion the whole account is built on. + #[test] + fn raw_triples_become_timestamped_fire_records() { + let raw = vec![("t-0".to_string(), 1_800_000_000_000, 1_800_000_000_450)]; + let records = to_fire_records(&raw); + assert_eq!(records.len(), 1); + assert_eq!(records[0].token, "t-0"); + assert_eq!(records[0].delay_ms(), 450); + } + + /// An unreadable entry has to shorten the log rather than become a + /// confident wrong delay: the shortfall against `polls` is what tells the + /// account this target cannot testify. + #[test] + fn an_entry_with_an_impossible_timestamp_is_dropped_rather_than_clamped() { + let raw = vec![ + ("good".to_string(), 1_800_000_000_000, 1_800_000_000_010), + ("bad".to_string(), u64::MAX, 1_800_000_000_010), + ]; + let records = to_fire_records(&raw); + assert_eq!(records.len(), 1); + assert_eq!(records[0].token, "good"); + + let log = TargetFireLog { + agent: "target-0".to_string(), + polls: Some(2), + fires: records, + error: None, + }; + assert!( + !log.is_complete(), + "a dropped entry must leave the log short of polls" + ); + } +} diff --git a/integration-tests/src/chaos/split.rs b/integration-tests/src/chaos/split.rs new file mode 100644 index 0000000000..29d7d2e381 --- /dev/null +++ b/integration-tests/src/chaos/split.rs @@ -0,0 +1,512 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Aiming a kill at one executor while still driving the agents it does not own. +//! +//! [`crate::chaos::pinned`] answers a different question. There, an operation +//! that was not on the dead pod says nothing, so the driver keeps only the +//! agents its chosen executor owns and discards the rest. Here every agent is +//! driven and the ones elsewhere are the run's own control group: on a +//! two-executor cluster roughly half the population is never touched, and +//! reporting one percentile across both would let a recovery that took its full +//! budget hide behind the half that was never disturbed. +//! +//! Both scenarios that work this way — S10's schedule targets and S11's promise +//! waiters — need exactly the same three things, which is why they live here +//! rather than being written twice: pick the executor owning the largest share, +//! refuse to proceed if that share is too small to mean anything, and re-check +//! the division immediately before the fault is injected. + +use crate::chaos::pinned::{owners_by_pod_in, pod_ip_of, routing_agent_id_in}; +use crate::chaos::workload::WorkloadContext; +use anyhow::Context; +use chrono::{DateTime, Utc}; +use golem_common::model::component::ComponentDto; +use golem_test_framework::config::{BenchmarkTestDependencies, TestDependencies}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use tracing::info; + +/// The smallest share of agents one executor must own for the run to mean +/// anything, as a divisor of the population. +/// +/// A two-executor cluster splits a hashed population roughly evenly, so a +/// quarter is a floor rather than an expectation. Below it the "affected" group +/// is too small for its percentile to say anything, and a run that reported one +/// anyway would be worse than one that refused. +const MIN_SHARE_DIVISOR: usize = 4; + +/// What the split is about, for the messages a reader eventually sees. +/// +/// Carried rather than hard-coded because the failure modes here are reported to +/// an operator mid-maintenance-window, and "the most-loaded executor owns only 6 +/// of 100 agents" is a worse thing to read at 3am than the same sentence naming +/// promise waiters. +#[derive(Debug, Clone, Copy)] +pub struct Subject<'a> { + /// Scenario code, used only to prefix log lines. + pub scenario: &'a str, + /// The component the agents live in. Ownership is per agent id and an agent + /// id contains its component, so this is not cosmetic. + pub component: &'a ComponentDto, + /// Agent type, e.g. `ScheduleCounter`. + pub agent_type: &'a str, + /// Plural noun for messages, e.g. `schedule targets`. + pub noun: &'a str, +} + +/// The fault window, as the workflow reported it. +#[derive(Debug, Clone, Copy)] +pub struct FaultWindow { + pub injected_at: DateTime, + /// Absent for a run that never saw the fault clear. + pub recovered_at: Option>, +} + +/// Which side of the fault an event fell on. +/// +/// Shared rather than written per scenario because the classification is the +/// same question every time — an event before the kill, while the executor was +/// gone, or after it came back — and because the three names end up in archived +/// results that a reader compares across scenarios. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Window { + BeforeFault, + DuringFault, + AfterFault, + /// The run never learned when the fault was injected, so nothing can be + /// placed relative to it. + Unknown, +} + +impl Window { + pub fn as_str(self) -> &'static str { + match self { + Window::BeforeFault => "before-fault", + Window::DuringFault => "during-fault", + Window::AfterFault => "after-fault", + Window::Unknown => "unknown", + } + } + + /// Where `at` falls relative to the fault. + pub fn of(at: DateTime, fault: Option) -> Self { + match fault { + None => Window::Unknown, + Some(window) if at < window.injected_at => Window::BeforeFault, + Some(FaultWindow { + recovered_at: Some(recovered), + .. + }) if at >= recovered => Window::AfterFault, + Some(_) => Window::DuringFault, + } + } +} + +impl std::fmt::Display for Window { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// When a window began, for the windows that have a fixed start. +/// +/// The fault's own boundaries come from the workflow's timestamps, which is +/// what makes them comparable across runs. The baseline has no boundary of its +/// own, so it starts at the first operation the run offered. +pub fn window_start( + window: Window, + fault: Option, + first_submitted: Option>, +) -> Option> { + match (window, fault) { + (Window::BeforeFault, Some(_)) => first_submitted, + (Window::DuringFault, Some(w)) => Some(w.injected_at), + (Window::AfterFault, Some(w)) => w.recovered_at, + _ => None, + } +} + +/// When a window closed. +/// +/// The mirror of [`window_start`], and needed wherever a gap has to be measured +/// against the window's own edges rather than against the first and last thing +/// that happened inside it: a stream that went silent for the whole back half +/// of a window has no later timestamp to compare against, so the edge is the +/// only thing that can show it. +pub fn window_end( + window: Window, + fault: Option, + last_completed: Option>, +) -> Option> { + match (window, fault) { + (Window::BeforeFault, Some(w)) => Some(w.injected_at), + (Window::DuringFault, Some(w)) => w.recovered_at.or(last_completed), + (Window::AfterFault, Some(_)) => last_completed, + _ => None, + } +} + +/// The longest stretch of a window in which nothing was answered. +/// +/// The window's own edges are the first and last comparison points, which is +/// what separates "answered steadily but slowly" from "answered nothing for two +/// minutes and then caught up in a burst". Those two produce the same count and +/// the same rate; only this tells them apart. +pub fn longest_silence_ms( + served_at: &[DateTime], + start: Option>, + end: Option>, +) -> Option { + let (start, end) = (start?, end?); + let mut marks: Vec> = served_at + .iter() + .copied() + .filter(|at| *at >= start && *at <= end) + .collect(); + marks.sort_unstable(); + marks.insert(0, start); + marks.push(end); + Some( + marks + .windows(2) + .map(|pair| (pair[1] - pair[0]).num_milliseconds().max(0) as u64) + .max() + .unwrap_or(0), + ) +} + +/// How long a window lasted, in seconds. +/// +/// The fault's own windows come from the workflow's timestamps, which is what +/// makes them comparable across runs. The two open-ended ones are bounded by +/// the workload instead: the baseline starts at the first operation offered, +/// and recovery ends at the last one that came back. +pub fn window_secs( + window: Window, + fault: Option, + first_submitted: Option>, + last_completed: Option>, +) -> f64 { + let seconds = |from: DateTime, to: DateTime| { + (to - from).num_milliseconds().max(0) as f64 / 1000.0 + }; + match (window, fault) { + (Window::BeforeFault, Some(w)) => first_submitted + .map(|first| seconds(first, w.injected_at)) + .unwrap_or(0.0), + (Window::DuringFault, Some(w)) => match (w.recovered_at, last_completed) { + (Some(recovered), _) => seconds(w.injected_at, recovered), + // A run that never saw the heal: the fault ran to whatever the last + // operation saw, which is the most that can be claimed. + (None, Some(last)) => seconds(w.injected_at, last), + (None, None) => 0.0, + }, + ( + Window::AfterFault, + Some(FaultWindow { + recovered_at: Some(recovered), + .. + }), + ) => last_completed + .map(|last| seconds(recovered, last)) + .unwrap_or(0.0), + _ => 0.0, + } +} + +/// Two decimal places, for the rates and shares that end up in a result. +pub fn round2(value: f64) -> f64 { + (value * 100.0).round() / 100.0 +} + +/// The executor the fault will be aimed at, and how the agents divide around it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PodSplit { + /// The executor endpoint as the shard-manager names it, e.g. + /// `10.0.14.207:9000`. + pub pod_address: String, + /// Host part of the address, which is what a Kubernetes `status.podIP` + /// field selector matches. + pub pod_ip: String, + /// Agents this executor owns: the population that has to survive recovery. + pub on_pod: Vec, + /// Agents owned by any other executor: the run's own control group. + pub elsewhere: Vec, + /// How the agents spread across executors, so a run that refused to proceed + /// says whether the cluster was lopsided or the pool too small. + pub targets_per_pod: BTreeMap, + /// Shard count the routing table reported. Ownership is a hash modulo this, + /// so a selection cannot be re-derived later without it. + pub number_of_shards: usize, +} + +impl PodSplit { + /// Which group an agent belongs to, or `None` for one the selection never + /// saw. + pub fn group_of(&self, agent: &str) -> Option { + if self.on_pod.iter().any(|a| a == agent) { + Some(Group::OnPod) + } else if self.elsewhere.iter().any(|a| a == agent) { + Some(Group::Elsewhere) + } else { + None + } + } +} + +/// Which side of the kill an agent was on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Group { + /// Owned by the executor the fault was aimed at. + OnPod, + /// Owned by some other executor: the control group. + Elsewhere, +} + +impl Group { + pub fn as_str(self) -> &'static str { + match self { + Group::OnPod => "on-pod", + Group::Elsewhere => "elsewhere", + } + } +} + +/// Chooses the executor to aim at: the one owning the largest share of agents. +/// +/// Fails rather than proceeding unaimed. Chaos Mesh's `mode: one` would pick a +/// pod at random, and a run that killed an executor owning six agents out of a +/// hundred would still produce a confident-looking report about recovery. +pub async fn select( + subject: Subject<'_>, + deps: &BenchmarkTestDependencies, + agents: &[String], +) -> anyhow::Result { + let table = deps + .shard_manager() + .get_routing_table() + .await + .with_context(|| { + format!( + "reading the routing table to aim the {} fault", + subject.noun + ) + })?; + + let by_pod = owners_by_pod_in(subject.component, &table, subject.agent_type, agents); + let targets_per_pod: BTreeMap = by_pod + .iter() + .map(|(pod, xs)| (pod.clone(), xs.len())) + .collect(); + + let (pod_address, on_pod) = by_pod + .iter() + .max_by_key(|(_, agents)| agents.len()) + .map(|(pod, agents)| (pod.clone(), agents.clone())) + .ok_or_else(|| { + anyhow::anyhow!( + "routing table assigned none of the {} {} to any executor", + agents.len(), + subject.noun + ) + })?; + + let floor = (agents.len() / MIN_SHARE_DIVISOR).max(1); + if on_pod.len() < floor { + anyhow::bail!( + "the most-loaded executor owns only {} of {} {}, below the {floor} needed for its \ + share to be worth measuring: {targets_per_pod:?}", + on_pod.len(), + agents.len(), + subject.noun + ); + } + + let elsewhere: Vec = agents + .iter() + .filter(|t| !on_pod.contains(t)) + .cloned() + .collect(); + + info!( + "{}: aiming at executor {pod_address}, which owns {} of {} {} ({} elsewhere, across {} \ + executors)", + subject.scenario, + on_pod.len(), + agents.len(), + subject.noun, + elsewhere.len(), + targets_per_pod.len() + ); + + Ok(PodSplit { + pod_ip: pod_ip_of(&pod_address), + pod_address, + on_pod, + elsewhere, + targets_per_pod, + number_of_shards: table.number_of_shards.value, + }) +} + +/// Re-checks, against a freshly read routing table, that the agents are still +/// divided the way the selection says. +/// +/// Called immediately before the readiness signal, for the same reason +/// [`crate::chaos::pinned`] does it: a rebalance between selection and injection +/// would leave the run reporting a control group that was actually the affected +/// one. +pub async fn verify_ownership( + subject: Subject<'_>, + deps: &BenchmarkTestDependencies, + split: &PodSplit, +) -> anyhow::Result<()> { + let table = deps + .shard_manager() + .get_routing_table() + .await + .with_context(|| { + format!( + "re-reading the routing table to verify {} ownership", + subject.noun + ) + })?; + + let mut drifted = Vec::new(); + for agent in &split.on_pod { + let owner = table + .lookup(&routing_agent_id_in( + subject.component, + subject.agent_type, + agent, + )) + .map(|pod| pod.to_string()); + if owner.as_deref() != Some(split.pod_address.as_str()) { + drifted.push(format!( + "{agent} now owned by {}", + owner.unwrap_or_else(|| "nobody".to_string()) + )); + } + } + + if !drifted.is_empty() { + anyhow::bail!( + "{} of {} {} are no longer owned by {}: {}", + drifted.len(), + split.on_pod.len(), + subject.noun, + split.pod_address, + drifted.join(", ") + ); + } + + info!( + "{}: verified all {} {} are still owned by {}", + subject.scenario, + split.on_pod.len(), + subject.noun, + split.pod_address + ); + Ok(()) +} + +/// The counters component's schedule targets, as S10 aims at them. +pub fn schedule_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S10", + component: &ctx.counters, + agent_type: crate::chaos::workload::SCHEDULE_COUNTER_AGENT, + noun: "schedule targets", + } +} + +/// The counters component's durable agents, as S3 aims at them. +/// +/// The same agent type the mixed workload's durable stream drives, because it +/// is the same population: S3's emitters exist to pace it per agent, not to +/// invent a new kind of agent. +pub fn counter_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S3", + component: &ctx.counters, + agent_type: crate::chaos::workload::COUNTER_AGENT, + noun: "counter agents", + } +} + +/// The counters component's revert agents, as S7 aims at them. +pub fn revert_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S7", + component: &ctx.counters, + agent_type: crate::chaos::workload::COUNTER_AGENT, + noun: "revert agents", + } +} + +/// The counters component's delete slots, as S6 aims at them. +pub fn delete_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S6", + component: &ctx.counters, + agent_type: crate::chaos::workload::COUNTER_AGENT, + noun: "delete agents", + } +} + +/// The promise component's waiters, as S11 aims at them. +pub fn waiter_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S11", + component: &ctx.promise, + agent_type: crate::chaos::waiters::PROMISE_WAITER_AGENT, + noun: "promise waiters", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec!["a".to_string(), "b".to_string()], + elsewhere: vec!["c".to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + #[test] + fn a_split_places_each_agent_in_exactly_one_group() { + let split = split(); + assert_eq!(split.group_of("a"), Some(Group::OnPod)); + assert_eq!(split.group_of("c"), Some(Group::Elsewhere)); + } + + /// An agent the selection never saw is not silently counted as a control: + /// the caller has to decide what an unknown agent means, because in every + /// scenario here it means the population drifted. + #[test] + fn an_agent_outside_the_selection_belongs_to_no_group() { + assert_eq!(split().group_of("z"), None); + } +} diff --git a/integration-tests/src/chaos/steady.rs b/integration-tests/src/chaos/steady.rs new file mode 100644 index 0000000000..f2d4a1247c --- /dev/null +++ b/integration-tests/src/chaos/steady.rs @@ -0,0 +1,122 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! One emitter per agent, each holding at most one operation (GOL-370). +//! +//! [`crate::chaos::workload`] drives streams: a shared rate, a shared in-flight +//! budget, and agents picked round-robin out of a pool. That is the right shape +//! when the thing being disturbed is a *stream*, and the wrong one when it is a +//! *place*. +//! +//! S3 cuts one executor off from worker-service. Every agent that executor owns +//! stalls, and those agents are spread evenly through every stream, so a shared +//! budget would be consumed by stalled operations within seconds and the agents +//! on the reachable executor would stop being submitted too. The run would then +//! show the undisturbed half degrading in lockstep with the disturbed one, and +//! the cause would be the driver rather than the platform. The mixed workload +//! already carries a comment about exactly this failure along the stream axis; +//! this module is the same lesson along the ownership axis. +//! +//! So: one task per agent, one operation in flight at a time, and a cadence +//! measured from the end of the previous operation rather than from a shared +//! clock. An agent whose executor is unreachable then contributes nothing and +//! costs nothing, which is what makes the two groups' throughput comparable. + +use crate::chaos::history::Stream; +use crate::chaos::workload::{self, WorkloadContext}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::Duration; +use tokio::task::JoinSet; +use tracing::info; + +/// The agents a run of `count` emitters drives, in index order. +/// +/// Delegates to [`WorkloadContext::agent_name`] rather than formatting a name +/// of its own, so the two cannot drift. That matters: the shared read-back and +/// exactly-once machinery both select on [`Stream::Durable`] and on names of +/// exactly that shape, and a second copy of the format here would empty both +/// silently rather than fail. +pub fn agent_names(ctx: &WorkloadContext, count: u32) -> Vec { + (0..count) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect() +} + +/// A running steady workload. Dropping the handle does not stop it — call +/// [`SteadyHandle::stop`], so operations in flight record themselves rather than +/// being cancelled. During a partition those are precisely the operations the +/// run exists to describe. +pub struct SteadyHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, +} + +impl SteadyHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + pub async fn stop(mut self) { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + info!( + "Chaos steady workload stopped after {} operations", + self.submitted() + ); + } +} + +/// Starts one emitter per agent and keeps them running until +/// [`SteadyHandle::stop`]. +pub fn start(ctx: WorkloadContext, agents: u32, interval: Duration) -> SteadyHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let mut tasks = JoinSet::new(); + + info!( + "Chaos steady workload starting: {agents} emitters, one operation each, {interval:?} \ + between them" + ); + + for index in 0..agents { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + + tasks.spawn(async move { + let mut seq: u64 = 0; + while stop.load(Ordering::Relaxed) == 0 { + submitted.fetch_add(1, Ordering::Relaxed); + workload::submit_one(&ctx, Stream::Durable, index, seq).await; + seq += 1; + // From the end of the operation, not from a fixed clock. An + // agent that just spent two minutes stalled must not then fire + // a burst of catch-up operations: that would put its recovery + // throughput above its baseline and make the recovery cell + // unreadable. + if stop.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(interval).await; + } + } + }); + } + + SteadyHandle { + stop, + tasks, + submitted, + } +} diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index 79416c03fd..51ab2ad401 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -48,9 +48,16 @@ //! handful of suspect keys on it, which the workflow turns into ready-made trace //! queries. One global counter would have produced a haystack instead. +use crate::chaos::fires::ScheduleFireReport; use crate::chaos::history::{Outcome, Phase, Stream}; +use crate::chaos::outage::StorageFaultReport; use crate::chaos::ownership::OwnershipSample; use crate::chaos::probe::KeyProbe; +use crate::chaos::reachability::ReachabilityReport; +use crate::chaos::resurrection::ResurrectionReport; +use crate::chaos::rollback::RollbackReport; +use crate::chaos::truncation::TruncationReport; +use crate::chaos::wakeups::WakeupReport; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::time::Duration; @@ -484,6 +491,62 @@ impl ExactlyOnceReport { } } +/// Whether a line a scenario reports is a finding or context. +/// +/// The distinction exists because CI branches on it. An annotation that fires +/// on every run — and one fires on every run if "routing settled before we +/// measured" counts as something needing review — trains its readers to ignore +/// it, which is worse than not having it. Context still reaches the report; it +/// just does not claim a human has to act. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NoteLevel { + /// Something a human should look at before trusting the run. + Attention, + /// Something a human needs in order to read the run, but which is not + /// itself a problem. + Context, +} + +/// One operator-facing line, and which of the two lists it belongs in. +/// +/// Scenarios build these as they go and hand the whole batch to +/// [`ChaosSummary::absorb`] at the end, so the classification lives next to the +/// condition that produced it rather than in whatever reads the result later. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Note { + pub level: NoteLevel, + pub message: String, +} + +impl Note { + /// A line that means a human should look before trusting the run. + pub fn attention(message: impl Into) -> Self { + Self { + level: NoteLevel::Attention, + message: message.into(), + } + } + + /// A line that a human needs in order to read the run, but which is not + /// itself a problem. + pub fn context(message: impl Into) -> Self { + Self { + level: NoteLevel::Context, + message: message.into(), + } + } + + /// Picks the level from a condition, for the common case where the same + /// sentence is a finding or context depending on the numbers in it. + pub fn leveled(needs_attention: bool, message: impl Into) -> Self { + if needs_attention { + Self::attention(message) + } else { + Self::context(message) + } + } +} + /// Everything the driver reports for a scenario. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -503,6 +566,51 @@ pub struct ChaosSummary { /// read as "checked, nothing found". #[serde(skip_serializing_if = "Option::is_none")] pub exactly_once: Option, + /// The scheduled-fire account, for scenarios that pair scheduled actions + /// against the registrations that made them. Absent for scenarios that do + /// not, rather than an empty report that would read as "checked, nothing + /// found". + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule_fires: Option, + /// The promise-wakeup account, for scenarios that pair completions against + /// the waiters they were supposed to resume. Absent for scenarios that do + /// not, for the same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub promise_wakeups: Option, + /// The reachability account, for scenarios that cut one executor off from + /// the tier that routes to it. Absent for scenarios that do not, for the + /// same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reachability: Option, + /// The truncation account, for scenarios that revert agent state. Absent + /// for scenarios that do not, for the same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub truncation: Option, + /// The resurrection account, for scenarios that delete agents. Absent for + /// scenarios that do not, for the same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resurrection: Option, + /// The rollback account, for scenarios that move agents between builds and + /// back. Absent for scenarios that do not. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rollback: Option, + /// The storage-fault account, for scenarios that break a storage + /// dependency underneath every executor at once — by taking it away or by + /// slowing it down. Absent for scenarios that do not, for the same reason + /// as `scheduleFires`. + /// + /// Serialised as `storageOutage` rather than under the field's own name. + /// The wire name predates the scenarios that degrade a store rather than + /// removing one, and every archived result and the report generator that + /// reads them use it. Renaming the field would cost a schema bump and + /// silently stop rendering the runs already in the bucket, which is a worse + /// outcome than one name that has outlived its accuracy. + #[serde( + rename = "storageOutage", + default, + skip_serializing_if = "Option::is_none" + )] + pub storage_fault: Option, /// Shard-ownership samples, in the order they were taken. Empty for /// scenarios that do not sample executor assignments. /// @@ -512,8 +620,19 @@ pub struct ChaosSummary { /// it interpretable. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub ownership: Vec, - /// The read-back verdicts that need a human, hoisted for scanning. + /// The verdicts and findings that need a human, hoisted for scanning. + /// + /// CI raises an annotation when this is non-empty, so nothing belongs here + /// that is true of a healthy run. Context goes in [`Self::notes`]. pub attention: Vec, + /// Lines a human needs in order to read the run, which are not themselves + /// problems: how the routing table looked before measuring, how much of the + /// mechanism under test the fault actually landed in, and so on. + /// + /// Kept out of [`Self::attention`] so that list keeps meaning "look at + /// this". Absent from older results, hence `default`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub notes: Vec, } impl ChaosSummary { @@ -621,8 +740,26 @@ impl ChaosSummary { .collect(), routing_snapshots, exactly_once: None, + schedule_fires: None, + promise_wakeups: None, + reachability: None, + truncation: None, + resurrection: None, + rollback: None, + storage_fault: None, ownership: Vec::new(), attention, + notes: Vec::new(), + } + } + + /// Files a batch of scenario notes into the two lists by their level. + pub fn absorb(&mut self, notes: impl IntoIterator) { + for note in notes { + match note.level { + NoteLevel::Attention => self.attention.push(note.message), + NoteLevel::Context => self.notes.push(note.message), + } } } @@ -640,6 +777,103 @@ impl ChaosSummary { self } + /// Attaches the scheduled-fire account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// More than the findings, unlike [`Self::with_exactly_once`]: an + /// unreadable target or a truncated fire log weakens every verdict the + /// report makes, and that has to be visible next to the verdicts rather + /// than only in the numbers underneath them. + pub fn with_schedule_fires(mut self, report: ScheduleFireReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.schedule_fires = Some(report); + self + } + + /// Attaches the promise-wakeup account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_schedule_fires`], and one extra reason for it: + /// a waiter that could not be read is normally the weakest outcome there is, + /// but for a suspended waiter it can be the strongest evidence in the run. + /// The report decides which, and only what it classifies as attention lands + /// there. + pub fn with_promise_wakeups(mut self, report: WakeupReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.promise_wakeups = Some(report); + self + } + + /// Attaches the reachability account and hoists everything it wants a human + /// to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_schedule_fires`]. The one line worth calling + /// out is the inconclusive case: a partition that never cut the executor off + /// produces a report full of healthy numbers, and that has to read as + /// "this run tested nothing" rather than as a pass. + pub fn with_reachability(mut self, report: ReachabilityReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.reachability = Some(report); + self + } + + /// Attaches the truncation account and hoists everything it wants a human + /// to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_schedule_fires`]. The line worth calling out + /// here is the inconclusive one: a kill that caught no revert in flight + /// proves nothing about crashing during a revert, and every clean number + /// underneath it describes reverts that completed either side of the fault. + pub fn with_truncation(mut self, report: TruncationReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.truncation = Some(report); + self + } + + /// Attaches the resurrection account and hoists everything it wants a human + /// to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_truncation`], including the inconclusive line: + /// a kill that caught no delete in flight proves nothing about crashing + /// during a deletion. + pub fn with_resurrection(mut self, report: ResurrectionReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.resurrection = Some(report); + self + } + + /// Attaches the rollback account and hoists everything it wants a human to + /// see into [`Self::attention`]. + /// + /// Same split as the others. The line worth calling out is the forward-leg + /// one: a rollback of agents that never left the old build proves nothing, + /// and that has to read as inconclusive rather than as a pass. + pub fn with_rollback(mut self, report: RollbackReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.rollback = Some(report); + self + } + + /// Attaches the storage-outage account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// Same split as the others. The line worth calling out is the + /// outage-not-observed one: a partition that failed to take hold leaves + /// every cell underneath it describing an undisturbed cluster, and that has + /// to read as "this run tested nothing" rather than as a pass. + pub fn with_storage_fault(mut self, report: StorageFaultReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.storage_fault = Some(report); + self + } + /// Attaches the shard-ownership samples and hoists their findings into /// [`Self::attention`]. /// @@ -688,6 +922,30 @@ pub enum TerminationReason { /// owners is an agent whose state can fork, and there is no instant at /// which that is legitimate. ShardOwnershipViolated { findings: u64, first: String }, + /// An agent the platform said it had deleted came back with its state, or a + /// deletion landed somewhere other than the two answers it was allowed. + /// Asserted for the same reason as `RevertTruncationViolated`: invoking a + /// deleted id creates a new agent, so there are exactly two legal values and + /// no band of doubt. See [`crate::chaos::resurrection`]. + AgentResurrected { findings: u64, first: String }, + /// A revert landed somewhere other than the two values it was allowed to. + /// Asserted rather than reported, and the only read-back in the suite that + /// earns that: the driver knows the counter's value before the revert and + /// exactly how many invocations it asked to take back, so there is no band + /// of doubt around the answer. See [`crate::chaos::truncation`]. + RevertTruncationViolated { findings: u64, first: String }, + /// A scheduled action the platform accepted never fired, fired twice, or + /// fired after being refused. Asserted rather than reported: unlike a + /// count-based read-back, each of these is a statement about one named + /// action paired with one named registration, with no band of doubt around + /// it. See [`crate::chaos::fires`]. + ScheduledFireViolated { findings: u64, first: String }, + /// A promise completion the platform accepted never woke its waiter, woke it + /// twice, or woke it after being refused. Asserted for the same reason as + /// [`Self::ScheduledFireViolated`]: each is a statement about one named + /// completion paired with one named waiter, with no band of doubt around it. + /// See [`crate::chaos::wakeups`]. + PromiseWakeupViolated { findings: u64, first: String }, /// An agent's durable state did not survive a component update. Asserted /// because an update is supposed to change what an agent runs and nothing /// about what it remembers — state that moved is the one outcome an update @@ -746,6 +1004,40 @@ mod tests { Utc.timestamp_opt(1_800_000_000 + secs, 0).unwrap() } + /// The field is `storage_fault` in Rust and `storageOutage` on disk, and + /// that mismatch is deliberate rather than an oversight. + /// + /// The wire name predates the scenarios that slow a store down instead of + /// removing one. Every result already in the bucket uses it, and so does + /// the report generator that renders them. Renaming it would stop those + /// runs rendering to buy nothing, so the `#[serde(rename)]` stays and this + /// test is what stops a later tidy-up from quietly dropping it. + #[test] + fn the_storage_fault_account_still_serialises_under_its_original_name() { + let summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), None).with_storage_fault( + StorageFaultReport::build( + &[], + None, + "db.example", + crate::chaos::OutageExpectation::WholeWorkload { + quiet_floor_percent: 50.0, + }, + std::time::Duration::from_secs(120), + ), + ); + + let json = serde_json::to_value(&summary).unwrap(); + assert!( + json.get("storageOutage").is_some(), + "the on-disk name must not drift, got keys: {:?}", + json.as_object().map(|o| o.keys().collect::>()) + ); + assert!( + json.get("storageFault").is_none(), + "renaming this field silently orphans every archived result" + ); + } + fn op(op_id: u64, stream: Stream, phase: Phase, outcome: Outcome) -> OperationRecord { OperationRecord { op_id, @@ -960,12 +1252,31 @@ mod tests { /// A reader must never have to wonder whether a stream was skipped or just /// had nothing to say. + /// + /// The waiter stream is here for a different reason from the other two, and + /// the distinction is worth keeping straight: those two have no durable + /// count to read, while this one has a count that is *weaker* than what the + /// scenario already does with it. Its absence from the count-based read-back + /// means the token pairing in `promiseWakeups` is the account, not that + /// nothing was checked. #[test] fn streams_without_readback_are_named_rather_than_omitted() { let summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), None); assert_eq!( summary.streams_without_readback, - vec![Stream::Ephemeral, Stream::Promise] + // `Revert` is here for a different reason from the other three. + // Those keep no comparable durable state; a revert agent does, but + // some of its acknowledged work was deliberately taken back, so a + // generic counter comparison would report every reverted increment + // as lost. `crate::chaos::truncation` judges those agents exactly + // instead, which is strictly stronger. + vec![ + Stream::Ephemeral, + Stream::Promise, + Stream::PromiseWait, + Stream::Revert, + Stream::Delete + ] ); } @@ -1061,4 +1372,64 @@ mod tests { assert_eq!(stats.count, 0); assert_eq!(stats.max_ms, 0); } + + fn empty_summary() -> ChaosSummary { + ChaosSummary::build(&[], Vec::new(), Vec::new(), None) + } + + #[test] + fn notes_are_filed_by_level_rather_than_all_into_attention() { + let mut summary = empty_summary(); + summary.absorb([ + Note::context("routing at start: 1024/1024 shards (settled before measuring)"), + Note::attention("WARNING: measured against an unsettled cluster"), + ]); + + assert_eq!(summary.attention.len(), 1); + assert!(summary.attention[0].contains("unsettled")); + assert_eq!(summary.notes.len(), 1); + assert!(summary.notes[0].contains("settled before measuring")); + } + + /// The reason the split exists. CI raises an annotation when `attention` is + /// non-empty, so a clean run has to leave it empty — otherwise the + /// annotation fires every time and stops meaning anything. + #[test] + fn a_run_with_only_context_raises_nothing_for_ci() { + let mut summary = empty_summary(); + summary.absorb([ + Note::context("routing at start: 1024/1024 shards (settled before measuring)"), + Note::context("S10 killed the executor with 353 actions pending"), + ]); + + assert!(summary.attention.is_empty()); + assert_eq!(summary.notes.len(), 2); + } + + #[test] + fn leveled_picks_the_list_from_the_condition() { + assert_eq!(Note::leveled(true, "x").level, NoteLevel::Attention); + assert_eq!(Note::leveled(false, "x").level, NoteLevel::Context); + } + + /// Older results have no `notes` key at all, and must still deserialise. + #[test] + fn a_result_written_before_notes_existed_still_reads() { + let mut summary = empty_summary(); + summary.absorb([Note::context("context")]); + let mut json: serde_json::Value = serde_json::to_value(&summary).unwrap(); + assert!(json.get("notes").is_some(), "notes are serialised when set"); + + json.as_object_mut().unwrap().remove("notes"); + let back: ChaosSummary = serde_json::from_value(json).unwrap(); + assert!(back.notes.is_empty()); + } + + /// An empty `notes` is omitted rather than written as `[]`, matching how + /// every other optional block in this result behaves. + #[test] + fn an_empty_notes_list_is_not_serialised() { + let json = serde_json::to_value(empty_summary()).unwrap(); + assert!(json.get("notes").is_none()); + } } diff --git a/integration-tests/src/chaos/truncation.rs b/integration-tests/src/chaos/truncation.rs new file mode 100644 index 0000000000..d0e39c28dc --- /dev/null +++ b/integration-tests/src/chaos/truncation.rs @@ -0,0 +1,731 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Did every revert land on a boundary, or did one tear (GOL-371)? +//! +//! The sharpest oracle in the suite, because it has no band of doubt in it. +//! Every other read-back compares a counter against a *range* — the width of +//! the range is the operations whose fate the driver could not determine. Here +//! the driver knows the counter's value immediately before the revert, because +//! the last increment of the round returned it, and it knows exactly how many +//! invocations the revert was asked to take back. So afterwards there are +//! exactly **two** legal values and nothing in between: +//! +//! * `V` — the revert never committed +//! * `V - N` — the revert committed +//! +//! Anything else is a defect, and which kind it is says what went wrong. A +//! value strictly between the two is a truncation that tore. A value below +//! `V - N` took back more than it was asked for. A value above `V` means state +//! grew across a revert. +//! +//! ### Why a partial truncation should be impossible, and why that is worth +//! testing anyway +//! +//! Reading `Worker::revert` in the executor: `RevertLastInvocations` walks back +//! to the nth `AgentInvocationStarted` entry and then commits **one** +//! `OplogEntry::revert` marking the region deleted. One entry cannot tear, so +//! the truncation itself is atomic by construction. +//! +//! The window worth killing into is the one *around* it. Reverting takes +//! `lock_stopped_worker`, so the worker is stopped first; then the entry is +//! committed; then `reattach_worker_status` runs, because — in the executor's +//! own words — "this commit will detach the worker status, immediately reattach +//! it so we see the up to date status". An executor that dies between the +//! commit and the reattach has left durable state changed and in-memory state +//! stale, which is the same shape as the S11 promise defect: the durable half +//! landed and the half that tells anyone about it did not. +//! +//! So the two findings this account expects to be able to make, if the platform +//! has a bug here, are [`TruncationViolation::AcknowledgedButNotApplied`] and +//! its opposite — not a torn counter. + +use crate::chaos::history::Outcome; +use crate::chaos::reverts::RevertRound; +use crate::chaos::split::{FaultWindow, Group, PodSplit, Window}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// The most findings the report carries. +const MAX_FINDINGS: usize = 50; + +/// What went wrong with one revert. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum TruncationViolation { + /// The counter landed strictly between the pre-revert and post-revert + /// values: some of the invocations were taken back and some were not. + PartialTruncation, + /// More was taken back than the revert was asked for. + OverTruncation, + /// The counter is higher after the revert than it was before it. + Divergent, + /// The platform confirmed the revert and the state never moved. The same + /// shape as an accepted promise completion that never woke its waiter. + AcknowledgedButNotApplied, + /// The platform refused the revert and the state moved anyway. + RefusedButApplied, +} + +impl TruncationViolation { + pub fn as_str(self) -> &'static str { + match self { + TruncationViolation::PartialTruncation => "partial-truncation", + TruncationViolation::OverTruncation => "over-truncation", + TruncationViolation::Divergent => "divergent", + TruncationViolation::AcknowledgedButNotApplied => "acknowledged-but-not-applied", + TruncationViolation::RefusedButApplied => "refused-but-applied", + } + } +} + +/// One violation, against one named round. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TruncationFinding { + pub violation: TruncationViolation, + pub agent: String, + pub round: u32, + pub window: Window, + /// The counter before the revert, what it should have become, and what it + /// actually became. Carried so a finding can be read without the history. + pub before: u64, + pub expected_after_commit: u64, + pub observed: u64, + pub detail: String, +} + +/// Rounds and their verdicts for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TruncationCell { + pub group: Group, + pub window: Window, + pub rounds: u64, + /// Reverts the platform confirmed that landed on the post-revert value. + pub applied: u64, + /// Reverts that left the agent exactly where it was. Legitimate only when + /// the platform never confirmed them. + pub not_applied: u64, + pub violations: u64, + /// Rounds the driver cannot judge: an increment that did not answer, so the + /// pre-revert value is unknown. + pub unjudgeable: u64, + /// Rounds no following increment ever probed. + pub unprobed: u64, +} + +/// Reverts the kill landed in the middle of. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RevertsCaught { + pub group: Group, + /// Reverts submitted before the kill that had not answered when it landed. + /// The population the scenario is actually about: a run that caught none of + /// them proves nothing about crash-during-revert, however clean it looks. + pub reverts: u64, + pub agents: usize, + pub confirmed: u64, + pub indeterminate: u64, + pub rejected: u64, +} + +/// The truncation account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TruncationReport { + /// What each round was configured to do, so an archived finding can be read + /// without the suite YAML to hand. + pub increments_per_round: u32, + pub revert_invocations: u32, + pub rounds_recorded: u64, + pub reverts_confirmed: u64, + pub reverts_indeterminate: u64, + pub reverts_rejected: u64, + /// Confirmed reverts that landed exactly on the post-revert value. + pub applied_exactly: u64, + /// Reverts the driver never heard back about that applied anyway — doubt + /// the platform resolved in its own favour, not a defect. + pub indeterminate_that_applied: u64, + /// Reverts the driver never heard back about that did not apply. Also not a + /// defect: the call may never have landed. + pub indeterminate_that_did_not: u64, + pub unjudgeable: u64, + pub unprobed: u64, + pub cells: Vec, + pub caught_by_the_kill: Vec, + pub findings: Vec, + pub findings_omitted: u64, +} + +/// One round's verdict. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Verdict { + Applied, + NotApplied, + Violation(TruncationViolation), + Unjudgeable, + Unprobed, +} + +/// Judges one round against the two values it is allowed to have landed on. +fn judge(round: &RevertRound) -> Verdict { + let Some(before) = round.before_revert else { + return Verdict::Unjudgeable; + }; + let Some(observed) = round.observed_after else { + return Verdict::Unprobed; + }; + let expected = before.saturating_sub(round.asked_to_revert as u64); + + if observed == expected { + // Applied. Legitimate unless the platform said it refused. + if round.outcome == Outcome::Rejected { + return Verdict::Violation(TruncationViolation::RefusedButApplied); + } + return Verdict::Applied; + } + if observed == before { + // Not applied. Legitimate unless the platform said it had been. + if round.outcome == Outcome::Confirmed { + return Verdict::Violation(TruncationViolation::AcknowledgedButNotApplied); + } + return Verdict::NotApplied; + } + if observed > before { + return Verdict::Violation(TruncationViolation::Divergent); + } + if observed < expected { + return Verdict::Violation(TruncationViolation::OverTruncation); + } + Verdict::Violation(TruncationViolation::PartialTruncation) +} + +impl TruncationReport { + /// Builds the account from the rounds the workload recorded. + pub fn build( + rounds: &[RevertRound], + split: &PodSplit, + fault: Option, + increments_per_round: u32, + revert_invocations: u32, + ) -> Self { + let mut cells: BTreeMap<(Group, Window), TruncationCell> = BTreeMap::new(); + let mut findings: Vec = Vec::new(); + let mut report = TruncationReport { + increments_per_round, + revert_invocations, + rounds_recorded: rounds.len() as u64, + reverts_confirmed: 0, + reverts_indeterminate: 0, + reverts_rejected: 0, + applied_exactly: 0, + indeterminate_that_applied: 0, + indeterminate_that_did_not: 0, + unjudgeable: 0, + unprobed: 0, + cells: Vec::new(), + caught_by_the_kill: Vec::new(), + findings: Vec::new(), + findings_omitted: 0, + }; + + for round in rounds { + let group = split.group_of(&round.agent).unwrap_or(Group::Elsewhere); + let window = Window::of(round.submitted_at, fault); + let cell = cells + .entry((group, window)) + .or_insert_with(|| TruncationCell { + group, + window, + rounds: 0, + applied: 0, + not_applied: 0, + violations: 0, + unjudgeable: 0, + unprobed: 0, + }); + cell.rounds += 1; + + match round.outcome { + Outcome::Confirmed => report.reverts_confirmed += 1, + Outcome::Indeterminate => report.reverts_indeterminate += 1, + Outcome::Rejected => report.reverts_rejected += 1, + } + + match judge(round) { + Verdict::Applied => { + cell.applied += 1; + if round.outcome == Outcome::Confirmed { + report.applied_exactly += 1; + } else { + report.indeterminate_that_applied += 1; + } + } + Verdict::NotApplied => { + cell.not_applied += 1; + if round.outcome != Outcome::Confirmed { + report.indeterminate_that_did_not += 1; + } + } + Verdict::Unjudgeable => { + cell.unjudgeable += 1; + report.unjudgeable += 1; + } + Verdict::Unprobed => { + cell.unprobed += 1; + report.unprobed += 1; + } + Verdict::Violation(violation) => { + cell.violations += 1; + let before = round.before_revert.unwrap_or_default(); + let observed = round.observed_after.unwrap_or_default(); + let expected = before.saturating_sub(round.asked_to_revert as u64); + findings.push(TruncationFinding { + violation, + agent: round.agent.clone(), + round: round.round, + window, + before, + expected_after_commit: expected, + observed, + detail: detail_for(violation, round, before, expected, observed), + }); + } + } + } + + // ── What the kill landed in the middle of ─────────────────────────── + if let Some(window) = fault { + let mut by_group: BTreeMap> = BTreeMap::new(); + for round in rounds { + let Some(group) = split.group_of(&round.agent) else { + continue; + }; + let unresolved = round.completed_at.is_none_or(|at| at >= window.injected_at); + if round.submitted_at < window.injected_at && unresolved { + by_group.entry(group).or_default().push(round); + } + } + for (group, caught) in by_group { + let agents: std::collections::BTreeSet<&str> = + caught.iter().map(|r| r.agent.as_str()).collect(); + report.caught_by_the_kill.push(RevertsCaught { + group, + reverts: caught.len() as u64, + agents: agents.len(), + confirmed: count(&caught, Outcome::Confirmed), + indeterminate: count(&caught, Outcome::Indeterminate), + rejected: count(&caught, Outcome::Rejected), + }); + } + } + + report.cells = cells.into_values().collect(); + report.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + report.findings = findings; + report + } + + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() || self.findings_omitted > 0 + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines: Vec = self + .findings + .iter() + .map(|f| { + format!( + "S7 {}: {} round {} — {}", + f.violation.as_str(), + f.agent, + f.round, + f.detail + ) + }) + .collect(); + if self.findings_omitted > 0 { + lines.push(format!( + "S7: {} further truncation finding(s) were dropped from the report", + self.findings_omitted + )); + } + + // The S10 lesson: a run that caught none of the mechanism proves + // nothing about it, however clean every other number looks. + let caught: u64 = self + .caught_by_the_kill + .iter() + .filter(|c| c.group == Group::OnPod) + .map(|c| c.reverts) + .sum(); + if caught == 0 { + lines.push( + "S7: the kill caught no revert in flight on the targeted executor, so this run \ + says nothing about crashing during a revert. Every verdict below describes \ + reverts that completed either side of it." + .to_string(), + ); + } + lines + } + + /// Lines a reader needs in order to interpret the run. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S7: {} rounds of {} increments then a revert of {}; {} confirmed, {} in doubt, {} \ + refused", + self.rounds_recorded, + self.increments_per_round, + self.revert_invocations, + self.reverts_confirmed, + self.reverts_indeterminate, + self.reverts_rejected + )]; + lines.push(format!( + "S7: {} confirmed reverts landed exactly on the post-revert value; {} in doubt \ + applied anyway, {} in doubt did not", + self.applied_exactly, self.indeterminate_that_applied, self.indeterminate_that_did_not + )); + if self.unjudgeable > 0 || self.unprobed > 0 { + lines.push(format!( + "S7: {} rounds could not be judged (an increment never answered) and {} were \ + never probed by a following increment", + self.unjudgeable, self.unprobed + )); + } + for caught in &self.caught_by_the_kill { + lines.push(format!( + "S7 {}: {} reverts across {} agents were unresolved when the kill landed — {} \ + confirmed, {} in doubt, {} refused", + caught.group.as_str(), + caught.reverts, + caught.agents, + caught.confirmed, + caught.indeterminate, + caught.rejected + )); + } + for cell in &self.cells { + lines.push(format!( + "S7 {} {}: {} rounds, {} applied, {} not applied, {} violations, {} unjudgeable, \ + {} unprobed", + cell.group.as_str(), + cell.window.as_str(), + cell.rounds, + cell.applied, + cell.not_applied, + cell.violations, + cell.unjudgeable, + cell.unprobed + )); + } + lines + } +} + +fn count(rounds: &[&RevertRound], outcome: Outcome) -> u64 { + rounds.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +fn detail_for( + violation: TruncationViolation, + round: &RevertRound, + before: u64, + expected: u64, + observed: u64, +) -> String { + let asked = round.asked_to_revert; + match violation { + TruncationViolation::PartialTruncation => format!( + "the agent was worth {before}, a revert of {asked} invocations should have left it \ + at {expected}, and it came back at {observed} — between the two, so part of the \ + truncation landed and part did not" + ), + TruncationViolation::OverTruncation => format!( + "the agent was worth {before} and a revert of {asked} invocations left it at \ + {observed}, below the {expected} it asked for: more was taken back than requested" + ), + TruncationViolation::Divergent => format!( + "the agent was worth {before} before a revert of {asked} invocations and came back \ + at {observed}, higher than it started: state grew across a revert" + ), + TruncationViolation::AcknowledgedButNotApplied => format!( + "the platform confirmed a revert of {asked} invocations and the agent is still \ + worth {before}, not the {expected} it acknowledged" + ), + TruncationViolation::RefusedButApplied => format!( + "the platform refused a revert of {asked} invocations and the agent moved from \ + {before} to {observed} anyway" + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{DateTime, TimeDelta, Utc}; + use std::collections::BTreeMap; + use test_r::test; + + const ON_POD: &str = "chaos-s7-revert-0000"; + const CONTROL: &str = "chaos-s7-revert-0001"; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-24T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![ON_POD.to_string()], + elsewhere: vec![CONTROL.to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(120)), + } + } + + /// One round: the agent was worth `before`, asked for 2 invocations back, + /// the platform answered `outcome`, and afterwards it read `observed`. + fn round( + agent: &str, + offset_secs: i64, + before: Option, + outcome: Outcome, + observed: Option, + ) -> RevertRound { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + RevertRound { + agent: agent.to_string(), + round: 0, + before_revert: before, + asked_to_revert: 2, + outcome, + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(80)), + observed_after: observed, + } + } + + fn build(rounds: &[RevertRound]) -> TruncationReport { + TruncationReport::build(rounds, &split(), Some(fault()), 4, 2) + } + + fn violations(report: &TruncationReport) -> Vec { + report.findings.iter().map(|f| f.violation).collect() + } + + /// The healthy shape: the platform said yes and the agent landed exactly on + /// the post-revert value. + #[test] + fn a_revert_that_landed_where_it_said_it_would_is_not_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(8))]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.applied_exactly, 1); + assert!(!report.has_violations()); + } + + /// The headline finding this scenario exists to make: the counter landed + /// between the two values it was allowed to have. + #[test] + fn a_counter_between_the_two_legal_values_is_a_torn_truncation() { + // Worth 10, asked for 4 back so 6 was legal, came back at 8. + let mut r = round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(8)); + r.asked_to_revert = 4; + let report = build(&[r]); + + assert_eq!( + violations(&report), + vec![TruncationViolation::PartialTruncation] + ); + let finding = &report.findings[0]; + assert_eq!(finding.before, 10); + assert_eq!(finding.expected_after_commit, 6); + assert_eq!(finding.observed, 8); + assert!(report.has_violations(), "this must be able to fail the run"); + } + + /// The same shape as an accepted promise completion that never woke its + /// waiter: the durable half was acknowledged and nothing moved. + #[test] + fn a_confirmed_revert_that_changed_nothing_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(10))]); + assert_eq!( + violations(&report), + vec![TruncationViolation::AcknowledgedButNotApplied] + ); + } + + /// Its opposite, and just as serious: refused, and applied regardless. + #[test] + fn a_refused_revert_that_applied_anyway_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Rejected, Some(8))]); + assert_eq!( + violations(&report), + vec![TruncationViolation::RefusedButApplied] + ); + } + + #[test] + fn taking_back_more_than_was_asked_for_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(5))]); + assert_eq!( + violations(&report), + vec![TruncationViolation::OverTruncation] + ); + } + + #[test] + fn state_growing_across_a_revert_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(11))]); + assert_eq!(violations(&report), vec![TruncationViolation::Divergent]); + } + + /// A revert the driver never heard back about is doubt, not damage. Both + /// answers are legitimate and neither is a finding. + #[test] + fn a_revert_in_doubt_may_land_either_way_without_being_a_finding() { + let report = build(&[ + round(ON_POD, -10, Some(10), Outcome::Indeterminate, Some(8)), + round(CONTROL, -10, Some(10), Outcome::Indeterminate, Some(10)), + ]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.indeterminate_that_applied, 1); + assert_eq!(report.indeterminate_that_did_not, 1); + } + + /// A round whose increments never answered has no pre-revert value, so it + /// cannot be judged. Counted, never guessed at. + #[test] + fn a_round_whose_increments_never_answered_is_not_judged() { + let report = build(&[round(ON_POD, -10, None, Outcome::Confirmed, Some(8))]); + assert_eq!(report.unjudgeable, 1); + assert!(report.findings.is_empty()); + } + + /// The last round an agent ran has nothing after it to probe with. It is + /// counted rather than assumed clean. + #[test] + fn a_round_no_increment_ever_probed_is_counted_as_unprobed() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, None)]); + assert_eq!(report.unprobed, 1); + assert!(report.findings.is_empty()); + } + + /// The S10 lesson: a kill that caught none of the mechanism proves nothing + /// about it, and the run has to say so rather than read as clean. + #[test] + fn a_kill_that_caught_no_revert_says_the_run_proved_nothing() { + // Both rounds resolved well before the kill. + let report = build(&[ + round(ON_POD, -100, Some(10), Outcome::Confirmed, Some(8)), + round(CONTROL, -100, Some(10), Outcome::Confirmed, Some(8)), + ]); + assert!(report.caught_by_the_kill.is_empty()); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("caught no revert in flight")), + "attention was {:?}", + report.attention_lines() + ); + } + + /// A revert still unresolved when the pod died is the population the whole + /// scenario is about, and it has to be counted apart from the rest. + #[test] + fn reverts_unresolved_when_the_pod_died_are_reported_separately() { + let mut caught = round(ON_POD, -1, Some(10), Outcome::Indeterminate, Some(10)); + caught.completed_at = Some(t0() + TimeDelta::seconds(30)); + let report = build(&[caught]); + + let entry = report + .caught_by_the_kill + .iter() + .find(|c| c.group == Group::OnPod) + .expect("the kill caught a revert"); + assert_eq!(entry.reverts, 1); + assert_eq!(entry.agents, 1); + assert_eq!(entry.indeterminate, 1); + assert!( + !report + .attention_lines() + .iter() + .any(|l| l.contains("caught no revert in flight")) + ); + } + + /// Rounds are split by which executor owned the agent and which side of the + /// kill they fell on, so a control group cannot hide a disturbed one. + #[test] + fn rounds_are_split_by_group_and_window() { + let report = build(&[ + round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(8)), + round(ON_POD, 10, Some(12), Outcome::Confirmed, Some(10)), + round(CONTROL, 10, Some(10), Outcome::Confirmed, Some(8)), + ]); + let cell = |g, w| { + report + .cells + .iter() + .find(|c| c.group == g && c.window == w) + .cloned() + }; + assert_eq!(cell(Group::OnPod, Window::BeforeFault).unwrap().rounds, 1); + assert_eq!(cell(Group::OnPod, Window::DuringFault).unwrap().rounds, 1); + assert_eq!( + cell(Group::Elsewhere, Window::DuringFault).unwrap().rounds, + 1 + ); + } + + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let rounds: Vec = (0..MAX_FINDINGS + 7) + .map(|_| round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(10))) + .collect(); + let report = build(&rounds); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 7); + assert!(report.has_violations()); + } + + /// A finding is read by an operator mid-window, so it has to state the + /// three numbers that make it interpretable without the history to hand. + #[test] + fn a_finding_states_the_numbers_that_make_it_readable() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(10))]); + let detail = &report.findings[0].detail; + assert!(detail.contains("10"), "{detail}"); + assert!(detail.contains('8'), "{detail}"); + assert!( + !detail.contains("Some("), + "Option formatting leaked: {detail}" + ); + } +} diff --git a/integration-tests/src/chaos/waiters.rs b/integration-tests/src/chaos/waiters.rs new file mode 100644 index 0000000000..61aecaca1e --- /dev/null +++ b/integration-tests/src/chaos/waiters.rs @@ -0,0 +1,697 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The suspended-waiter workload (GOL-377). +//! +//! Every other stream in this suite keeps an agent *busy*. This one keeps a pool +//! of them asleep. Each waiter repeats one round: +//! +//! 1. `arm(token)` creates a promise and returns it. Fast, and the only part the +//! driver needs an answer from. +//! 2. `wait(token, promise)` parks the agent on that promise. The invocation +//! stays open — on the platform as a suspended worker, in the driver as a +//! task holding a connection. +//! 3. After `dwellMillis`, the driver completes the promise from the outside. +//! 4. The wakeup ends the round, and the waiter starts the next one. +//! +//! ## Why one promise per waiter +//! +//! An agent runs its invocations one at a time, so a waiter parked in `wait` +//! cannot be armed again until it wakes. That is not a limitation to work +//! around; it is what makes the pool size mean something exact. With `waiters` +//! agents and one promise each, the number of agents standing suspended at the +//! instant the pod dies is a known constant rather than a sample — the same +//! property [`crate::chaos::pinned`] gets from one in-flight operation per +//! agent, for the same reason. +//! +//! It also makes a lost wakeup visible while the run is still going. A waiter +//! whose completion never arrives can never start another round, so it simply +//! stops producing. See [`WaiterHandle::stalled`]. +//! +//! ## Why the dwell +//! +//! `dwellMillis` decides how the population divides at the moment of the kill. +//! Every waiter is either parked-and-waiting (the dwell) or being-woken (the +//! completion round trip), and the dwell is far the longer of the two — so the +//! kill lands mostly in the first, which is the state this scenario is about, +//! and occasionally in the second, which is the narrower race S8 already covers +//! for ordinary invocations. +//! +//! The dwell also has to comfortably exceed the workflow's inject-and-verify +//! path — signal poll, `kubectl apply`, waiting for `AllInjected` — or every +//! promise armed before the kill would already have been completed by the time +//! the pod died, and the run would measure nothing. +//! +//! ## Why completions are retried like everything else +//! +//! The suite's retry policy — one same-key retry, transport failures only — +//! applies here too, and a completion is the one operation where that could +//! plausibly be accused of hiding the defect: `complete` writes with +//! `set_if_not_exists` and re-triggers the wakeup, so a retry can repair a +//! completion whose wakeup was lost. +//! +//! It cannot hide what this scenario asks, because the question is scoped to +//! completions the platform *confirmed*. A retry only happens when the previous +//! attempt did not return success, so a confirmed completion is always a single +//! accepted call. "The platform said yes and the waiter never woke" is exactly +//! as detectable with the retry as without it, and the retry keeps the workload +//! behaving like a client anyone would actually write. + +use crate::chaos::PromiseConfig; +use crate::chaos::history::{Stream, WaiterWakeupLog, WakeupRecord}; +use crate::chaos::workload::{self, WorkloadContext}; +use chrono::{DateTime, TimeZone, Utc}; +use golem_common::base_model::agent::ParsedAgentId; +use golem_common::model::PromiseId; +use golem_common::{agent_id, data_value}; +use golem_test_framework::dsl::TestDsl; +use golem_wasm::{FromValue, ValueAndType}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; +use tokio::task::JoinSet; +use tracing::{info, warn}; + +/// Agent type exported by the promise component for this workload. +pub const PROMISE_WAITER_AGENT: &str = "PromiseWaiter"; + +/// Payload a completion carries. Nothing reads it; the token is what identifies +/// a round, and it travels through the agent rather than through the promise. +const COMPLETION_PAYLOAD: &[u8] = b"chaos-s11"; + +/// How long a waiter's loop waits for its own wakeup before concluding the +/// waiter is parked for good and standing down. +/// +/// Deliberately a multiple of the wakeup budget rather than a fixed number: the +/// budget is already the run's statement about what a recovery may cost, and +/// anything that treats "slow" as "lost" would turn a bad p99 into a false +/// finding. Standing down is not a verdict either — the read-back still asks the +/// waiter what happened, and a waiter that woke late says so. +const STALL_MULTIPLE: u32 = 4; + +/// Floor under [`STALL_MULTIPLE`], for a configuration with a very short budget. +const MIN_STALL_TIMEOUT: Duration = Duration::from_secs(180); + +/// How many waiters are read back at once. Same reasoning as +/// [`crate::chaos::scenarios::read_back_agents`]: reads do not mutate, and +/// walking a few hundred of them one at a time behind a per-read ceiling +/// outlasts the maintenance window. +const READ_CONCURRENCY: usize = 16; + +/// Ceiling on one wakeup-log read. Generous, because it happens on a cluster +/// that has just been through a fault, and because the answer carries every +/// wakeup the waiter recorded rather than one number. +const READ_TIMEOUT: Duration = Duration::from_secs(60); + +/// How long the smoke round waits for its one wakeup before giving up. +/// +/// Generous against a cluster that is merely cold, short enough that a broken +/// completion path costs seconds rather than a baseline. +const SMOKE_WAKE_TIMEOUT: Duration = Duration::from_secs(60); + +/// How often the smoke round asks whether its waiter came back. +const SMOKE_POLL_INTERVAL: Duration = Duration::from_secs(2); + +/// A running waiter workload. +/// +/// As elsewhere, dropping the handle does not stop it: call +/// [`WaiterHandle::stop`] so rounds in flight record themselves instead of being +/// cancelled mid-completion. +pub struct WaiterHandle { + tasks: JoinSet<()>, + running: Arc, + rounds: Arc, + stalled: Arc, +} + +impl WaiterHandle { + /// Rounds started across all waiters. + pub fn rounds(&self) -> u64 { + self.rounds.load(Ordering::Relaxed) + } + + /// Waiters that stood down because a wakeup never arrived. + /// + /// The live half of this scenario's oracle. A waiter counted here was parked + /// on a promise the driver had completed, and stayed parked long enough that + /// no recovery budget explains it. The read-back then decides whether it + /// eventually woke. + pub fn stalled(&self) -> u64 { + self.stalled.load(Ordering::Relaxed) + } + + pub async fn stop(mut self) { + self.running.store(false, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + } +} + +/// Warms every waiter so the baseline measures resident agents rather than cold +/// starts. Returns how many answered. +pub async fn warm(ctx: &WorkloadContext, waiters: &[String]) -> usize { + let mut warmed = 0; + let mut set = JoinSet::new(); + for waiter in waiters { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + set.spawn(async move { + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.clone()); + ctx.user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakes", data_value!()) + .await + .is_ok() + }); + } + while let Some(result) = set.join_next().await { + if matches!(result, Ok(true)) { + warmed += 1; + } + } + warmed +} + +/// Drives one whole round against a throwaway waiter, before the run commits to +/// anything. +/// +/// The first S11 run spent its entire baseline and then aborted on a count, +/// because `arm` and `complete` were both healthy and only `wait` was refused — +/// so the operation totals looked normal right up until nothing had woken. The +/// scenario knew something was wrong three minutes after it could have. +/// +/// This is the cheap version of that question, asked first and answered with the +/// platform's own error rather than with a zero. It uses an agent outside the +/// waiter pool, so a failed smoke test leaves the measured population untouched. +/// +/// Errors are returned rather than recorded: this is a pre-flight check, not +/// part of the workload, and its whole value is in saying *why*. +pub async fn smoke_test(ctx: &WorkloadContext, dwell: Duration) -> Result<(), String> { + let waiter = format!("{}-promise-smoke", ctx.key_prefix); + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.clone()); + let token = format!("{waiter}-smoke"); + + let created = ctx + .user + .invoke_and_await_agent(&ctx.promise, &parsed, "arm", data_value!(token.clone())) + .await + .map_err(|e| format!("arm failed: {e:#}"))?; + let promise = created + .into_return_value_and_type() + .ok_or_else(|| "arm returned no promise id".to_string())?; + let id = PromiseId::from_value(promise.value.clone()) + .map_err(|e| format!("arm returned a promise id the driver cannot parse: {e}"))?; + + // Fire-and-forget on purpose. A `wait` that parks correctly never returns + // while we are looking at it, so awaiting it here would prove nothing and + // block forever — but a `wait` the platform refuses fails at the API + // boundary, which is exactly the failure this is here to catch. + ctx.user + .invoke_agent( + &ctx.promise, + &parsed, + "wait", + data_value!(token.clone(), promise), + ) + .await + .map_err(|e| format!("wait was refused: {e:#}"))?; + + tokio::time::sleep(dwell).await; + ctx.user + .complete_promise(&id, COMPLETION_PAYLOAD.to_vec()) + .await + .map_err(|e| format!("complete_promise failed: {e:#}"))?; + + // The waiter has to actually come back. Polled rather than awaited for the + // same reason as above. + let deadline = tokio::time::Instant::now() + SMOKE_WAKE_TIMEOUT; + loop { + if let Ok(value) = ctx + .user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakes", data_value!()) + .await + && let Some(wakes) = value + .into_return_value_and_type() + .and_then(|v| u32::from_value(v.value).ok()) + && wakes > 0 + { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "the smoke waiter was armed, told to wait and completed, and did not wake \ + within {SMOKE_WAKE_TIMEOUT:?}" + )); + } + tokio::time::sleep(SMOKE_POLL_INTERVAL).await; + } +} + +/// Starts one loop per waiter. +pub fn start(ctx: WorkloadContext, waiters: &[String], config: &PromiseConfig) -> WaiterHandle { + let running = Arc::new(AtomicBool::new(true)); + let rounds = Arc::new(AtomicU64::new(0)); + let stalled = Arc::new(AtomicU64::new(0)); + let mut tasks = JoinSet::new(); + + let stall_timeout = (config.wakeup_budget() * STALL_MULTIPLE).max(MIN_STALL_TIMEOUT); + info!( + "S11: starting {} waiters, {:?} dwell, standing a waiter down after {stall_timeout:?} \ + without a wakeup", + waiters.len(), + config.dwell() + ); + + for waiter in waiters { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + let running = running.clone(); + let rounds = rounds.clone(); + let stalled = stalled.clone(); + let dwell = config.dwell(); + tasks.spawn(async move { + run_waiter(ctx, waiter, dwell, stall_timeout, running, rounds, stalled).await; + }); + } + + WaiterHandle { + tasks, + running, + rounds, + stalled, + } +} + +/// One waiter's rounds, until the workload stops or the waiter stands down. +async fn run_waiter( + ctx: WorkloadContext, + waiter: String, + dwell: Duration, + stall_timeout: Duration, + running: Arc, + rounds: Arc, + stalled: Arc, +) { + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.clone()); + let mut round = 0u64; + + while running.load(Ordering::Relaxed) { + let token = ctx.idempotency_key(&waiter, round); + round += 1; + rounds.fetch_add(1, Ordering::Relaxed); + + let Some(armed) = arm(&ctx, &waiter, &parsed, &token).await else { + // Nothing to complete and nobody parked. The next round tries again + // after a dwell, which keeps a wholly unreachable platform from + // spinning. + tokio::time::sleep(dwell).await; + continue; + }; + + // Park the waiter. Held open deliberately: the driver's own view of the + // wakeup is one of the two independent answers this scenario collects, + // and the fault is expected to take it away. + let wait_task = { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + let parsed = parsed.clone(); + let token = token.clone(); + let promise = armed.value.clone(); + tokio::spawn(async move { wait(&ctx, &waiter, &parsed, &token, &promise).await }) + }; + + tokio::time::sleep(dwell).await; + complete(&ctx, &waiter, &token, &armed.id).await; + + // The handle is dropped rather than aborted on the timeout path, which + // leaves the invocation running. That is deliberate: aborting it would + // throw away the one record that says how the round ended, and the agent + // is parked either way — the driver's task is not what is holding it. + match tokio::time::timeout(stall_timeout, wait_task).await { + Ok(_) => {} + Err(_) => { + warn!( + "S11: waiter {waiter} has not woken {stall_timeout:?} after its completion \ + on token {token}; standing it down" + ); + stalled.fetch_add(1, Ordering::Relaxed); + return; + } + } + } +} + +/// One armed promise, in the two forms the round needs it. +/// +/// Both halves are kept because the two consumers want different things and +/// only one of them will accept a re-encoding. +#[derive(Clone)] +struct Armed { + /// For [`TestDsl::complete_promise`], the external REST API, which takes a + /// typed promise id and serialises it itself. + id: PromiseId, + /// For the `wait` invocation, which passes the promise back *into* an + /// agent. + /// + /// This has to be the value the agent handed out, verbatim. Re-encoding the + /// parsed [`PromiseId`] does not work: `data_value!` derives the parameter's + /// type from `IntoValue`, which names the record's fields with their WIT + /// spellings (`agent-id`, `oplog-idx`), while the agent's generated + /// parameter schema declares them in the component model's own spelling + /// (`agent_id`, `oplog_idx`). The type checker rejects the call outright. + /// Threading the original value through sidesteps the question, because its + /// type came from the agent in the first place. + /// + /// `crate::benchmarks::density::promise` keeps both halves for exactly this + /// reason. + value: ValueAndType, +} + +/// Creates the round's promise, recording the invocation. +/// +/// The result comes back through a cell rather than a return value because +/// [`workload::run_operation`] owns the retry rule and the failure +/// classification for every stream in the suite, and it reports outcomes rather +/// than payloads. Duplicating it here to get one value back would put the two +/// load-bearing rules of every chaos scenario in two places. +async fn arm( + ctx: &WorkloadContext, + waiter: &str, + parsed: &ParsedAgentId, + token: &str, +) -> Option { + let cell: Arc>> = Arc::new(std::sync::Mutex::new(None)); + let sink = cell.clone(); + let ctx2 = ctx.clone(); + let parsed2 = parsed.clone(); + + workload::run_operation( + ctx, + Stream::PromiseWait, + waiter.to_string(), + "arm", + format!("{token}-arm"), + |key| { + let ctx = ctx2.clone(); + let parsed = parsed2.clone(); + let sink = sink.clone(); + let token = token.to_string(); + async move { + let created = ctx + .user + .invoke_and_await_agent_with_key( + &ctx.promise, + &parsed, + &key, + "arm", + data_value!(token), + ) + .await?; + let value = created + .into_return_value_and_type() + .ok_or_else(|| anyhow::anyhow!("arm returned no promise id"))?; + let id = PromiseId::from_value(value.value.clone()) + .map_err(|e| anyhow::anyhow!("invalid promise id: {e}"))?; + *sink.lock().unwrap() = Some(Armed { id, value }); + Ok(None) + } + }, + ) + .await; + + cell.lock().unwrap().clone() +} + +/// Parks the waiter until its promise resolves. +async fn wait( + ctx: &WorkloadContext, + waiter: &str, + parsed: &ParsedAgentId, + token: &str, + promise: &ValueAndType, +) { + let ctx2 = ctx.clone(); + let parsed2 = parsed.clone(); + workload::run_operation( + ctx, + Stream::PromiseWait, + waiter.to_string(), + "wait", + format!("{token}-wait"), + |key| { + let ctx = ctx2.clone(); + let parsed = parsed2.clone(); + let token = token.to_string(); + let promise = promise.clone(); + async move { + ctx.user + .invoke_and_await_agent_with_key( + &ctx.promise, + &parsed, + &key, + "wait", + data_value!(token, promise), + ) + .await?; + Ok(None) + } + }, + ) + .await; +} + +/// Completes the round's promise from outside the agent. +/// +/// Recorded under the round's token rather than under a key of its own, which is +/// what joins this record to the wakeup the waiter logs. The completion API is +/// keyed by promise id and takes no idempotency key of its own — a retry is the +/// same completion because it names the same promise, and `set_if_not_exists` on +/// the platform side is what makes that true. +async fn complete(ctx: &WorkloadContext, waiter: &str, token: &str, promise_id: &PromiseId) { + let ctx2 = ctx.clone(); + workload::run_operation( + ctx, + Stream::PromiseWait, + waiter.to_string(), + "complete", + token.to_string(), + |_key| { + let ctx = ctx2.clone(); + let promise_id = promise_id.clone(); + async move { + ctx.user + .complete_promise(&promise_id, COMPLETION_PAYLOAD.to_vec()) + .await?; + Ok(None) + } + }, + ) + .await; +} + +/// Reads every waiter's wakeup log. +pub async fn read_logs(ctx: &WorkloadContext, waiters: &[String]) -> Vec { + let mut logs = Vec::with_capacity(waiters.len()); + for chunk in waiters.chunks(READ_CONCURRENCY) { + let mut set = JoinSet::new(); + for waiter in chunk { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + set.spawn(async move { read_log(&ctx, &waiter).await }); + } + while let Some(result) = set.join_next().await { + match result { + Ok(log) => logs.push(log), + Err(e) => warn!("S11: a wakeup-log read task failed: {e}"), + } + } + } + logs.sort_by(|a, b| a.agent.cmp(&b.agent)); + logs +} + +async fn read_log(ctx: &WorkloadContext, waiter: &str) -> WaiterWakeupLog { + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.to_string()); + + let wakes = match read_within(waiter, "wakes", async { + ctx.user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakes", data_value!()) + .await + .map_err(|e| format!("{e:#}")) + }) + .await + { + Ok(value) => value + .into_return_value_and_type() + .and_then(|v| u32::from_value(v.value).ok()) + .map(|v| v as u64), + Err(e) => { + return WaiterWakeupLog { + agent: waiter.to_string(), + wakes: None, + wakeups: Vec::new(), + error: Some(e), + }; + } + }; + + match read_within(waiter, "wakeups", async { + ctx.user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakeups", data_value!()) + .await + .map_err(|e| format!("{e:#}")) + }) + .await + { + Ok(value) => { + let wakeups = value + .into_return_value_and_type() + .map(|v| parse_wakeups(v.value)) + .unwrap_or_default(); + WaiterWakeupLog { + agent: waiter.to_string(), + wakes, + wakeups, + error: None, + } + } + Err(e) => WaiterWakeupLog { + agent: waiter.to_string(), + wakes, + wakeups: Vec::new(), + error: Some(e), + }, + } +} + +/// A read-back invocation under [`READ_TIMEOUT`]. +/// +/// A timeout is a verdict, not an error to propagate. It is also the loudest +/// thing this scenario can observe: a waiter that will not answer `wakeups` is +/// usually a waiter still parked on a promise that was completed long ago, and +/// [`crate::chaos::wakeups`] treats that case differently from an ordinary +/// unreadable agent. +async fn read_within(waiter: &str, what: &str, read: F) -> Result +where + F: std::future::Future>, +{ + match tokio::time::timeout(READ_TIMEOUT, read).await { + Ok(result) => result, + Err(_) => { + warn!("S11: reading {what} on waiter {waiter} timed out after {READ_TIMEOUT:?}"); + Err(format!("{what} timed out after {READ_TIMEOUT:?}")) + } + } +} + +/// Turns the agent's `(token, armed_millis, woken_millis)` triples into records. +fn parse_wakeups(value: golem_wasm::Value) -> Vec { + let raw: Vec<(String, u64, u64)> = match Vec::<(String, u64, u64)>::from_value(value) { + Ok(raw) => raw, + Err(e) => { + warn!("S11: could not read a wakeup log: {e}"); + return Vec::new(); + } + }; + raw.into_iter() + .map(|(token, armed_millis, woken_millis)| WakeupRecord { + token, + armed_at: from_millis(armed_millis), + woken_at: from_millis(woken_millis), + }) + .collect() +} + +/// Epoch milliseconds as the agent stamped them. +/// +/// A zero means the agent had no armed time for the token, which the component +/// only produces if its own arm log rolled over. It is kept as the epoch rather +/// than dropped so the resulting nonsense interval is visible instead of the +/// wakeup silently going missing. +fn from_millis(millis: u64) -> DateTime { + Utc.timestamp_millis_opt(millis as i64) + .single() + .unwrap_or_else(|| Utc.timestamp_nanos(0)) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + #[test] + fn a_stall_timeout_scales_with_the_wakeup_budget() { + let config = PromiseConfig { + waiters: 10, + dwell_millis: 5000, + wakeup_budget_secs: 120, + }; + assert_eq!( + (config.wakeup_budget() * STALL_MULTIPLE).max(MIN_STALL_TIMEOUT), + Duration::from_secs(480) + ); + } + + /// A budget short enough that four of it would call an ordinary recovery a + /// stall still gets the floor. + #[test] + fn a_short_wakeup_budget_still_gets_the_floor() { + let config = PromiseConfig { + waiters: 10, + dwell_millis: 5000, + wakeup_budget_secs: 10, + }; + assert_eq!( + (config.wakeup_budget() * STALL_MULTIPLE).max(MIN_STALL_TIMEOUT), + MIN_STALL_TIMEOUT + ); + } + + #[test] + fn an_agent_timestamp_of_zero_stays_visible_as_the_epoch() { + assert_eq!(from_millis(0).timestamp_millis(), 0); + } + + /// Why `wait` is handed the value `arm` returned rather than a re-encoded + /// [`PromiseId`]. + /// + /// This is not a style preference, and getting it wrong is not caught by + /// anything local: the first S11 run had every single `wait` refused by the + /// type checker in two milliseconds, with `arm` and `complete` both clean, + /// because a re-encoded promise id names its fields the way WIT does and the + /// agent's generated parameter schema names them the way the component model + /// does. The two spellings never meet. + /// + /// So the rule is: a value that came *out* of an agent goes back *into* one + /// verbatim. This test pins the mismatch that makes the rule necessary, so + /// that a future change to either derive shows up here rather than on a + /// cluster. + #[test] + fn a_re_encoded_promise_id_does_not_name_its_fields_the_way_an_agent_declares_them() { + use golem_common::model::PromiseId; + use golem_wasm::IntoValue; + use golem_wasm::analysis::AnalysedType; + + let AnalysedType::Record(record) = PromiseId::get_type() else { + panic!("a promise id is a record"); + }; + let names: Vec<&str> = record.fields.iter().map(|f| f.name.as_str()).collect(); + + // What `data_value!` would ship: the WIT spellings. + assert!(names.contains(&"agent-id"), "got {names:?}"); + assert!(names.contains(&"oplog-idx"), "got {names:?}"); + + // What the agent's parameter schema actually asks for, and does not get. + assert!(!names.contains(&"agent_id"), "got {names:?}"); + assert!(!names.contains(&"oplog_idx"), "got {names:?}"); + } +} diff --git a/integration-tests/src/chaos/wakeups.rs b/integration-tests/src/chaos/wakeups.rs new file mode 100644 index 0000000000..55506a84e2 --- /dev/null +++ b/integration-tests/src/chaos/wakeups.rs @@ -0,0 +1,1120 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pairing completions against wakeups (GOL-377). +//! +//! S11's question is not "how many promises resolved". It is, for each +//! completion the platform *accepted*, whether the agent suspended on that +//! promise was resumed — once, and only once. Counting cannot answer it: a lost +//! wakeup and a duplicate wakeup cancel out in a total, and neither can be +//! localised to an agent afterwards. +//! +//! So every round carries a token. The driver mints it, `arm` records it against +//! the promise it created, and the waiter writes it into its wakeup log when it +//! is resumed. Pairing the two turns each failure into a statement about one +//! named round: this completion, accepted at this time, against this waiter, +//! never woke it. +//! +//! ## What counts as a finding, and what only counts as doubt +//! +//! A completion the platform confirmed is a promise the platform made. If the +//! waiter never woke and its log is whole, that is a finding, full stop. +//! +//! A completion that failed in an *indeterminate* way is not. From the client +//! side a dropped connection is indistinguishable from a request that arrived +//! and executed, and a pod kill produces exactly these. If such a round woke, +//! the platform resolved the doubt in its own favour and the report says so; if +//! it did not, nothing is proven and the round is counted as inconclusive rather +//! than as a loss. +//! +//! ## The waiter that will not answer +//! +//! One case here is unlike anything the other scenarios see. A waiter is parked +//! *inside* an invocation, so a waiter that never wakes cannot answer a read +//! either — its `wakeups` read queues behind the `wait` that is still running. +//! +//! An unreadable agent is normally the weakest possible outcome: it means the +//! run cannot say. Here it is nearly the opposite. A waiter whose completion was +//! confirmed, which then stopped producing rounds *and* could not be read, is a +//! worker wedged on a promise that was resolved long ago. The report separates +//! that case ([`WakeupViolation::NeverWoke`], with the read failure as its +//! detail) from an agent that merely timed out while otherwise healthy. +//! +//! ## Two clocks, and which number to believe +//! +//! The waiter stamps `armedAt` and `wokenAt` from the executor's clock; the +//! driver stamps the completion from its own. So the headline delay — +//! completion accepted to waiter resumed — spans two clocks, exactly as the +//! scheduled-fire delay does in [`crate::chaos::fires`], and the same guard +//! applies: `minDelayMs` is reported per cell so skew shows up as a negative +//! number instead of quietly flattering a percentile. +//! +//! There is one cross-check S10 cannot make. `parkedMs` — armed to woken — is +//! stamped at both ends by the executor, so it carries no skew at all. It is not +//! the delay, because it also contains the round's deliberate dwell, but on a +//! healthy baseline `parked - dwell` and the cross-clock delay should agree. A +//! gap between them is skew, and the report carries both rather than picking. + +use crate::chaos::history::{OperationRecord, Outcome, Stream, WaiterWakeupLog, WakeupRecord}; +use crate::chaos::split::{FaultWindow, PodSplit, Window}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +/// Ceiling on how many findings the report carries. +/// +/// A run that lost every wakeup would otherwise produce tens of thousands of +/// them and an artifact nobody can open. The count is reported separately, so +/// truncation is stated rather than inferred from a suspiciously round number. +const MAX_FINDINGS: usize = 200; + +/// The method name the completion operations are recorded under. +const COMPLETE_METHOD: &str = "complete"; + +/// The method name the parking invocations are recorded under. +const WAIT_METHOD: &str = "wait"; + +/// What [`WAIT_METHOD`] operations append to their round's token to form their +/// own idempotency key. +const WAIT_KEY_SUFFIX: &str = "-wait"; + +/// Whether a waiter was on the executor the fault killed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WaiterGroup { + /// Owned by the killed executor when the driver signalled readiness. + OnKilledExecutor, + /// Owned by an executor the fault left alone: the run's own control group. + Elsewhere, +} + +impl WaiterGroup { + pub fn as_str(self) -> &'static str { + match self { + WaiterGroup::OnKilledExecutor => "on-killed-executor", + WaiterGroup::Elsewhere => "elsewhere", + } + } +} + +impl std::fmt::Display for WaiterGroup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// What went wrong with one round. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WakeupViolation { + /// A completion the platform accepted whose waiter was never resumed. + NeverWoke, + /// One completion, two or more wakeups. A promise resolved twice, or a + /// recovery that replayed the resume without deduplicating it. + WokeMoreThanOnce, + /// A wakeup for a round whose completion the platform definitively refused. + WokeDespiteRejection, +} + +impl WakeupViolation { + pub fn as_str(self) -> &'static str { + match self { + WakeupViolation::NeverWoke => "never-woke", + WakeupViolation::WokeMoreThanOnce => "woke-more-than-once", + WakeupViolation::WokeDespiteRejection => "woke-despite-rejection", + } + } +} + +impl std::fmt::Display for WakeupViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, against one round. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupFinding { + pub violation: WakeupViolation, + pub token: String, + pub agent: String, + pub window: Window, + pub detail: String, +} + +/// Wakeup delay for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupDelayStats { + pub group: WaiterGroup, + pub window: Window, + /// Percentiles over delays clamped at zero, so skew cannot flatter them. + pub delay: LatencyStats, + /// The most negative delay seen, which is the clock skew between the driver + /// and the executor rather than a waiter woken before it was completed. + pub min_delay_ms: i64, + /// Wakeups whose delay exceeded the configured budget. + pub over_budget: u64, + /// Armed-to-woken, on the executor's clock alone. Carries the round's dwell + /// as well as the delay, and carries no skew. + pub parked: LatencyStats, + /// How much longer the *caller* waited than the platform actually took. + /// + /// The `wait` invocation's own duration, less the round's dwell, less the + /// delay the waiter recorded. On a healthy round this is a few milliseconds + /// of round trip. A large value means the platform woke the agent on time + /// and the answer did not come back — which is a different defect from a + /// slow wakeup, and invisible in [`Self::delay`]. + pub client_excess: LatencyStats, + /// Rounds whose caller waited longer than the whole wakeup budget *after* + /// the waiter had already woken. + pub client_stalled: u64, +} + +/// The promise-wakeup account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupReport { + /// What resuming a suspended waiter is allowed to cost, from the suite YAML. + /// Recorded so a percentile in an archived result can be read years later + /// against the number it was judged by rather than against today's config. + pub wakeup_budget_ms: u64, + /// The dwell each round held before its completion, so `parked` can be read + /// without the suite YAML to hand. + pub dwell_ms: u64, + pub completions_confirmed: u64, + pub completions_indeterminate: u64, + pub completions_rejected: u64, + /// Wakeups the waiters recorded, including any whose token is unknown. + pub wakeups_recorded: u64, + /// Accepted completions paired with exactly one wakeup. + pub woke_once: u64, + /// Completions the driver was never sure of that woke anyway — doubt the + /// platform resolved in its own favour. + pub indeterminate_that_woke: u64, + /// Completions the driver was never sure of that never woke. Not a finding: + /// the completion may never have landed. + pub inconclusive: u64, + /// Completions whose waiter could not testify, because its log was + /// unreadable or truncated *and* nothing else about it looked wrong. + pub unverifiable: u64, + /// Wakeups whose token no completion claims. Zero on a healthy run: agent + /// names carry the run nonce, so nothing from an earlier run can appear. + pub unknown_tokens: u64, + /// Waiters the read-back could not reach at all. + pub waiters_unreadable: Vec, + /// Waiters whose wakeup log hit the component's cap. + pub waiters_truncated: Vec, + /// Waiters that stopped producing rounds during the run because a wakeup + /// never arrived, as the workload itself observed. The live half of the + /// oracle — see [`crate::chaos::waiters::WaiterHandle::stalled`]. + pub waiters_stood_down: u64, + /// Waiters that stood down *and* could not then be read: parked inside an + /// invocation, which is what a wedged worker looks like from outside. + pub waiters_wedged: Vec, + pub delay: Vec, + /// Rounds across every cell whose caller waited past the budget after the + /// wakeup had already happened. See [`WakeupDelayStats::client_excess`]. + pub client_stalled_total: u64, + /// The worst such gap, in milliseconds. + pub client_stall_worst_ms: u64, + /// How many of those callers had to retry to get their answer at all. A + /// stall that only ends on a retry is a request that was never coming back. + pub client_stall_retried: u64, + pub findings: Vec, + /// Findings past [`MAX_FINDINGS`], which the report drops rather than + /// carries. Non-zero means `findings` is a sample. + pub findings_omitted: u64, +} + +/// One round, as the pairing sees it. +struct Round<'a> { + token: &'a str, + agent: &'a str, + outcome: Outcome, + submitted_at: DateTime, +} + +impl WakeupReport { + /// Pairs completions against wakeups. + /// + /// `records` is the whole history; only the `complete` operations of the + /// waiter stream are considered — `arm` and `wait` are recorded for the + /// timeline, not for this account. + #[allow(clippy::too_many_arguments)] + pub fn build( + records: &[OperationRecord], + logs: &[WaiterWakeupLog], + split: &PodSplit, + fault: Option, + dwell: Duration, + wakeup_budget: Duration, + stood_down: u64, + ) -> Self { + let budget_ms = wakeup_budget.as_millis().min(u64::MAX as u128) as u64; + let dwell_ms = dwell.as_millis().min(u64::MAX as u128) as u64; + + let mut wakeups_by_token: BTreeMap<&str, Vec<&WakeupRecord>> = BTreeMap::new(); + let mut complete_log: BTreeMap<&str, bool> = BTreeMap::new(); + let mut waiters_unreadable = Vec::new(); + let mut waiters_truncated = Vec::new(); + let mut wakeups_recorded = 0u64; + + for log in logs { + complete_log.insert(log.agent.as_str(), log.is_complete()); + if log.error.is_some() { + waiters_unreadable.push(log.agent.clone()); + } else if !log.is_complete() { + waiters_truncated.push(log.agent.clone()); + } + for wakeup in &log.wakeups { + wakeups_recorded += 1; + wakeups_by_token + .entry(wakeup.token.as_str()) + .or_default() + .push(wakeup); + } + } + + let on_killed: BTreeSet<&str> = split.on_pod.iter().map(|s| s.as_str()).collect(); + let mut report = Self::empty(budget_ms, dwell_ms); + report.wakeups_recorded = wakeups_recorded; + report.waiters_stood_down = stood_down; + + // A stalled waiter that then could not be read is the wedged case. It is + // computed before the round loop because the loop uses it to decide + // whether an unreadable waiter excuses a missing wakeup or convicts it. + let unreadable: BTreeSet<&str> = waiters_unreadable.iter().map(|s| s.as_str()).collect(); + + let mut cells: BTreeMap<(WaiterGroup, Window), DelayCell> = BTreeMap::new(); + let mut claimed: BTreeSet<&str> = BTreeSet::new(); + let waits = wait_observations(records); + let mut stall_worst_ms = 0u64; + let mut stall_retried = 0u64; + + for round in completions(records) { + let group = if on_killed.contains(round.agent) { + WaiterGroup::OnKilledExecutor + } else { + WaiterGroup::Elsewhere + }; + let window = Window::of(round.submitted_at, fault); + let wakeups = wakeups_by_token + .get(round.token) + .cloned() + .unwrap_or_default(); + claimed.insert(round.token); + + match round.outcome { + Outcome::Confirmed => report.completions_confirmed += 1, + Outcome::Indeterminate => report.completions_indeterminate += 1, + Outcome::Rejected => report.completions_rejected += 1, + } + + if wakeups.len() > 1 { + report.push_finding(WakeupFinding { + violation: WakeupViolation::WokeMoreThanOnce, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "{} wakeups recorded for one completion, at {}", + wakeups.len(), + wakeups + .iter() + .map(|w| w.woken_at.to_rfc3339()) + .collect::>() + .join(", ") + ), + }); + } + + if let Some(wakeup) = wakeups.first() { + if round.outcome == Outcome::Rejected { + report.push_finding(WakeupFinding { + violation: WakeupViolation::WokeDespiteRejection, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "the completion was definitively refused, and the waiter woke at {}", + wakeup.woken_at.to_rfc3339() + ), + }); + } else { + if round.outcome == Outcome::Confirmed { + report.woke_once += 1; + } else { + report.indeterminate_that_woke += 1; + } + let delay_ms = (wakeup.woken_at - round.submitted_at).num_milliseconds(); + let cell = cells.entry((group, window)).or_default(); + cell.push(delay_ms, wakeup.parked_ms(), budget_ms); + + // The caller's own view of the same round. `wait` covers the + // dwell as well as the wakeup, so the dwell comes off before + // the two are compared. + if let Some(observed) = waits.get(round.token) { + let client_ms = observed.duration_ms.saturating_sub(dwell_ms); + let excess = client_ms.saturating_sub(delay_ms.max(0) as u64); + cell.push_client(excess, budget_ms); + if excess > budget_ms { + stall_worst_ms = stall_worst_ms.max(excess); + if observed.attempts > 1 { + stall_retried += 1; + } + } + } + } + continue; + } + + // No wakeup. What that proves depends on the completion's outcome + // and on whether the waiter could testify at all. + match round.outcome { + Outcome::Rejected => {} + Outcome::Indeterminate => report.inconclusive += 1, + Outcome::Confirmed => { + let log_whole = complete_log.get(round.agent).copied().unwrap_or(false); + if log_whole { + report.push_finding(WakeupFinding { + violation: WakeupViolation::NeverWoke, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "the completion was accepted at {} and the waiter's whole wakeup \ + log has no entry for it", + round.submitted_at.to_rfc3339() + ), + }); + } else if unreadable.contains(round.agent) && stood_down > 0 { + // The wedged case: the workload watched this waiter stop + // producing, and the read-back then could not reach it. + // Both symptoms of one worker still parked on a promise + // that was resolved. + report.push_finding(WakeupFinding { + violation: WakeupViolation::NeverWoke, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "the completion was accepted at {} and the waiter has answered \ + nothing since, which is what a worker parked on a resolved \ + promise looks like from outside", + round.submitted_at.to_rfc3339() + ), + }); + if !report.waiters_wedged.iter().any(|w| w == round.agent) { + report.waiters_wedged.push(round.agent.to_string()); + } + } else { + report.unverifiable += 1; + } + } + } + } + + report.unknown_tokens = wakeups_by_token + .keys() + .filter(|token| !claimed.contains(*token)) + .count() as u64; + report.waiters_unreadable = waiters_unreadable; + report.waiters_truncated = waiters_truncated; + report.delay = cells + .into_iter() + .map(|((group, window), cell)| cell.into_stats(group, window)) + .collect(); + report.client_stalled_total = report.delay.iter().map(|c| c.client_stalled).sum(); + report.client_stall_worst_ms = stall_worst_ms; + report.client_stall_retried = stall_retried; + report + } + + fn empty(wakeup_budget_ms: u64, dwell_ms: u64) -> Self { + Self { + wakeup_budget_ms, + dwell_ms, + completions_confirmed: 0, + completions_indeterminate: 0, + completions_rejected: 0, + wakeups_recorded: 0, + woke_once: 0, + indeterminate_that_woke: 0, + inconclusive: 0, + unverifiable: 0, + unknown_tokens: 0, + waiters_unreadable: Vec::new(), + waiters_truncated: Vec::new(), + waiters_stood_down: 0, + waiters_wedged: Vec::new(), + delay: Vec::new(), + client_stalled_total: 0, + client_stall_worst_ms: 0, + client_stall_retried: 0, + findings: Vec::new(), + findings_omitted: 0, + } + } + + fn push_finding(&mut self, finding: WakeupFinding) { + if self.findings.len() < MAX_FINDINGS { + self.findings.push(finding); + } else { + self.findings_omitted += 1; + } + } + + /// Whether the run found anything that fails it. + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() || self.findings_omitted > 0 + } + + /// Total findings, including any the report dropped. + pub fn violations(&self) -> u64 { + self.findings.len() as u64 + self.findings_omitted + } + + /// The p99 wakeup delay on the killed executor's waiters, during the fault: + /// the one number this scenario exists to produce. + pub fn fault_window_p99_ms(&self) -> Option { + self.delay + .iter() + .find(|cell| { + cell.group == WaiterGroup::OnKilledExecutor && cell.window == Window::DuringFault + }) + .map(|cell| cell.delay.p99_ms) + } + + /// Lines an operator has to act on. + pub fn attention_lines(&self) -> Vec { + let mut lines = Vec::new(); + if self.has_violations() { + lines.push(format!( + "S11 found {} promise-wakeup violations: {}", + self.violations(), + self.findings + .iter() + .take(3) + .map(|f| format!("{} on {}", f.violation, f.token)) + .collect::>() + .join(", ") + )); + } + if !self.waiters_wedged.is_empty() { + lines.push(format!( + "S11 left {} waiters wedged: {} — each stopped producing rounds and then \ + answered no read, which is a worker still parked on a resolved promise", + self.waiters_wedged.len(), + self.waiters_wedged.join(", ") + )); + } + if !self.waiters_unreadable.is_empty() || !self.waiters_truncated.is_empty() { + lines.push(format!( + "S11 could not take a whole account from {} waiters ({} unreadable, {} \ + truncated), so {} accepted completions are unverified either way", + self.waiters_unreadable.len() + self.waiters_truncated.len(), + self.waiters_unreadable.len(), + self.waiters_truncated.len(), + self.unverifiable + )); + } + if self.client_stalled_total > 0 { + lines.push(format!( + "S11: {} rounds woke on time and the caller was not told for up to {}ms — {} of \ + them only got an answer by retrying. The waiters' own logs say the platform \ + resumed them promptly, so this is the response path, not the wakeup", + self.client_stalled_total, self.client_stall_worst_ms, self.client_stall_retried + )); + } + if self.unknown_tokens > 0 { + lines.push(format!( + "S11 recorded {} wakeups whose token no completion claims — agent names carry \ + the run nonce, so this should be impossible", + self.unknown_tokens + )); + } + lines + } + + /// Lines that explain the account without claiming anything is wrong. + pub fn note_lines(&self) -> Vec { + let mut lines = Vec::new(); + if self.waiters_stood_down > 0 && self.waiters_wedged.is_empty() { + lines.push(format!( + "S11 stood {} waiters down after a slow wakeup, and every one of them was \ + readable afterwards — late rather than lost", + self.waiters_stood_down + )); + } + if self.indeterminate_that_woke > 0 { + lines.push(format!( + "S11 had {} completions fail in a way that proves nothing, whose waiters woke \ + anyway", + self.indeterminate_that_woke + )); + } + if self.inconclusive > 0 { + lines.push(format!( + "S11 had {} completions that neither succeeded nor demonstrably landed, whose \ + waiters did not wake — not losses, because the completion may never have \ + arrived", + self.inconclusive + )); + } + lines + } +} + +/// Delays accumulated for one cell before they become percentiles. +#[derive(Default)] +struct DelayCell { + delays: Vec, + parked: Vec, + client_excess: Vec, + client_stalled: u64, + min_delay_ms: i64, + over_budget: u64, + any: bool, +} + +impl DelayCell { + fn push(&mut self, delay_ms: i64, parked_ms: i64, budget_ms: u64) { + if !self.any || delay_ms < self.min_delay_ms { + self.min_delay_ms = delay_ms; + } + self.any = true; + let clamped = delay_ms.max(0) as u64; + if clamped > budget_ms { + self.over_budget += 1; + } + self.delays.push(clamped); + self.parked.push(parked_ms.max(0) as u64); + } + + /// Records how much longer the caller waited than the platform took. + fn push_client(&mut self, excess_ms: u64, budget_ms: u64) { + if excess_ms > budget_ms { + self.client_stalled += 1; + } + self.client_excess.push(excess_ms); + } + + fn into_stats(self, group: WaiterGroup, window: Window) -> WakeupDelayStats { + WakeupDelayStats { + group, + window, + delay: LatencyStats::from_durations(self.delays), + min_delay_ms: self.min_delay_ms, + over_budget: self.over_budget, + parked: LatencyStats::from_durations(self.parked), + client_excess: LatencyStats::from_durations(self.client_excess), + client_stalled: self.client_stalled, + } + } +} + +/// What the driver's own `wait` invocation cost, per round. +/// +/// Keyed by the round's token: the `wait` operation is recorded under +/// `{token}-wait`, which is what lets the caller's view be joined to the +/// waiter's own. +struct WaitObservation { + duration_ms: u64, + attempts: u32, +} + +fn wait_observations(records: &[OperationRecord]) -> BTreeMap<&str, WaitObservation> { + records + .iter() + .filter(|r| r.stream == Stream::PromiseWait && r.method == WAIT_METHOD) + .filter_map(|r| { + r.idempotency_key + .strip_suffix(WAIT_KEY_SUFFIX) + .map(|token| { + ( + token, + WaitObservation { + duration_ms: r.duration_ms, + attempts: r.attempts, + }, + ) + }) + }) + .collect() +} + +/// The completion operations, which are the rounds this report is about. +fn completions(records: &[OperationRecord]) -> impl Iterator> { + records + .iter() + .filter(|r| r.stream == Stream::PromiseWait && r.method == COMPLETE_METHOD) + .map(|r| Round { + token: r.idempotency_key.as_str(), + agent: r.agent.as_str(), + outcome: r.outcome, + submitted_at: r.submitted_at, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::Phase; + use test_r::test; + + fn at(secs: i64) -> DateTime { + DateTime::from_timestamp(secs, 0).unwrap() + } + + fn at_ms(millis: i64) -> DateTime { + DateTime::from_timestamp_millis(millis).unwrap() + } + + /// The fault: injected at 100s, cleared at 200s. + fn fault() -> Option { + Some(FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + }) + } + + fn split_of(on_pod: &[&str], elsewhere: &[&str]) -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: on_pod.iter().map(|s| s.to_string()).collect(), + elsewhere: elsewhere.iter().map(|s| s.to_string()).collect(), + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn completion( + agent: &str, + token: &str, + submitted: DateTime, + outcome: Outcome, + ) -> OperationRecord { + OperationRecord { + op_id: 1, + stream: Stream::PromiseWait, + phase: Phase::Fault, + agent: agent.to_string(), + method: COMPLETE_METHOD.to_string(), + idempotency_key: token.to_string(), + submitted_at: submitted, + completed_at: Some(submitted), + attempts: 1, + outcome, + duration_ms: 0, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + } + } + + /// A whole log: `wakes` matches the entries, so an absent wakeup is a + /// statement rather than a gap. + fn log(agent: &str, wakeups: Vec) -> WaiterWakeupLog { + WaiterWakeupLog { + agent: agent.to_string(), + wakes: Some(wakeups.len() as u64), + wakeups, + error: None, + } + } + + fn wakeup(token: &str, armed: DateTime, woken: DateTime) -> WakeupRecord { + WakeupRecord { + token: token.to_string(), + armed_at: armed, + woken_at: woken, + } + } + + fn build( + records: &[OperationRecord], + logs: &[WaiterWakeupLog], + split: &PodSplit, + stood_down: u64, + ) -> WakeupReport { + WakeupReport::build( + records, + logs, + split, + fault(), + Duration::from_secs(5), + Duration::from_secs(60), + stood_down, + ) + } + + #[test] + fn a_completion_paired_with_one_wakeup_is_clean() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.completions_confirmed, 1); + assert_eq!(report.woke_once, 1); + assert!(!report.has_violations()); + } + + /// The headline failure. An accepted completion is a promise the platform + /// made; a whole log with no entry for it is the platform not keeping it. + #[test] + fn an_accepted_completion_that_never_woke_its_waiter_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", Vec::new())]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.findings.len(), 1); + assert_eq!(report.findings[0].violation, WakeupViolation::NeverWoke); + assert_eq!(report.findings[0].token, "t-1"); + assert_eq!(report.findings[0].window, Window::DuringFault); + } + + /// A truncated log cannot testify, so the same missing wakeup proves + /// nothing. Calling it a loss would turn a component-side cap into a + /// platform defect. + #[test] + fn a_missing_wakeup_on_a_truncated_log_is_unverifiable_rather_than_lost() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![WaiterWakeupLog { + agent: "w-1".to_string(), + wakes: Some(9000), + wakeups: Vec::new(), + error: None, + }]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert!(!report.has_violations()); + assert_eq!(report.unverifiable, 1); + assert_eq!(report.waiters_truncated, vec!["w-1".to_string()]); + } + + /// The wedged case, and the one place where an unreadable agent convicts + /// rather than excuses: the workload watched this waiter stop producing, and + /// the read-back then could not reach it. Both are symptoms of one worker + /// still parked on a promise that was resolved. + #[test] + fn a_waiter_that_stood_down_and_then_answered_nothing_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![WaiterWakeupLog { + agent: "w-1".to_string(), + wakes: None, + wakeups: Vec::new(), + error: Some("wakeups timed out after 60s".to_string()), + }]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 1); + + assert_eq!(report.findings.len(), 1); + assert_eq!(report.findings[0].violation, WakeupViolation::NeverWoke); + assert_eq!(report.waiters_wedged, vec!["w-1".to_string()]); + assert_eq!(report.unverifiable, 0); + } + + /// The same unreadable waiter, with nothing having stood down, is only an + /// unreadable waiter. A read that timed out on an otherwise healthy run says + /// nothing about whether the completion landed. + #[test] + fn an_unreadable_waiter_that_never_stood_down_is_unverifiable() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![WaiterWakeupLog { + agent: "w-1".to_string(), + wakes: None, + wakeups: Vec::new(), + error: Some("wakeups timed out after 60s".to_string()), + }]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert!(!report.has_violations()); + assert_eq!(report.unverifiable, 1); + assert!(report.waiters_wedged.is_empty()); + } + + #[test] + fn one_completion_and_two_wakeups_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log( + "w-1", + vec![ + wakeup("t-1", at(105), at(111)), + wakeup("t-1", at(105), at(160)), + ], + )]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!( + report.findings[0].violation, + WakeupViolation::WokeMoreThanOnce + ); + } + + #[test] + fn a_wakeup_for_a_refused_completion_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Rejected)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!( + report.findings[0].violation, + WakeupViolation::WokeDespiteRejection + ); + assert_eq!(report.completions_rejected, 1); + } + + /// A pod kill produces indeterminate completions by the dozen. One that woke + /// anyway is the platform resolving the doubt in its own favour, and the + /// report says so rather than counting it beside the confirmed ones. + #[test] + fn an_indeterminate_completion_that_woke_is_recorded_separately() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Indeterminate)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.indeterminate_that_woke, 1); + assert_eq!(report.woke_once, 0); + assert!(!report.has_violations()); + } + + /// And one that did not wake proves nothing at all: from the client side a + /// dropped connection is indistinguishable from a request that arrived. + #[test] + fn an_indeterminate_completion_that_never_woke_is_inconclusive_rather_than_lost() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Indeterminate)]; + let logs = vec![log("w-1", Vec::new())]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.inconclusive, 1); + assert!(!report.has_violations()); + } + + /// The control group is the whole reason the kill is aimed. Mixing the two + /// would let a recovery that took its full budget hide behind the waiters + /// that were never touched. + #[test] + fn delays_are_split_by_group_and_by_window() { + let records = vec![ + completion("w-1", "t-1", at(110), Outcome::Confirmed), + completion("w-2", "t-2", at(110), Outcome::Confirmed), + completion("w-1", "t-3", at(50), Outcome::Confirmed), + ]; + let logs = vec![ + log( + "w-1", + vec![ + wakeup("t-1", at(105), at(150)), + wakeup("t-3", at(45), at(51)), + ], + ), + log("w-2", vec![wakeup("t-2", at(105), at(111))]), + ]; + let report = build(&records, &logs, &split_of(&["w-1"], &["w-2"]), 0); + + let killed_during = report + .delay + .iter() + .find(|c| c.group == WaiterGroup::OnKilledExecutor && c.window == Window::DuringFault) + .unwrap(); + assert_eq!(killed_during.delay.max_ms, 40_000); + + let control_during = report + .delay + .iter() + .find(|c| c.group == WaiterGroup::Elsewhere && c.window == Window::DuringFault) + .unwrap(); + assert_eq!(control_during.delay.max_ms, 1_000); + + assert!(report.delay.iter().any(|c| c.window == Window::BeforeFault)); + assert_eq!(report.fault_window_p99_ms(), Some(40_000)); + } + + /// The driver and the executor keep different clocks, so a wakeup can look + /// as though it happened before the completion that caused it. That is skew, + /// and it has to be visible rather than clamped silently into a percentile. + #[test] + fn clock_skew_shows_as_a_negative_minimum_rather_than_flattering_the_percentiles() { + let records = vec![completion("w-1", "t-1", at_ms(110_000), Outcome::Confirmed)]; + let logs = vec![log( + "w-1", + vec![wakeup("t-1", at_ms(105_000), at_ms(109_700))], + )]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + let cell = &report.delay[0]; + assert_eq!(cell.min_delay_ms, -300); + assert_eq!(cell.delay.max_ms, 0); + } + + /// `parked` is stamped at both ends by the executor, so it is the one number + /// in the report free of that skew — which is what makes it worth carrying + /// beside the delay rather than instead of it. + #[test] + fn the_parked_interval_is_reported_on_the_executors_own_clock() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(150))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.delay[0].parked.max_ms, 45_000); + assert_eq!(report.dwell_ms, 5_000); + } + + #[test] + fn a_delay_past_the_budget_is_counted_without_failing_the_run() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(180))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.delay[0].over_budget, 1); + assert!(!report.has_violations()); + } + + /// Agent names carry the run nonce, so a wakeup nobody asked for should be + /// impossible. If it happens the report says so instead of dropping it. + #[test] + fn a_wakeup_no_completion_claims_is_counted() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log( + "w-1", + vec![ + wakeup("t-1", at(105), at(111)), + wakeup("t-stray", at(105), at(112)), + ], + )]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.unknown_tokens, 1); + assert_eq!(report.wakeups_recorded, 2); + } + + /// `arm` and `wait` are recorded for the timeline. Counting them as rounds + /// would treble every total in the report. + #[test] + fn only_the_completion_operations_are_counted_as_rounds() { + let mut armed = completion("w-1", "t-1-arm", at(109), Outcome::Confirmed); + armed.method = "arm".to_string(); + let mut waited = completion("w-1", "t-1-wait", at(109), Outcome::Confirmed); + waited.method = "wait".to_string(); + let records = vec![ + armed, + waited, + completion("w-1", "t-1", at(110), Outcome::Confirmed), + ]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.completions_confirmed, 1); + assert_eq!(report.woke_once, 1); + } + + /// A run that lost everything must still produce an artifact somebody can + /// open, and must say that it truncated rather than leaving a suspiciously + /// round number of findings. + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let records: Vec = (0..MAX_FINDINGS + 10) + .map(|i| completion("w-1", &format!("t-{i}"), at(110), Outcome::Confirmed)) + .collect(); + let logs = vec![log("w-1", Vec::new())]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 10); + assert_eq!(report.violations(), MAX_FINDINGS as u64 + 10); + } + + /// The caller's view and the waiter's own can disagree, and when they do the + /// waiter's is the one that describes the platform. + /// + /// This is not hypothetical: the first completed S11 run woke every one of + /// 21,863 completions on time, and 89 of those callers were not told for 125 + /// seconds. A report with only [`WakeupReport::delay`] in it would have + /// called that run flawless. + #[test] + fn a_wakeup_the_caller_was_not_told_about_is_counted_separately_from_the_delay() { + let mut waited = completion("w-1", "t-1-wait", at(110), Outcome::Confirmed); + waited.method = "wait".to_string(); + // Parked 5s, then 125s before the caller heard anything back. + waited.duration_ms = 130_000; + waited.attempts = 2; + + let records = vec![ + completion("w-1", "t-1", at(110), Outcome::Confirmed), + waited, + ]; + // The waiter itself woke one second after the completion. + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + // The platform did its job, and the delay table says so. + assert_eq!(report.woke_once, 1); + assert_eq!(report.delay[0].delay.max_ms, 1_000); + assert_eq!(report.delay[0].over_budget, 0); + assert!(!report.has_violations()); + + // And the caller still waited two minutes past that. + assert_eq!(report.client_stalled_total, 1); + assert_eq!(report.client_stall_worst_ms, 124_000); + assert_eq!(report.client_stall_retried, 1); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("response path, not the wakeup")) + ); + } + + /// A healthy round's caller waits the dwell plus the wakeup and nothing + /// more, so it must not be counted as stalled. + #[test] + fn a_prompt_round_records_no_client_stall() { + let mut waited = completion("w-1", "t-1-wait", at(110), Outcome::Confirmed); + waited.method = "wait".to_string(); + waited.duration_ms = 5_040; + + let records = vec![ + completion("w-1", "t-1", at(110), Outcome::Confirmed), + waited, + ]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(110))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.client_stalled_total, 0); + assert!(report.attention_lines().is_empty()); + } + + /// Standing waiters down is normal on a run where recovery was slow. It only + /// becomes an attention line when those waiters then could not be read. + #[test] + fn waiters_that_stood_down_but_answered_afterwards_are_context_not_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(190))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 3); + + assert!(report.attention_lines().is_empty()); + assert!( + report + .note_lines() + .iter() + .any(|line| line.contains("late rather than lost")) + ); + } +} diff --git a/integration-tests/src/chaos/workload.rs b/integration-tests/src/chaos/workload.rs index d702805951..ebe6b8ae6b 100644 --- a/integration-tests/src/chaos/workload.rs +++ b/integration-tests/src/chaos/workload.rs @@ -63,10 +63,10 @@ use tokio::task::JoinSet; use tracing::{debug, info, warn}; /// Agent type names exported by the counters component. -const COUNTER_AGENT: &str = "Counter"; +pub(crate) const COUNTER_AGENT: &str = "Counter"; const EPHEMERAL_COUNTER_AGENT: &str = "EphemeralCounter"; -const SCHEDULE_EMITTER_AGENT: &str = "ScheduleEmitter"; -const SCHEDULE_COUNTER_AGENT: &str = "ScheduleCounter"; +pub(crate) const SCHEDULE_EMITTER_AGENT: &str = "ScheduleEmitter"; +pub(crate) const SCHEDULE_COUNTER_AGENT: &str = "ScheduleCounter"; const PROMISE_AGENT: &str = "PromiseAgent"; const QUOTA_COUNTER_AGENT: &str = "QuotaCounter"; @@ -113,6 +113,10 @@ const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(120); /// a fault and a slow answer is still an answer. const READ_TIMEOUT: Duration = Duration::from_secs(30); +/// Ceiling on one fire-log read. Larger than [`READ_TIMEOUT`] because the +/// answer carries every fire the target recorded rather than one number. +const FIRE_READ_TIMEOUT: Duration = Duration::from_secs(60); + /// Payload written when completing a promise. const PROMISE_PAYLOAD: &[u8] = b"chaos"; @@ -177,6 +181,12 @@ impl WorkloadContext { format!("{}-scheduled-target-{index:04}", self.key_prefix) } + /// The suspended-waiter scenario's agent, in the promise component rather + /// than the counters one (GOL-377). + pub fn waiter_name(&self, index: u32) -> String { + format!("{}-promise-waiter-{index:04}", self.key_prefix) + } + /// The deterministic key for one operation. Same inputs, same key — that is /// what makes a retry the same operation rather than a new one. pub fn idempotency_key(&self, agent: &str, seq: u64) -> String { @@ -192,6 +202,28 @@ struct AttemptResult { class: Option, } +/// What one operation ended up as. +/// +/// Returned rather than only recorded because one scenario has to chain on it: +/// [`crate::chaos::reverts`] judges a revert by the value the *next* increment +/// reports, so it needs each operation's answer in hand rather than having to +/// go looking for its own record in a history every other emitter is also +/// appending to. +#[derive(Debug, Clone)] +pub struct OperationOutcome { + pub outcome: Outcome, + /// The value the operation returned, for the methods that return one. + pub value: Option, + /// The last error, verbatim, for callers that need to tell one refusal from + /// another. [`crate::chaos::deletions`] does: a delete refused because the + /// agent was not there means something quite different from any other + /// refusal. + /// + /// Only ever `Some` on a failure, so the clone costs nothing on the path + /// every scenario actually runs. + pub error: Option, +} + /// Runs one operation with the configured bounded, same-key retry, and records /// it in the history. /// @@ -207,7 +239,8 @@ pub(crate) async fn run_operation( method: &str, key: String, invoke: F, -) where +) -> OperationOutcome +where F: Fn(IdempotencyKey) -> Fut, Fut: std::future::Future>>, { @@ -309,6 +342,7 @@ pub(crate) async fn run_operation( ); } + let error = last.error.as_ref().map(|e| format!("{e:#}")); ctx.history.record(OperationRecord { op_id, stream, @@ -323,10 +357,49 @@ pub(crate) async fn run_operation( duration_ms: started.elapsed().as_millis().min(u64::MAX as u128) as u64, returned_value: last.value, first_attempt_value, - error: last.error.as_ref().map(|e| format!("{e:#}")), + error: error.clone(), error_class: last.class, attempt_log, }); + + OperationOutcome { + outcome, + value: last.value, + error, + } +} + +/// One `Counter.increment`, recorded under `stream`. +/// +/// Split out of [`submit_one`] so [`crate::chaos::reverts`] can drive +/// increments on its own stream through exactly the same retry rule and +/// classification, rather than growing a second copy of them. +pub(crate) async fn increment_counter( + ctx: &WorkloadContext, + stream: Stream, + agent: &str, + key: String, +) -> OperationOutcome { + let parsed: ParsedAgentId = agent_id!(COUNTER_AGENT, agent.to_string()); + let ctx2 = ctx.clone(); + run_operation(ctx, stream, agent.to_string(), "increment", key, |k| { + let ctx = ctx2.clone(); + let parsed = parsed.clone(); + async move { + let value = ctx + .user + .invoke_and_await_agent_with_key( + &ctx.counters, + &parsed, + &k, + "increment", + data_value!(), + ) + .await?; + Ok(as_u32(value)) + } + }) + .await } /// Extracts a `u32` return value, if the agent returned one. Absent values are @@ -481,6 +554,25 @@ pub(crate) async fn submit_one(ctx: &WorkloadContext, stream: Stream, index: u32 Stream::PinnedHttp => { warn!("Chaos mixed workload cannot drive the pinned stream; see chaos::pinned"); } + // Driven by `crate::chaos::waiters` for the same reason: its agents run + // a round at their own pace rather than at a shared rate, and each one + // is blocked until its promise resolves. + Stream::PromiseWait => { + warn!("Chaos mixed workload cannot drive the waiter stream; see chaos::waiters"); + } + // Driven by `crate::chaos::deletions`: a round builds an agent up and + // then deletes it outright, and the value the *next* round's first + // increment returns says whether it stayed deleted. + Stream::Delete => { + warn!("Chaos mixed workload cannot drive the delete stream; see chaos::deletions"); + } + // Driven by `crate::chaos::reverts`: a round is a run of increments + // followed by a revert that takes some of them back, and the value the + // *next* round's first increment returns is what says whether that + // revert landed. A shared rate cannot express that ordering. + Stream::Revert => { + warn!("Chaos mixed workload cannot drive the revert stream; see chaos::reverts"); + } Stream::Durable => { let agent = ctx.agent_name(Stream::Durable, index); let key = ctx.idempotency_key(&agent, seq); @@ -671,15 +763,24 @@ pub(crate) async fn submit_one(ctx: &WorkloadContext, stream: Stream, index: u32 /// A timeout is reported as an unreadable agent rather than propagated: the /// read-back already models "could not be read" as a verdict of its own, and an /// agent that will not answer is exactly that. -async fn read_with_timeout(what: &str, agent: &str, read: F) -> Result +async fn read_with_timeout(what: &str, agent: &str, read: F) -> Result +where + F: std::future::Future>, +{ + read_within(what, agent, READ_TIMEOUT, read).await +} + +/// [`read_with_timeout`] with the ceiling named by the caller, for reads whose +/// answer is much bigger than a counter. +async fn read_within(what: &str, agent: &str, timeout: Duration, read: F) -> Result where - F: std::future::Future>, + F: std::future::Future>, { - match tokio::time::timeout(READ_TIMEOUT, read).await { + match tokio::time::timeout(timeout, read).await { Ok(result) => result, Err(_) => { - warn!("Chaos: reading {what} on {agent} timed out after {READ_TIMEOUT:?}"); - Err(format!("{what} timed out after {READ_TIMEOUT:?}")) + warn!("Chaos: reading {what} on {agent} timed out after {timeout:?}"); + Err(format!("{what} timed out after {timeout:?}")) } } } @@ -780,6 +881,34 @@ pub async fn read_quota_counter(ctx: &WorkloadContext, agent: &str) -> Result Result, String> { + read_within("fires", agent, FIRE_READ_TIMEOUT, async { + let parsed: ParsedAgentId = agent_id!(SCHEDULE_COUNTER_AGENT, agent.to_string()); + match ctx + .user + .invoke_and_await_agent(&ctx.counters, &parsed, "fires", data_value!()) + .await + { + Ok(value) => value + .into_return_value() + .and_then(|v| Vec::<(String, u64, u64)>::from_value(v).ok()) + .ok_or_else(|| "fires returned no readable value".to_string()), + Err(e) => Err(format!("{e:#}")), + } + }) + .await +} + /// Reads back how many scheduled polls actually fired on a target agent. pub async fn read_polls(ctx: &WorkloadContext, agent: &str) -> Result { read_with_timeout("polls", agent, async { diff --git a/test-components/agent-counters-v2/src/lib.rs b/test-components/agent-counters-v2/src/lib.rs index 83e5a4f656..6e448fb3b0 100644 --- a/test-components/agent-counters-v2/src/lib.rs +++ b/test-components/agent-counters-v2/src/lib.rs @@ -317,6 +317,27 @@ impl EphemeralSingletonCounter for EphemeralSingletonCounterImpl { } } +/// Ceiling on the fire log below. +/// +/// Far above what a chaos run produces — a few hundred fires per target — so it +/// only stops a misconfigured cadence from growing agent state without bound. +/// `polls` keeps counting past it, which is what makes truncation visible: a +/// reader that gets fewer entries than polls knows the log is short rather than +/// the fires missing. +const MAX_FIRE_LOG: usize = 10_000; + +/// Wall clock in milliseconds since the epoch. +/// +/// Read inside the agent rather than passed in, because the question S10 asks is +/// when the *platform* ran the action. The read is durable, so an agent that +/// replays reports the original fire time instead of the replay's. +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_millis() as u64) + .unwrap_or(0) +} + /// Near-no-op target for schedule-density. The scheduled action under test is /// dispatching this method, not its guest-side work. #[agent_definition] @@ -328,20 +349,40 @@ trait ScheduleCounter { /// increment is far cheaper than anything the dispatch itself costs. fn poll(&mut self); - /// How many times `poll` has fired. Read after recovery to compare against - /// the number of actions the driver scheduled. + /// How many scheduled actions have fired, whether through `poll` or + /// `fire`. Read after recovery to compare against the number of actions the + /// driver scheduled. fn polls(&self) -> u32; + + /// `poll`, with enough recorded to identify the individual action (GOL-378). + /// + /// S10 kills the executor holding claimed but unacknowledged scheduled + /// actions, and a count cannot answer what it asks. Two questions need the + /// individual action: did *this* registration fire, and how far past its due + /// time did lease recovery put it. `token` is the registering invocation's + /// idempotency key, so a duplicate fire is a repeated token rather than an + /// arithmetic argument about totals. + fn fire(&mut self, token: String, scheduled_millis: u64); + + /// The fire log: one `(token, scheduled_millis, observed_millis)` per fire, + /// in the order the actions ran. + fn fires(&self) -> Vec<(String, u64, u64)>; } struct ScheduleCounterImpl { _id: String, polls: u32, + fires: Vec<(String, u64, u64)>, } #[agent_implementation] impl ScheduleCounter for ScheduleCounterImpl { fn new(id: String) -> Self { - Self { _id: id, polls: 0 } + Self { + _id: id, + polls: 0, + fires: Vec::new(), + } } fn poll(&mut self) { @@ -351,6 +392,17 @@ impl ScheduleCounter for ScheduleCounterImpl { fn polls(&self) -> u32 { self.polls } + + fn fire(&mut self, token: String, scheduled_millis: u64) { + self.polls += 1; + if self.fires.len() < MAX_FIRE_LOG { + self.fires.push((token, scheduled_millis, now_millis())); + } + } + + fn fires(&self) -> Vec<(String, u64, u64)> { + self.fires.clone() + } } /// Schedules no-op polls on durable targets. Keeping scheduling separate from @@ -366,6 +418,14 @@ trait ScheduleEmitter { nanoseconds: u32, context_spans: u32, ); + + /// Registers a `fire` carrying the token that identifies this registration. + /// + /// A second method rather than a payload on `schedule_poll_at`: the density + /// benchmark and the four scenarios already on the poll path measure + /// dispatch cost, and giving their scheduled action an argument and a + /// growing log would change what they measure. + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String); } struct ScheduleEmitterImpl { @@ -396,6 +456,19 @@ impl ScheduleEmitter for ScheduleEmitterImpl { nanoseconds, }); } + + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String) { + let mut target = ScheduleCounterClient::get(target_name); + let scheduled_millis = seconds * 1000 + (nanoseconds / 1_000_000) as u64; + target.schedule_fire( + token, + scheduled_millis, + Datetime { + seconds, + nanoseconds, + }, + ); + } } #[agent_definition(ephemeral)] diff --git a/test-components/agent-counters/src/lib.rs b/test-components/agent-counters/src/lib.rs index 75a4d0511c..8c38771bd1 100644 --- a/test-components/agent-counters/src/lib.rs +++ b/test-components/agent-counters/src/lib.rs @@ -317,6 +317,27 @@ impl EphemeralSingletonCounter for EphemeralSingletonCounterImpl { } } +/// Ceiling on the fire log below. +/// +/// Far above what a chaos run produces — a few hundred fires per target — so it +/// only stops a misconfigured cadence from growing agent state without bound. +/// `polls` keeps counting past it, which is what makes truncation visible: a +/// reader that gets fewer entries than polls knows the log is short rather than +/// the fires missing. +const MAX_FIRE_LOG: usize = 10_000; + +/// Wall clock in milliseconds since the epoch. +/// +/// Read inside the agent rather than passed in, because the question S10 asks is +/// when the *platform* ran the action. The read is durable, so an agent that +/// replays reports the original fire time instead of the replay's. +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_millis() as u64) + .unwrap_or(0) +} + /// Near-no-op target for schedule-density. The scheduled action under test is /// dispatching this method, not its guest-side work. #[agent_definition] @@ -328,20 +349,40 @@ trait ScheduleCounter { /// increment is far cheaper than anything the dispatch itself costs. fn poll(&mut self); - /// How many times `poll` has fired. Read after recovery to compare against - /// the number of actions the driver scheduled. + /// How many scheduled actions have fired, whether through `poll` or + /// `fire`. Read after recovery to compare against the number of actions the + /// driver scheduled. fn polls(&self) -> u32; + + /// `poll`, with enough recorded to identify the individual action (GOL-378). + /// + /// S10 kills the executor holding claimed but unacknowledged scheduled + /// actions, and a count cannot answer what it asks. Two questions need the + /// individual action: did *this* registration fire, and how far past its due + /// time did lease recovery put it. `token` is the registering invocation's + /// idempotency key, so a duplicate fire is a repeated token rather than an + /// arithmetic argument about totals. + fn fire(&mut self, token: String, scheduled_millis: u64); + + /// The fire log: one `(token, scheduled_millis, observed_millis)` per fire, + /// in the order the actions ran. + fn fires(&self) -> Vec<(String, u64, u64)>; } struct ScheduleCounterImpl { _id: String, polls: u32, + fires: Vec<(String, u64, u64)>, } #[agent_implementation] impl ScheduleCounter for ScheduleCounterImpl { fn new(id: String) -> Self { - Self { _id: id, polls: 0 } + Self { + _id: id, + polls: 0, + fires: Vec::new(), + } } fn poll(&mut self) { @@ -351,6 +392,17 @@ impl ScheduleCounter for ScheduleCounterImpl { fn polls(&self) -> u32 { self.polls } + + fn fire(&mut self, token: String, scheduled_millis: u64) { + self.polls += 1; + if self.fires.len() < MAX_FIRE_LOG { + self.fires.push((token, scheduled_millis, now_millis())); + } + } + + fn fires(&self) -> Vec<(String, u64, u64)> { + self.fires.clone() + } } /// Schedules no-op polls on durable targets. Keeping scheduling separate from @@ -366,6 +418,14 @@ trait ScheduleEmitter { nanoseconds: u32, context_spans: u32, ); + + /// Registers a `fire` carrying the token that identifies this registration. + /// + /// A second method rather than a payload on `schedule_poll_at`: the density + /// benchmark and the four scenarios already on the poll path measure + /// dispatch cost, and giving their scheduled action an argument and a + /// growing log would change what they measure. + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String); } struct ScheduleEmitterImpl { @@ -396,6 +456,19 @@ impl ScheduleEmitter for ScheduleEmitterImpl { nanoseconds, }); } + + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String) { + let mut target = ScheduleCounterClient::get(target_name); + let scheduled_millis = seconds * 1000 + (nanoseconds / 1_000_000) as u64; + target.schedule_fire( + token, + scheduled_millis, + Datetime { + seconds, + nanoseconds, + }, + ); + } } #[agent_definition(ephemeral)] diff --git a/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs b/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs index 6767bfd62e..0edc3e3a5d 100644 --- a/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs +++ b/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs @@ -30,3 +30,114 @@ impl PromiseAgent for PromiseAgentImpl { golem_rust::complete_promise(&promise_id, &vec![0; payload_size as usize]) } } + +/// Ceiling on the wakeup log below. +/// +/// Far above what a chaos run produces — a few hundred wakeups per waiter — so +/// it only stops a misconfigured cadence from growing agent state without +/// bound. `wakes` keeps counting past it, which is what makes truncation +/// visible: a reader that gets fewer entries than wakes knows the log is short +/// rather than the wakeups missing. +const MAX_WAKEUP_LOG: usize = 10_000; + +/// Wall clock in milliseconds since the epoch. +/// +/// Read inside the agent rather than passed in, because the question S11 asks is +/// when the *platform* resumed the waiter. The read is durable, so an agent that +/// replays reports the original wakeup time instead of the replay's. +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_millis() as u64) + .unwrap_or(0) +} + +/// A durable agent that parks on a promise and records being woken (GOL-377). +/// +/// [`PromiseAgent`] above creates and completes promises in one breath, which is +/// what the mixed workload's promise stream needs and says nothing about +/// recovery. S11 needs the other half: an agent genuinely suspended on a promise +/// whose executor is then killed, and a durable record of whether the completion +/// ever reached it. +/// +/// The record has to live in agent state rather than in the invocation's return +/// value, because the interesting case is exactly the one where the caller's +/// connection died with the executor. A caller that got an error learns nothing; +/// the log outlives the connection and answers anyway. +#[agent_definition] +pub trait PromiseWaiter { + fn new(name: String) -> Self; + + /// Creates a promise for this waiter and returns it without waiting. + /// + /// Separate from [`PromiseWaiter::wait`] because the completer needs the + /// promise id, and an invocation that blocks cannot return one. `token` is + /// the driver's idempotency key for this round, and is what pairs a wakeup + /// back to the completion that caused it. + fn arm(&mut self, token: String) -> PromiseId; + + /// Blocks until `promise_id` is completed, then records the wakeup. + /// + /// Suspends the agent, so the executor holds no thread for it: this is the + /// state S11 kills an executor in. + fn wait(&mut self, token: String, promise_id: PromiseId); + + /// The wakeup log: one `(token, armed_millis, woken_millis)` per wakeup, in + /// the order the waiter was resumed. + fn wakeups(&self) -> Vec<(String, u64, u64)>; + + /// How many wakeups happened, whether or not the log kept them. + fn wakes(&self) -> u32; +} + +struct PromiseWaiterImpl { + _name: String, + /// When each token was armed, so a wakeup can report the interval it was + /// parked without the driver having to join two clocks. + armed: Vec<(String, u64)>, + wakeups: Vec<(String, u64, u64)>, + wakes: u32, +} + +#[agent_implementation] +impl PromiseWaiter for PromiseWaiterImpl { + fn new(name: String) -> Self { + Self { + _name: name, + armed: Vec::new(), + wakeups: Vec::new(), + wakes: 0, + } + } + + fn arm(&mut self, token: String) -> PromiseId { + if self.armed.len() >= MAX_WAKEUP_LOG { + self.armed.remove(0); + } + self.armed.push((token, now_millis())); + golem_rust::create_promise() + } + + fn wait(&mut self, token: String, promise_id: PromiseId) { + let _ = golem_rust::blocking_await_promise(&promise_id); + self.wakes += 1; + if self.wakeups.len() < MAX_WAKEUP_LOG { + let armed_millis = self + .armed + .iter() + .rev() + .find(|(armed_token, _)| armed_token == &token) + .map(|(_, at)| *at) + .unwrap_or(0); + self.wakeups.push((token, armed_millis, now_millis())); + } + } + + fn wakeups(&self) -> Vec<(String, u64, u64)> { + self.wakeups.clone() + } + + fn wakes(&self) -> u32 { + self.wakes + } +}