From e8ad30a98c533424a909ae1e7658878244ee84a2 Mon Sep 17 00:00:00 2001 From: edwh Date: Wed, 2 Sep 2026 14:02:18 +0100 Subject: [PATCH 1/2] Make the production deploy check the site, and restart once if it is down Today's outage: production was pushed at 12:02, deploy-fly-prod built the image and ran a rolling update, the machine came up at 12:08 without passing its health check, and flyctl gave up: Error: failed to update machine 7813112c002178: Unrecoverable error: timeout reached waiting for health checks to pass The job exited 1 and stopped there, leaving the machine started but out of the proxy pool. With one machine that is a total outage, and it stayed down for 43 minutes. Restarting that same image brought it back immediately, so nothing was wrong with the build - the deploy simply had everything it needed to fix this and instead walked away. So don't stop at flyctl's exit code. Ask the question that actually matters - is the site serving? - by fetching robots.txt from outside Fly, the same static path the health check uses, so the answer doesn't depend on the database. If it isn't serving, restart each machine once, re-checking after each so we stop as soon as it recovers. Never loop: a restart that doesn't help means something a restart cannot fix, and that needs a person. A deploy that only passes because of the restart says so loudly. This path existing is not a reason to stop asking why a machine failed to come up healthy. Co-Authored-By: Claude Opus 5 (1M context) --- .circleci/config.yml | 70 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 23053e2d45..5190de0a62 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -225,7 +225,75 @@ jobs: name: Deploy to restarters (production) command: | export FLY_API_TOKEN="FlyV1 $FLY_API_TOKEN" - flyctl deploy --app restarters --remote-only + # Deliberately not failing the job here. A rolling deploy that + # times out waiting for health checks leaves the machine on the new + # image but out of the proxy pool, which with a single machine is a + # total outage - and that is precisely when we must not stop and + # walk away. The verify step below decides whether the site is + # actually serving, and that is the thing we care about. + if flyctl deploy --app restarters --remote-only; then + echo ok > /tmp/deploy-result + else + echo failed > /tmp/deploy-result + echo "flyctl reported failure; the verify step will check whether production is serving." + fi + no_output_timeout: 15m + - run: + name: Verify production is serving, restarting once if not + command: | + export FLY_API_TOKEN="FlyV1 $FLY_API_TOKEN" + + # Probe from outside Fly, on the same path the Fly health check + # uses. robots.txt is static, so a 200 means nginx is up and the + # proxy has a healthy instance to route to - it does not depend on + # the database being reachable. + serving() { + for _ in $(seq 1 "$1"); do + code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 15 \ + https://restarters.net/robots.txt || echo 000) + [ "$code" = "200" ] && return 0 + sleep 10 + done + return 1 + } + + if serving 12; then + echo "Production is serving." + [ "$(cat /tmp/deploy-result)" = failed ] && \ + echo "NOTE: flyctl exited non-zero but the site is up - check the deploy log above." + exit 0 + fi + + echo "==============================================" + echo "Production is NOT serving after the deploy." + echo "==============================================" + flyctl status --app restarters || true + + # On 2026-09-02 a deploy timed out waiting for health checks and + # left the machine started but critical for 43 minutes; restarting + # that same image brought it straight back. Try that once per + # machine before giving up, re-checking after each so we stop as + # soon as the site returns - and never loop, because a restart that + # does not help means something a restart cannot fix. + command -v jq >/dev/null 2>&1 || sudo apt-get install -y jq + for id in $(flyctl machines list --app restarters --json | jq -r '.[].id'); do + echo "Restarting machine $id" + flyctl machine restart "$id" --app restarters || true + if serving 18; then + echo "==============================================" + echo "RECOVERED after restarting $id." + echo "The deploy could not bring this machine up healthy on" + echo "its own - worth finding out why rather than relying on" + echo "this path." + echo "==============================================" + exit 0 + fi + done + + echo "Still not serving after restarting. Production needs a human." + flyctl status --app restarters || true + flyctl logs --app restarters --no-tail 2>&1 | tail -50 || true + exit 1 no_output_timeout: 15m workflows: From 021ae9ef395e0b715edf229b04fd75d5eb36e1ff Mon Sep 17 00:00:00 2001 From: edwh Date: Thu, 3 Sep 2026 16:58:54 +0100 Subject: [PATCH 2/2] Check the app works, not just that it answers The post-deploy probe only fetched robots.txt. That is static, and it is also the path the Fly health check uses - so it reports success in exactly the case it most needs to catch. On 2026-09-03 restarters-dev served robots.txt with a passing health check while every database-backed page returned 500, because it pointed at a database app that no longer existed. This check would have called that deploy fine. Two probes now, answering different questions. robots.txt still asks whether anything is serving, because that is what decides a restart is worth trying. /api/homepage_data - public, cheap, aggregates over the database - asks whether the app actually works, and that is what decides pass or fail, including after a restart. Serving but not working is called out and deliberately does not restart: a restart cannot fix a bad config or an unreachable database, and cycling a machine there would only hide the fault. Co-Authored-By: Claude Opus 5 (1M context) --- .circleci/config.yml | 47 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5190de0a62..1738ec9e98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -243,10 +243,14 @@ jobs: command: | export FLY_API_TOKEN="FlyV1 $FLY_API_TOKEN" - # Probe from outside Fly, on the same path the Fly health check - # uses. robots.txt is static, so a 200 means nginx is up and the - # proxy has a healthy instance to route to - it does not depend on - # the database being reachable. + # Two probes, because they answer different questions. + # + # robots.txt is static: a 200 means nginx is up and the proxy has an + # instance to route to. That is also the path the Fly health check + # uses - which is exactly why it cannot be the only probe. On + # 2026-09-03 restarters-dev served robots.txt, and reported a passing + # health check, while every database-backed page returned 500, + # because it was pointed at a database app that no longer existed. serving() { for _ in $(seq 1 "$1"); do code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 15 \ @@ -257,13 +261,39 @@ jobs: return 1 } - if serving 12; then - echo "Production is serving." + # homepage_data is public, cheap and aggregates over the database, so + # it answers the question that matters: does the app actually work? + working() { + for _ in $(seq 1 "$1"); do + body=$(curl -sS --max-time 20 https://restarters.net/api/homepage_data || true) + case "$body" in *'"items_fixed"'*) return 0 ;; esac + sleep 10 + done + return 1 + } + + if working 12; then + echo "Production is serving and talking to the database." [ "$(cat /tmp/deploy-result)" = failed ] && \ echo "NOTE: flyctl exited non-zero but the site is up - check the deploy log above." exit 0 fi + # Up, but not working: the machine is serving and a restart will not + # fix a bad config or an unreachable database. Restarting here would + # just cycle a healthy-looking machine and hide the real fault. + if serving 3; then + echo "==============================================" + echo "Production is serving but NOT working - the app" + echo "answers robots.txt while /api/homepage_data does" + echo "not. That is a config or database fault, which" + echo "a restart cannot fix. Needs a human." + echo "==============================================" + flyctl status --app restarters || true + flyctl logs --app restarters --no-tail 2>&1 | tail -50 || true + exit 1 + fi + echo "==============================================" echo "Production is NOT serving after the deploy." echo "==============================================" @@ -279,7 +309,8 @@ jobs: for id in $(flyctl machines list --app restarters --json | jq -r '.[].id'); do echo "Restarting machine $id" flyctl machine restart "$id" --app restarters || true - if serving 18; then + # Recovered means working, not merely answering robots.txt. + if working 18; then echo "==============================================" echo "RECOVERED after restarting $id." echo "The deploy could not bring this machine up healthy on" @@ -290,7 +321,7 @@ jobs: fi done - echo "Still not serving after restarting. Production needs a human." + echo "Still not working after restarting. Production needs a human." flyctl status --app restarters || true flyctl logs --app restarters --no-tail 2>&1 | tail -50 || true exit 1