diff --git a/.circleci/config.yml b/.circleci/config.yml index 23053e2d45..1738ec9e98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -225,7 +225,106 @@ jobs: name: Deploy to restarters (production) command: | export FLY_API_TOKEN="FlyV1 $FLY_API_TOKEN" - flyctl deploy --app restarters --remote-only + # Deliberately not failing the job here. A rolling deploy that + # times out waiting for health checks leaves the machine on the new + # image but out of the proxy pool, which with a single machine is a + # total outage - and that is precisely when we must not stop and + # walk away. The verify step below decides whether the site is + # actually serving, and that is the thing we care about. + if flyctl deploy --app restarters --remote-only; then + echo ok > /tmp/deploy-result + else + echo failed > /tmp/deploy-result + echo "flyctl reported failure; the verify step will check whether production is serving." + fi + no_output_timeout: 15m + - run: + name: Verify production is serving, restarting once if not + command: | + export FLY_API_TOKEN="FlyV1 $FLY_API_TOKEN" + + # Two probes, because they answer different questions. + # + # robots.txt is static: a 200 means nginx is up and the proxy has an + # instance to route to. That is also the path the Fly health check + # uses - which is exactly why it cannot be the only probe. On + # 2026-09-03 restarters-dev served robots.txt, and reported a passing + # health check, while every database-backed page returned 500, + # because it was pointed at a database app that no longer existed. + serving() { + for _ in $(seq 1 "$1"); do + code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 15 \ + https://restarters.net/robots.txt || echo 000) + [ "$code" = "200" ] && return 0 + sleep 10 + done + return 1 + } + + # homepage_data is public, cheap and aggregates over the database, so + # it answers the question that matters: does the app actually work? + working() { + for _ in $(seq 1 "$1"); do + body=$(curl -sS --max-time 20 https://restarters.net/api/homepage_data || true) + case "$body" in *'"items_fixed"'*) return 0 ;; esac + sleep 10 + done + return 1 + } + + if working 12; then + echo "Production is serving and talking to the database." + [ "$(cat /tmp/deploy-result)" = failed ] && \ + echo "NOTE: flyctl exited non-zero but the site is up - check the deploy log above." + exit 0 + fi + + # Up, but not working: the machine is serving and a restart will not + # fix a bad config or an unreachable database. Restarting here would + # just cycle a healthy-looking machine and hide the real fault. + if serving 3; then + echo "==============================================" + echo "Production is serving but NOT working - the app" + echo "answers robots.txt while /api/homepage_data does" + echo "not. That is a config or database fault, which" + echo "a restart cannot fix. Needs a human." + echo "==============================================" + flyctl status --app restarters || true + flyctl logs --app restarters --no-tail 2>&1 | tail -50 || true + exit 1 + fi + + echo "==============================================" + echo "Production is NOT serving after the deploy." + echo "==============================================" + flyctl status --app restarters || true + + # On 2026-09-02 a deploy timed out waiting for health checks and + # left the machine started but critical for 43 minutes; restarting + # that same image brought it straight back. Try that once per + # machine before giving up, re-checking after each so we stop as + # soon as the site returns - and never loop, because a restart that + # does not help means something a restart cannot fix. + command -v jq >/dev/null 2>&1 || sudo apt-get install -y jq + for id in $(flyctl machines list --app restarters --json | jq -r '.[].id'); do + echo "Restarting machine $id" + flyctl machine restart "$id" --app restarters || true + # Recovered means working, not merely answering robots.txt. + if working 18; then + echo "==============================================" + echo "RECOVERED after restarting $id." + echo "The deploy could not bring this machine up healthy on" + echo "its own - worth finding out why rather than relying on" + echo "this path." + echo "==============================================" + exit 0 + fi + done + + echo "Still not working after restarting. Production needs a human." + flyctl status --app restarters || true + flyctl logs --app restarters --no-tail 2>&1 | tail -50 || true + exit 1 no_output_timeout: 15m workflows: