Make the production deploy check the site, and restart once if it is down

.circleci/config.yml

@@ -225,7 +225,106 @@ jobs:
           name: Deploy to restarters (production)
           command: |
             export FLY_API_TOKEN="FlyV1 $FLY_API_TOKEN"
-            flyctl deploy --app restarters --remote-only
+            # Deliberately not failing the job here.  A rolling deploy that
+            # times out waiting for health checks leaves the machine on the new
+            # image but out of the proxy pool, which with a single machine is a
+            # total outage - and that is precisely when we must not stop and
+            # walk away.  The verify step below decides whether the site is
+            # actually serving, and that is the thing we care about.
+            if flyctl deploy --app restarters --remote-only; then
+              echo ok > /tmp/deploy-result
+            else
+              echo failed > /tmp/deploy-result
+              echo "flyctl reported failure; the verify step will check whether production is serving."
+            fi
+          no_output_timeout: 15m
+      - run:
+          name: Verify production is serving, restarting once if not
+          command: |
+            export FLY_API_TOKEN="FlyV1 $FLY_API_TOKEN"
+
+            # Two probes, because they answer different questions.
+            #
+            # robots.txt is static: a 200 means nginx is up and the proxy has an
+            # instance to route to.  That is also the path the Fly health check
+            # uses - which is exactly why it cannot be the only probe.  On
+            # 2026-09-03 restarters-dev served robots.txt, and reported a passing
+            # health check, while every database-backed page returned 500,
+            # because it was pointed at a database app that no longer existed.
+            serving() {
+              for _ in $(seq 1 "$1"); do
+                code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 15 \
+                       https://restarters.net/robots.txt || echo 000)
+                [ "$code" = "200" ] && return 0
+                sleep 10
+              done
+              return 1
+            }
+
+            # homepage_data is public, cheap and aggregates over the database, so
+            # it answers the question that matters: does the app actually work?
+            working() {
+              for _ in $(seq 1 "$1"); do
+                body=$(curl -sS --max-time 20 https://restarters.net/api/homepage_data || true)
+                case "$body" in *'"items_fixed"'*) return 0 ;; esac
+                sleep 10
+              done
+              return 1
+            }
+
+            if working 12; then
+              echo "Production is serving and talking to the database."
+              [ "$(cat /tmp/deploy-result)" = failed ] && \
+                echo "NOTE: flyctl exited non-zero but the site is up - check the deploy log above."
+              exit 0
+            fi
+
+            # Up, but not working: the machine is serving and a restart will not
+            # fix a bad config or an unreachable database.  Restarting here would
+            # just cycle a healthy-looking machine and hide the real fault.
+            if serving 3; then
+              echo "=============================================="
+              echo "Production is serving but NOT working - the app"
+              echo "answers robots.txt while /api/homepage_data does"
+              echo "not.  That is a config or database fault, which"
+              echo "a restart cannot fix.  Needs a human."
+              echo "=============================================="
+              flyctl status --app restarters || true
+              flyctl logs --app restarters --no-tail 2>&1 | tail -50 || true
+              exit 1
+            fi
+
+            echo "=============================================="
+            echo "Production is NOT serving after the deploy."
+            echo "=============================================="
+            flyctl status --app restarters || true
+
+            # On 2026-09-02 a deploy timed out waiting for health checks and
+            # left the machine started but critical for 43 minutes; restarting
+            # that same image brought it straight back.  Try that once per
+            # machine before giving up, re-checking after each so we stop as
+            # soon as the site returns - and never loop, because a restart that
+            # does not help means something a restart cannot fix.
+            command -v jq >/dev/null 2>&1 || sudo apt-get install -y jq
+            for id in $(flyctl machines list --app restarters --json | jq -r '.[].id'); do
+              echo "Restarting machine $id"
+              flyctl machine restart "$id" --app restarters || true
+              # Recovered means working, not merely answering robots.txt.
+              if working 18; then
+                echo "=============================================="
+                echo "RECOVERED after restarting $id."
+                echo "The deploy could not bring this machine up healthy on"
+                echo "its own - worth finding out why rather than relying on"
+                echo "this path."
+                echo "=============================================="
+                exit 0
+              fi
+            done
+
+            echo "Still not working after restarting. Production needs a human."
+            flyctl status --app restarters || true
+            flyctl logs --app restarters --no-tail 2>&1 | tail -50 || true
+            exit 1
           no_output_timeout: 15m
 
 workflows: