Retry identity-service calls through cold starts

app/auth/identity.py

@@ -5,13 +5,21 @@
 does the signed token say this is?". The token is verified against the
 service's public JWKS, so RouteOptimizer never holds a private key.
 """
+import logging
 import os
+import time
 
 import jwt
 import requests
 from jwt import PyJWKClient
 
-TIMEOUT_SECONDS = 5
+TIMEOUT_SECONDS = 20  # the free-tier service can be slow while it wakes
+ATTEMPTS = 3
+RETRY_DELAY_SECONDS = 4
+TRANSIENT_STATUSES = (502, 503, 504)
+WAKING_MESSAGE = "The sign-in service is waking up. Please try again in about a minute."
+
+log = logging.getLogger(__name__)
 
 
 class IdentityError(Exception):
@@ -64,19 +72,36 @@ def verify_access_token(token):
     return claims
 
 
+def _post(path, payload):
+    """POST to identity-service, retrying while it wakes from idle.
+
+    Connection errors, timeouts and 502/503/504 are treated as "still waking"
+    and retried. Any other status is a real answer and is returned untouched
+    for the caller to interpret. Raises IdentityError(WAKING_MESSAGE) once
+    every attempt has failed.
+    """
+    for attempt in range(1, ATTEMPTS + 1):
+        try:
+            response = requests.post(
+                f"{_base_url()}{path}", json=payload, timeout=TIMEOUT_SECONDS
+            )
+        except requests.RequestException as exc:
+            log.warning("identity %s attempt %d failed: %s", path, attempt, exc.__class__.__name__)
+        else:
+            if response.status_code not in TRANSIENT_STATUSES:
+                return response
+            log.warning("identity %s attempt %d: HTTP %d", path, attempt, response.status_code)
+        if attempt < ATTEMPTS:
+            time.sleep(RETRY_DELAY_SECONDS)
+    raise IdentityError(WAKING_MESSAGE)
+
+
 def authenticate(email, password):
     """Sign in against identity-service and return verified access-token claims.
 
     Raises IdentityError with a user-safe message on any failure.
     """
-    try:
-        response = requests.post(
-            f"{_base_url()}/auth/login",
-            json={"email": email, "password": password},
-            timeout=TIMEOUT_SECONDS,
-        )
-    except requests.RequestException as exc:
-        raise IdentityError("The sign-in service is unavailable. Try again shortly.") from exc
+    response = _post("/auth/login", {"email": email, "password": password})
 
     if response.status_code == 400:
         # Identity-service answers bad credentials with 400 and {"error": "..."}.
@@ -102,14 +127,10 @@ def register(name, email, password):
     parts = name.strip().split(None, 1)
     first = parts[0] if parts else "User"
     last = parts[1] if len(parts) > 1 else "-"  # identity-service requires a last name
-    try:
-        response = requests.post(
-            f"{_base_url()}/auth/register",
-            json={"email": email, "password": password, "firstName": first, "lastName": last},
-            timeout=TIMEOUT_SECONDS,
-        )
-    except requests.RequestException as exc:
-        raise IdentityError("The sign-in service is unavailable. Try again shortly.") from exc
+    response = _post(
+        "/auth/register",
+        {"email": email, "password": password, "firstName": first, "lastName": last},
+    )
 
     if response.status_code == 400:
         try:

tests/test_identity_client.py

@@ -0,0 +1,78 @@
+"""Retry behaviour of the identity-service client (no network, no sleeping)."""
+import pytest
+import requests
+
+from app.auth import identity
+
+
+class FakeResponse:
+    def __init__(self, status_code):
+        self.status_code = status_code
+        self.ok = status_code < 400
+
+    def json(self):
+        return {"error": "Bad credentials"}
+
+
+def _script(monkeypatch, outcomes):
+    """Replace requests.post. Each call consumes the next outcome (an int
+    status or an exception); the last outcome repeats forever."""
+    calls = []
+    remaining = list(outcomes)
+
+    def fake_post(url, json=None, timeout=None):
+        calls.append(url)
+        outcome = remaining.pop(0) if len(remaining) > 1 else remaining[0]
+        if isinstance(outcome, Exception):
+            raise outcome
+        return FakeResponse(outcome)
+
+    monkeypatch.setattr(identity.requests, "post", fake_post)
+    return calls
+
+
+@pytest.fixture
+def no_sleep(monkeypatch):
+    slept = []
+    monkeypatch.setattr(identity.time, "sleep", slept.append)
+    return slept
+
+
+def test_first_attempt_success_does_not_sleep(monkeypatch, no_sleep):
+    calls = _script(monkeypatch, [200])
+    assert identity._post("/auth/login", {}).status_code == 200
+    assert len(calls) == 1 and no_sleep == []
+
+
+def test_retries_a_502_then_succeeds(monkeypatch, no_sleep):
+    calls = _script(monkeypatch, [502, 200])
+    assert identity._post("/auth/login", {}).status_code == 200
+    assert len(calls) == 2 and no_sleep == [identity.RETRY_DELAY_SECONDS]
+
+
+def test_retries_connection_errors_and_timeouts(monkeypatch, no_sleep):
+    calls = _script(monkeypatch, [requests.ConnectionError(), requests.Timeout(), 200])
+    assert identity._post("/auth/login", {}).status_code == 200
+    assert len(calls) == 3
+
+
+def test_gives_up_with_the_waking_message(monkeypatch, no_sleep):
+    calls = _script(monkeypatch, [503])
+    with pytest.raises(identity.IdentityError) as exc:
+        identity._post("/auth/login", {})
+    assert str(exc.value) == identity.WAKING_MESSAGE
+    assert len(calls) == identity.ATTEMPTS
+    assert len(no_sleep) == identity.ATTEMPTS - 1
+
+
+@pytest.mark.parametrize("status", [400, 401, 409, 500])
+def test_real_answers_are_not_retried(monkeypatch, no_sleep, status):
+    calls = _script(monkeypatch, [status])
+    assert identity._post("/auth/login", {}).status_code == status
+    assert len(calls) == 1 and no_sleep == []
+
+
+def test_bad_credentials_still_surface_the_service_message(monkeypatch, no_sleep):
+    _script(monkeypatch, [400])
+    with pytest.raises(identity.IdentityError, match="Bad credentials"):
+        identity.authenticate("a@b.co", "wrong")