Harden the appscode.com serving path against intermittent 5xx

charts/ace/templates/gateway/backend-traffic-policy.yaml

@@ -0,0 +1,34 @@
+{{ if (index .Values "gateway" "enabled") }}
+
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: BackendTrafficPolicy
+metadata:
+  name: {{ include "ace.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  targetRefs:
+  - group: gateway.networking.k8s.io
+    kind: HTTPRoute
+    name: {{ include "ace.fullname" . }}
+  retry:
+    numRetries: 2
+    retryOn:
+      # Only triggers where the backend provably never saw the request, so retrying a
+      # non-idempotent POST (/api/v1/register, /api/v1/license/issue) stays safe.
+      triggers:
+      - connect-failure
+      - refused-stream
+      - reset-before-request
+    perRetry:
+      backOff:
+        baseInterval: 100ms
+        maxInterval: 1s
+  healthCheck:
+    passive:
+      # Eject a replica that is failing but still listed in the endpoints, which readiness
+      # alone takes up to 30s to do.
+      consecutiveGatewayErrors: 3
+      interval: 5s
+      baseEjectionTime: 30s
+      maxEjectionPercent: 50
+{{ end }}

charts/ace/templates/gateway/route-main.yaml

@@ -38,9 +38,11 @@ spec:
       namespace: {{ .Release.Namespace }}
       port: 80
       weight: 1
-    timeouts: # They default to 15s.
-      backendRequest: 2m
-      request: 2m
+    # Kept under Cloudflare's 100s origin timeout so a slow backend surfaces as a
+    # gateway 504 rather than a Cloudflare 520/524. They default to 15s.
+    timeouts:
+      backendRequest: 55s
+      request: 55s
   - matches:
     - path:
         type: PathPrefix
@@ -53,8 +55,8 @@ spec:
       port: 80
       weight: 1
     timeouts:
-      backendRequest: 2m
-      request: 2m
+      backendRequest: 55s
+      request: 55s
   - matches:
     - path:
         type: PathPrefix
@@ -101,8 +103,8 @@ spec:
       port: 80
       weight: 1
     timeouts:
-      backendRequest: 2m
-      request: 2m
+      backendRequest: 55s
+      request: 55s
   {{- end }}
   {{- if (index .Values "perses" "enabled") }}
   - matches:
@@ -117,8 +119,8 @@ spec:
       port: 8080
       weight: 1
     timeouts:
-      backendRequest: 2m
-      request: 2m
+      backendRequest: 55s
+      request: 55s
   {{- end }}
   - matches:
     - path:
@@ -132,6 +134,6 @@ spec:
       port: 4000
       weight: 1
     timeouts:
-      backendRequest: 2m
-      request: 2m
+      backendRequest: 55s
+      request: 55s
 {{- end }}

charts/platform-api/templates/statefulset.yaml

@@ -68,16 +68,26 @@ spec:
               port: http
             # wait for cert and nats to be ready
             initialDelaySeconds: 60
+            timeoutSeconds: 5
             failureThreshold: 18
             periodSeconds: 30
           livenessProbe:
             httpGet:
               path: /api/v1/version
               port: http
+            # /api/v1/version is served by the same process as every other request, so it
+            # stalls whenever the process does. Restart only a genuinely wedged process
+            # (~2m); the 1s/30s default kills all replicas together on transient slowness.
+            timeoutSeconds: 10
+            periodSeconds: 20
+            failureThreshold: 6
           readinessProbe:
             httpGet:
               path: /api/v1/version
               port: http
+            timeoutSeconds: 5
+            periodSeconds: 10
+            failureThreshold: 3
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           {{- with .Values.envFrom }}

charts/service-gateway/templates/gateway/gwclass.yaml

@@ -3,6 +3,7 @@
   "vaultServer" .Values.vaultServer
   "frontendTLSSecretRef" (dict "name" (printf "%s-gw-cert" (include "tenant.name" .)) "namespace" .Release.Namespace)
 }}
+{{- $multiReplica := and (eq .Values.envoy.provisionerType "Deployment") (gt (int (.Values.envoy.replicas | default 1)) 1) }}
 apiVersion: gateway.networking.k8s.io/v1beta1
 kind: GatewayClass
 metadata:
@@ -57,10 +58,24 @@ spec:
           securityContext:
             {{- toYaml .Values.envoy.securityContext | nindent 12 }}
           {{- end }}
-        {{- if .Values.envoy.nodeSelector }}
+        {{- if or .Values.envoy.nodeSelector $multiReplica }}
         pod:
+          {{- with .Values.envoy.nodeSelector }}
           nodeSelector:
-            {{- toYaml .Values.envoy.nodeSelector | nindent 12 }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- if $multiReplica }}
+          topologySpreadConstraints:
+          - maxSkew: 1
+            topologyKey: kubernetes.io/hostname
+            # ScheduleAnyway so a cluster with fewer nodes than replicas still schedules.
+            whenUnsatisfiable: ScheduleAnyway
+            labelSelector:
+              matchLabels:
+                app.kubernetes.io/name: envoy
+                app.kubernetes.io/component: proxy
+                gateway.envoyproxy.io/owning-gatewayclass: {{ include "tenant.name" . }}
+          {{- end }}
         {{- end }}
         patch:
           value:
@@ -77,6 +92,11 @@ spec:
                       {{- toYaml .Values.envoy.securityContext | nindent 22 }}
                     {{- end }}
 
+      {{- if $multiReplica }}
+      envoyPDB:
+        maxUnavailable: 1
+      {{- end }}
+
       envoyService:
         {{- $extraDomains := list }}
         {{- if and (eq (dig "mode" "" (.Values.gatewayClass.annotations | default dict)) "hosted") (eq .Values.infra.host "appscode.com") }}