diff --git a/charts/ace/templates/gateway/backend-traffic-policy.yaml b/charts/ace/templates/gateway/backend-traffic-policy.yaml new file mode 100644 index 000000000..3aee50bdc --- /dev/null +++ b/charts/ace/templates/gateway/backend-traffic-policy.yaml @@ -0,0 +1,34 @@ +{{ if (index .Values "gateway" "enabled") }} + +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: {{ include "ace.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: {{ include "ace.fullname" . }} + retry: + numRetries: 2 + retryOn: + # Only triggers where the backend provably never saw the request, so retrying a + # non-idempotent POST (/api/v1/register, /api/v1/license/issue) stays safe. + triggers: + - connect-failure + - refused-stream + - reset-before-request + perRetry: + backOff: + baseInterval: 100ms + maxInterval: 1s + healthCheck: + passive: + # Eject a replica that is failing but still listed in the endpoints, which readiness + # alone takes up to 30s to do. + consecutiveGatewayErrors: 3 + interval: 5s + baseEjectionTime: 30s + maxEjectionPercent: 50 +{{ end }} diff --git a/charts/ace/templates/gateway/route-main.yaml b/charts/ace/templates/gateway/route-main.yaml index 29ebef165..48373c798 100644 --- a/charts/ace/templates/gateway/route-main.yaml +++ b/charts/ace/templates/gateway/route-main.yaml @@ -38,9 +38,11 @@ spec: namespace: {{ .Release.Namespace }} port: 80 weight: 1 - timeouts: # They default to 15s. - backendRequest: 2m - request: 2m + # Kept under Cloudflare's 100s origin timeout so a slow backend surfaces as a + # gateway 504 rather than a Cloudflare 520/524. They default to 15s. + timeouts: + backendRequest: 55s + request: 55s - matches: - path: type: PathPrefix @@ -53,8 +55,8 @@ spec: port: 80 weight: 1 timeouts: - backendRequest: 2m - request: 2m + backendRequest: 55s + request: 55s - matches: - path: type: PathPrefix @@ -101,8 +103,8 @@ spec: port: 80 weight: 1 timeouts: - backendRequest: 2m - request: 2m + backendRequest: 55s + request: 55s {{- end }} {{- if (index .Values "perses" "enabled") }} - matches: @@ -117,8 +119,8 @@ spec: port: 8080 weight: 1 timeouts: - backendRequest: 2m - request: 2m + backendRequest: 55s + request: 55s {{- end }} - matches: - path: @@ -132,6 +134,6 @@ spec: port: 4000 weight: 1 timeouts: - backendRequest: 2m - request: 2m + backendRequest: 55s + request: 55s {{- end }} diff --git a/charts/platform-api/templates/statefulset.yaml b/charts/platform-api/templates/statefulset.yaml index f1a08d28c..515fc4e3a 100644 --- a/charts/platform-api/templates/statefulset.yaml +++ b/charts/platform-api/templates/statefulset.yaml @@ -68,16 +68,26 @@ spec: port: http # wait for cert and nats to be ready initialDelaySeconds: 60 + timeoutSeconds: 5 failureThreshold: 18 periodSeconds: 30 livenessProbe: httpGet: path: /api/v1/version port: http + # /api/v1/version is served by the same process as every other request, so it + # stalls whenever the process does. Restart only a genuinely wedged process + # (~2m); the 1s/30s default kills all replicas together on transient slowness. + timeoutSeconds: 10 + periodSeconds: 20 + failureThreshold: 6 readinessProbe: httpGet: path: /api/v1/version port: http + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 3 resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.envFrom }} diff --git a/charts/service-gateway/templates/gateway/gwclass.yaml b/charts/service-gateway/templates/gateway/gwclass.yaml index 45e769fd3..6988e84cc 100644 --- a/charts/service-gateway/templates/gateway/gwclass.yaml +++ b/charts/service-gateway/templates/gateway/gwclass.yaml @@ -3,6 +3,7 @@ "vaultServer" .Values.vaultServer "frontendTLSSecretRef" (dict "name" (printf "%s-gw-cert" (include "tenant.name" .)) "namespace" .Release.Namespace) }} +{{- $multiReplica := and (eq .Values.envoy.provisionerType "Deployment") (gt (int (.Values.envoy.replicas | default 1)) 1) }} apiVersion: gateway.networking.k8s.io/v1beta1 kind: GatewayClass metadata: @@ -57,10 +58,24 @@ spec: securityContext: {{- toYaml .Values.envoy.securityContext | nindent 12 }} {{- end }} - {{- if .Values.envoy.nodeSelector }} + {{- if or .Values.envoy.nodeSelector $multiReplica }} pod: + {{- with .Values.envoy.nodeSelector }} nodeSelector: - {{- toYaml .Values.envoy.nodeSelector | nindent 12 }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if $multiReplica }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + # ScheduleAnyway so a cluster with fewer nodes than replicas still schedules. + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: envoy + app.kubernetes.io/component: proxy + gateway.envoyproxy.io/owning-gatewayclass: {{ include "tenant.name" . }} + {{- end }} {{- end }} patch: value: @@ -77,6 +92,11 @@ spec: {{- toYaml .Values.envoy.securityContext | nindent 22 }} {{- end }} + {{- if $multiReplica }} + envoyPDB: + maxUnavailable: 1 + {{- end }} + envoyService: {{- $extraDomains := list }} {{- if and (eq (dig "mode" "" (.Values.gatewayClass.annotations | default dict)) "hosted") (eq .Values.infra.host "appscode.com") }}