Testnet observability: metrics, cold-path probe, health workflow

.github/k8s/sam-box-canary-template.yaml

@@ -59,6 +59,7 @@ spec:
     metadata:
       labels:
         app: box-canary-${ENV_NAME}
+        sam-canary: "true"
     spec:
       serviceAccountName: sam-box-sa
       # nano-init ships as its own image with no shell in it, so the binary is
@@ -84,6 +85,11 @@ spec:
           # the socket's permissions are the credential.
           - "--bind-addr="
           - "--socket-path=/var/run/sam/node.sock"
+          # The one TCP port, and it carries nothing an API token would gate.
+          - "--metrics-addr=0.0.0.0:9090"
+        ports:
+        - containerPort: 9090
+          name: metrics
         resources:
           requests:
             cpu: 50m
@@ -107,6 +113,10 @@ spec:
           - "--sidecar-socket=/var/run/sam/node.sock"
           - "--egress-allow=example.com"
           - "--log-level=debug"
+          - "--metrics-addr=0.0.0.0:9091"
+        ports:
+        - containerPort: 9091
+          name: box-metrics
         resources:
           requests:
             cpu: 20m

.github/k8s/sam-canary-sa-template.yaml

@@ -0,0 +1,8 @@
+# Identity shared by every canary node in the namespace; deploy.yaml's policy
+# seed binds it to the sam-canary role. Applied with the namespaces so it
+# exists before any canary references it.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sam-node-sa
+  namespace: sam-canary-${ENV_NAME}

.github/k8s/sam-monitoring-template.yaml

@@ -0,0 +1,53 @@
+# Scrape targets for Google Managed Prometheus, which GKE runs by default.
+# A PodMonitoring only sees pods in its own namespace, so the canaries get
+# their own below.
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+  name: sam-control-plane-${ENV_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  selector:
+    matchLabels:
+      app: sam-control-plane-${ENV_NAME}
+  endpoints:
+  - port: http
+    path: /metrics
+    interval: 30s
+---
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+  name: sam-router-${ENV_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  selector:
+    matchLabels:
+      app: sam-router-${ENV_NAME}
+  endpoints:
+  - port: metrics
+    path: /metrics
+    interval: 30s
+---
+# Every canary pod carries sam-canary=true; a pod without one of these named
+# ports is simply not a target for that endpoint.
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+  name: sam-canaries-${ENV_NAME}
+  namespace: sam-canary-${ENV_NAME}
+spec:
+  selector:
+    matchLabels:
+      sam-canary: "true"
+  endpoints:
+  - port: metrics
+    path: /metrics
+    interval: 30s
+  - port: box-metrics
+    path: /metrics
+    interval: 30s
+  targetLabels:
+    fromPod:
+    - from: app
+      to: canary

.github/k8s/sam-node-everything-template.yaml

@@ -30,6 +30,7 @@ spec:
     metadata:
       labels:
         app: everything-canary-${ENV_NAME}
+        sam-canary: "true"
     spec:
       serviceAccountName: sam-node-sa
       containers:
@@ -58,8 +59,11 @@ spec:
           - "--insecure-control-plane"
           - "--jwt-path=/var/run/secrets/tokens/sam-token"
           - "--bind-addr=127.0.0.1:8080"
+          - "--metrics-addr=0.0.0.0:9090"
         ports:
         - containerPort: 8080
+        - containerPort: 9090
+          name: metrics
         resources:
           requests:
             cpu: 50m

.github/k8s/sam-node-openclaw-template.yaml

@@ -47,6 +47,7 @@ spec:
     metadata:
       labels:
         app: openclaw-canary-${ENV_NAME}
+        sam-canary: "true"
     spec:
       serviceAccountName: sam-node-sa
       containers:
@@ -114,8 +115,11 @@ spec:
           - "--insecure-control-plane"
           - "--jwt-path=/var/run/secrets/tokens/sam-token"
           - "--bind-addr=127.0.0.1:8080"
+          - "--metrics-addr=0.0.0.0:9090"
         ports:
         - containerPort: 8080
+        - containerPort: 9090
+          name: metrics
         resources:
           requests:
             cpu: 50m

.github/k8s/sam-node-openrouter-template.yaml

@@ -32,6 +32,7 @@ spec:
     metadata:
       labels:
         app: openrouter-canary-${ENV_NAME}
+        sam-canary: "true"
     spec:
       serviceAccountName: sam-node-sa
       containers:
@@ -67,8 +68,11 @@ spec:
           - "--insecure-control-plane"
           - "--jwt-path=/var/run/secrets/tokens/sam-token"
           - "--bind-addr=127.0.0.1:8080"
+          - "--metrics-addr=0.0.0.0:9090"
         ports:
         - containerPort: 8080
+        - containerPort: 9090
+          name: metrics
         resources:
           requests:
             cpu: 500m

.github/k8s/sam-node-vllm-template.yaml

@@ -46,6 +46,7 @@ spec:
     metadata:
       labels:
         app: vllm-canary-${ENV_NAME}
+        sam-canary: "true"
     spec:
       serviceAccountName: sam-node-sa
       nodeSelector:
@@ -107,8 +108,11 @@ spec:
           - "--insecure-control-plane"
           - "--jwt-path=/var/run/secrets/tokens/sam-token"
           - "--bind-addr=127.0.0.1:8080"
+          - "--metrics-addr=0.0.0.0:9090"
         ports:
         - containerPort: 8080
+        - containerPort: 9090
+          name: metrics
         resources:
           requests:
             cpu: 50m