feat(runner): microVM boot canary startup preflight (RIG-3148)

go/cmd/compass-runner/main.go

@@ -194,6 +194,14 @@ type podmanPreflighter interface {
 	VerifyUsernsRemapSupport(ctx context.Context) error
 }
 
+// canaryBooter is the microVM backend's dynamic host-capability probe: it really
+// boots a throwaway VM through the backend's own verbs, proving the whole boot
+// chain. Kept a DISTINCT single-method interface from microVMPreflighter (not a
+// widened two-method probe) so the single-method-probe discipline holds.
+type canaryBooter interface {
+	BootCanary(ctx context.Context) (runtime.CanaryReport, error)
+}
+
 // verifyBackendPreflight runs the selected engine's static host-capability
 // preflight. It dispatches on the engine's concrete type, first match wins,
 // probing the microVM backend before podman; no engine satisfies both today, so
@@ -204,14 +212,38 @@ type podmanPreflighter interface {
 func verifyBackendPreflight(ctx context.Context, engine runtime.ContainerRuntime) error {
 	switch e := engine.(type) {
 	case microVMPreflighter:
-		return e.VerifyMicroVMSupport(ctx)
+		return runMicroVMPreflight(ctx, e, engine)
 	case podmanPreflighter:
 		return e.VerifyUsernsRemapSupport(ctx)
 	default:
 		return fmt.Errorf("backend %T exposes no startup preflight probe", engine)
 	}
 }
 
+// runMicroVMPreflight runs the microVM backend's two-stage startup gate: the
+// static VerifyMicroVMSupport check, then — only once it passes — the dynamic
+// BootCanary, logging the returned CanaryReport at info. A microVM engine that
+// satisfies microVMPreflighter but not canaryBooter is a fail-closed startup
+// error naming the type, never a silent skip (same posture as the neither-probe
+// default). Split out so verifyBackendPreflight stays within funlen.
+func runMicroVMPreflight(ctx context.Context, pre microVMPreflighter, engine runtime.ContainerRuntime) error {
+	if err := pre.VerifyMicroVMSupport(ctx); err != nil {
+		return err
+	}
+	canary, ok := engine.(canaryBooter)
+	if !ok {
+		return fmt.Errorf("microVM backend %T exposes no boot canary probe", engine)
+	}
+	report, err := canary.BootCanary(ctx)
+	if err != nil {
+		return err
+	}
+	slog.Info("microvm boot canary passed",
+		"boot_latency", report.BootLatency,
+		"guest_rss_bytes", report.GuestRSSBytes)
+	return nil
+}
+
 // setupOtel installs the tracer and meter providers off the env-only OTLP
 // endpoint, returning one shutdown that flushes both. When
 // OTEL_EXPORTER_OTLP_ENDPOINT is empty the providers are no-ops and the shutdown

go/cmd/compass-runner/main_test.go

@@ -20,6 +20,7 @@ import (
 var (
 	_ microVMPreflighter = (*runtime.MicroVMRuntime)(nil)
 	_ podmanPreflighter  = (*runtime.PodmanCLI)(nil)
+	_ canaryBooter       = (*runtime.MicroVMRuntime)(nil)
 )
 
 // parseMount is the operator surface for --mount: a malformed value must be
@@ -86,14 +87,41 @@ func (e podmanOnlyEngine) VerifyUsernsRemapSupport(context.Context) error {
 	return e.err
 }
 
-// microVMOnlyEngine exposes only the microVM probe.
+// microVMOnlyEngine exposes the microVM static probe AND the canary probe — a
+// real microVM engine satisfies both, and the gate now runs the canary after the
+// static check passes, so a fake missing BootCanary would trip the fail-closed
+// canary assertion rather than exercise the static-probe dispatch.
 type microVMOnlyEngine struct {
+	runtime.ContainerRuntime
+	called       *bool
+	canaryCalled *bool
+	err          error
+	report       runtime.CanaryReport
+	canaryErr    error
+}
+
+func (e microVMOnlyEngine) VerifyMicroVMSupport(context.Context) error {
+	*e.called = true
+	return e.err
+}
+
+func (e microVMOnlyEngine) BootCanary(context.Context) (runtime.CanaryReport, error) {
+	if e.canaryCalled != nil {
+		*e.canaryCalled = true
+	}
+	return e.report, e.canaryErr
+}
+
+// microVMNoCanaryEngine exposes ONLY the static microVM probe, not the canary —
+// a microVM backend that cannot boot-canary. The gate must fail closed on it,
+// naming the type, never silently skipping the canary.
+type microVMNoCanaryEngine struct {
 	runtime.ContainerRuntime
 	called *bool
 	err    error
 }
 
-func (e microVMOnlyEngine) VerifyMicroVMSupport(context.Context) error {
+func (e microVMNoCanaryEngine) VerifyMicroVMSupport(context.Context) error {
 	*e.called = true
 	return e.err
 }
@@ -123,6 +151,10 @@ func (e bothProbesEngine) VerifyUsernsRemapSupport(context.Context) error {
 	return e.err
 }
 
+func (e bothProbesEngine) BootCanary(context.Context) (runtime.CanaryReport, error) {
+	return runtime.CanaryReport{}, nil
+}
+
 // verifyBackendPreflight dispatches on the selected engine's concrete type
 // (RIG-2496): microVM first, then podman, first match wins; the matched probe
 // runs and its error is returned verbatim; an engine exposing neither probe is a
@@ -141,14 +173,53 @@ func TestVerifyBackendPreflight(t *testing.T) {
 		}
 	})
 
-	t.Run("microvm probe dispatched", func(t *testing.T) {
-		called := false
-		err := verifyBackendPreflight(context.Background(), microVMOnlyEngine{called: &called})
+	t.Run("microvm static probe then canary dispatched", func(t *testing.T) {
+		called, canaryCalled := false, false
+		err := verifyBackendPreflight(context.Background(),
+			microVMOnlyEngine{called: &called, canaryCalled: &canaryCalled})
 		if err != nil {
 			t.Fatalf("verifyBackendPreflight = %v, want nil", err)
 		}
 		if !called {
-			t.Error("microVM probe was not called")
+			t.Error("microVM static probe was not called")
+		}
+		if !canaryCalled {
+			t.Error("boot canary was not called after the static probe passed")
+		}
+	})
+
+	t.Run("static probe error skips the canary", func(t *testing.T) {
+		called, canaryCalled := false, false
+		err := verifyBackendPreflight(context.Background(),
+			microVMOnlyEngine{called: &called, canaryCalled: &canaryCalled, err: sentinel})
+		if !errors.Is(err, sentinel) {
+			t.Fatalf("verifyBackendPreflight = %v, want the sentinel error", err)
+		}
+		if canaryCalled {
+			t.Error("boot canary ran after the static probe failed")
+		}
+	})
+
+	t.Run("canary error returned verbatim", func(t *testing.T) {
+		called := false
+		err := verifyBackendPreflight(context.Background(),
+			microVMOnlyEngine{called: &called, canaryErr: sentinel})
+		if !errors.Is(err, sentinel) {
+			t.Errorf("verifyBackendPreflight = %v, want the canary sentinel error", err)
+		}
+	})
+
+	t.Run("microVM without canary is fail-closed naming the type", func(t *testing.T) {
+		called := false
+		err := verifyBackendPreflight(context.Background(), microVMNoCanaryEngine{called: &called})
+		if err == nil {
+			t.Fatal("verifyBackendPreflight = nil, want a fail-closed canary refusal")
+		}
+		if !called {
+			t.Error("microVM static probe was not called")
+		}
+		if !strings.Contains(err.Error(), "microVMNoCanaryEngine") {
+			t.Errorf("error %q does not name the engine type", err)
 		}
 	})
 

go/internal/microvmtest/canary_microvm_test.go

@@ -22,7 +22,10 @@
 // microVM is V2a's job and needs the runtime this record does not own. Asserting
 // the resolved Env is fully populated and the two image paths exist on disk is
 // the strongest claim this slice can make WITHOUT a boot — and it is a real
-// assertion, never a skip-always stub.
+// assertion, never a skip-always stub. This is distinct from the V5 boot canary,
+// runtime.(*MicroVMRuntime).BootCanary (microvm_preflight.go), which DOES do a
+// real Create→Start→Exec→Remove boot as the microVM startup preflight; despite
+// the shared "canary" word the two are unrelated artifacts (record §(g)).
 //
 // It lives in the EXTERNAL test package `microvmtest_test` (not in-package) and
 // calls the EXPORTED microvmtest.Require, for two reasons that both matter:

go/internal/runtime/boot_canary_microvm_test.go

@@ -0,0 +1,67 @@
+//go:build microvm && unix
+
+package runtime
+
+// The KVM-gated BootCanary e2e (record §(e)/W3 test cycle): it drives the real
+// (*MicroVMRuntime).BootCanary against live hardware, proving the whole boot
+// chain — KVM, vsock, image, guest supervisor, exec gate — end to end, and that
+// the canary owns its own teardown (no orphan session, no leftover runtime dir).
+// It calls microvmtest.Require(t) first (skip-on-absent-KVM, hard-fail under
+// COMPASS_REQUIRE_MICROVM=1) and passes t.Context() as the caller ctx, mirroring
+// TestMicroVMQBudget + e2eConfig. It rides the existing CI microVM leg
+// (go test -tags microvm -race -timeout 15m ./...), budgeted well inside 15m.
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/RigelBuild/compass/go/internal/microvmtest"
+)
+
+// TestBootCanary boots a real canary VM through BootCanary and asserts the report
+// is populated (BootLatency in (0, canaryDeadline], GuestRSSBytes > 0) and that
+// BootCanary tore down its own VM and runtime dir — nothing left in the session
+// table and no leftover <RunRoot>/microvm/* dir. Unlike TestMicroVMQBudget (which
+// Removes in cleanup), the canary owns its teardown, so this asserts it happened.
+func TestBootCanary(t *testing.T) {
+	env := microvmtest.Require(t)
+	cfg := e2eConfig(t, env)
+	m := NewMicroVMRuntime(cfg)
+
+	report, err := m.BootCanary(t.Context())
+	if err != nil {
+		t.Fatalf("BootCanary = %v, want nil", err)
+	}
+	if report.BootLatency <= 0 {
+		t.Errorf("BootLatency = %v, want > 0", report.BootLatency)
+	}
+	if report.BootLatency > canaryDeadline {
+		t.Errorf("BootLatency = %v, want <= the %v canary deadline", report.BootLatency, canaryDeadline)
+	}
+	if report.GuestRSSBytes <= 0 {
+		t.Errorf("GuestRSSBytes = %d, want > 0", report.GuestRSSBytes)
+	}
+	t.Logf("BootCanary: boot latency = %s, guest PSS = %d bytes", report.BootLatency, report.GuestRSSBytes)
+
+	// BootCanary owns its teardown: no session leaked in the table.
+	m.mu.Lock()
+	n := len(m.sessions)
+	m.mu.Unlock()
+	if n != 0 {
+		t.Errorf("session table has %d entries after BootCanary, want 0 (canary must tear down its own session)", n)
+	}
+
+	// And no per-session runtime dir left under <RunRoot>/microvm.
+	microvmDir := filepath.Join(cfg.RunRoot, "microvm")
+	entries, statErr := os.ReadDir(microvmDir)
+	if statErr != nil {
+		if os.IsNotExist(statErr) {
+			return // never created, or fully removed — both fine
+		}
+		t.Fatalf("reading %s: %v", microvmDir, statErr)
+	}
+	if len(entries) != 0 {
+		t.Errorf("%s has %d leftover session dirs after BootCanary, want 0", microvmDir, len(entries))
+	}
+}