From 01f98ba88b7421b44011364c90e658f53f17eb70 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sun, 6 Sep 2026 15:02:36 -0600 Subject: [PATCH 01/18] Embed sush in sled-agent This begins the integration of the [Support Shell](https://github.com/oxidecomputer/sush) ([RFD 620](https://rfd.shared.oxide.computer/rfd/0620)) service into the wider control plane, starting with sled-agent. Sled-agent now builds the sush job manager during bootstrap as a long-running task and serves its HTTP API on the bootstrap network from the start. Once the sled has an underlay address, it serves the same API there, too. In addition to its client-facing API, sush also gossips sessions, jobs, and events over sprockets on the bootstrap network using [rumors](https://github.com/oxidecomputer/rumors). Gossip peers are discovered using the bootstore's existing DDM polling loop, and a map polled from MGS resolves job targets given as cubby numbers. The records sush must trust across reboots (its gossip identity and its execution boundary) live on the M.2s' cluster datasets. Job output is recorded on the ramdisk until an encrypted debug dataset is mounted, then moves there with a raised size limit. The signatures on all jobs must chain to one of the configured certs; the cert here is a sandbox root which **must be replaced with a production cert before shipping** (see oxidecomputer/sush#46). Co-Authored-By: Claude Mythos 5 --- Cargo.lock | 376 ++++++++++++++- Cargo.toml | 24 +- clients/ddm-admin-client/src/lib.rs | 21 + common/src/address.rs | 2 + dev-tools/ls-apis/tests/api_dependencies.out | 2 +- sled-agent/Cargo.toml | 7 +- sled-agent/config-reconciler/src/handle.rs | 22 + sled-agent/src/bootstrap/bootstore_setup.rs | 17 +- sled-agent/src/config.rs | 78 ++++ sled-agent/src/lib.rs | 1 + sled-agent/src/long_running_tasks.rs | 49 +- sled-agent/src/server.rs | 30 +- sled-agent/src/sush.rs | 427 ++++++++++++++++++ smf/sled-agent/gimlet-standalone/config.toml | 5 + .../gimlet-standalone/sush-sandbox.pem | 11 + smf/sled-agent/gimlet/config.toml | 5 + smf/sled-agent/gimlet/sush-sandbox.pem | 11 + smf/sled-agent/non-gimlet/config.toml | 5 + smf/sled-agent/non-gimlet/sush-sandbox.pem | 11 + workspace-hack/Cargo.toml | 40 +- 20 files changed, 1098 insertions(+), 46 deletions(-) create mode 100644 sled-agent/src/sush.rs create mode 100644 smf/sled-agent/gimlet-standalone/sush-sandbox.pem create mode 100644 smf/sled-agent/gimlet/sush-sandbox.pem create mode 100644 smf/sled-agent/non-gimlet/sush-sandbox.pem diff --git a/Cargo.lock b/Cargo.lock index d65c05264e3..69566a16986 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -656,6 +656,21 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "before" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rumors?rev=be1d76947f75f27e7f53be438a93d350019dd026#be1d76947f75f27e7f53be438a93d350019dd026" +dependencies = [ + "bytes", + "dashu-int", + "dsi-bitstream", + "serde", + "serde_json", + "static_assertions", + "suanpan", + "thiserror 2.0.18", +] + [[package]] name = "bhyve_api" version = "0.0.0" @@ -1024,6 +1039,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bytesize" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" + [[package]] name = "bzip2" version = "0.4.4" @@ -2571,6 +2592,25 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dashu-base" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64b04cdfc4c8533100fe00304eb9687173bda47b1f1dac8af12ba13712ed49d" + +[[package]] +name = "dashu-int" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ee98721d5d223e5b64b642dd9588b79d9ef415554b13720308b77d628c3be6" +dependencies = [ + "cfg-if", + "dashu-base", + "num-modular", + "rustversion", + "static_assertions", +] + [[package]] name = "data-encoding" version = "2.10.0" @@ -3440,6 +3480,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dsi-bitstream" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f3e8a7ab0f1148cf7a792807d330f1e417dda3f4f122526e609a83925b4823" +dependencies = [ + "num-primitive", +] + [[package]] name = "dsl_auto_type" version = "0.2.0" @@ -4759,6 +4808,11 @@ name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "headers" @@ -5104,6 +5158,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573" +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.10.1" @@ -6748,6 +6808,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c" +dependencies = [ + "hashbrown 0.17.0", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -8568,6 +8637,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-modular" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd8e500409e6cd603b03e477c26a6caecdc27ac58979a53e881c75eafc079f44" + +[[package]] +name = "num-primitive" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0178502a58a2514f927965f80216c0ea2533b5364061544ea8e09b79343d4183" + [[package]] name = "num-rational" version = "0.4.2" @@ -9702,6 +9783,8 @@ dependencies = [ "fmd-adm", "fmd-adm-sys", "futures", + "gateway-client", + "gateway-types", "glob", "guppy", "hex", @@ -9785,6 +9868,9 @@ dependencies = [ "sprockets-tls", "strum 0.27.2", "subprocess", + "sush-api", + "sush-common", + "sush-server", "tar", "test-strategy", "thiserror 2.0.18", @@ -9879,6 +9965,7 @@ dependencies = [ "base64ct", "bitflags 1.3.2", "bitflags 2.11.0", + "blake3", "bstr", "buf-list", "byte-wrapper", @@ -9895,6 +9982,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "crossterm 0.28.1", + "crypto-bigint 0.5.5", "crypto-common 0.1.7", "crypto-common 0.2.2", "ctutils", @@ -9904,6 +9992,7 @@ dependencies = [ "der 0.7.10", "derive_more 2.1.1", "derive_more-impl", + "dice-verifier 0.3.0-pre0 (git+https://github.com/oxidecomputer/dice-util?rev=4a39ef08d81e5177edee0bddb1146032aa21074d)", "digest 0.10.7", "digest 0.11.3", "dof 0.3.0", @@ -9931,6 +10020,7 @@ dependencies = [ "group 0.13.0", "hashbrown 0.15.5", "hashbrown 0.16.1", + "hashbrown 0.17.0", "heck 0.4.1", "hex", "hickory-proto 0.25.2", @@ -9950,6 +10040,7 @@ dependencies = [ "lalrpop-util", "lazy_static", "libc", + "linux-raw-sys 0.12.1", "linux-raw-sys 0.4.15", "log", "managed", @@ -10007,6 +10098,7 @@ dependencies = [ "smallvec 1.15.1", "spin", "spki 0.7.3", + "sprockets-tls", "string_cache", "strum 0.26.3", "strum 0.27.2", @@ -10786,6 +10878,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", + "sha2 0.10.9", +] + [[package]] name = "p256" version = "0.14.0" @@ -10825,6 +10929,20 @@ dependencies = [ "sha2 0.11.0", ] +[[package]] +name = "p521" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" +dependencies = [ + "base16ct 0.2.0", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", + "rand_core 0.6.4", + "sha2 0.10.9", +] + [[package]] name = "p521" version = "0.14.0" @@ -12089,6 +12207,16 @@ dependencies = [ "psl-types", ] +[[package]] +name = "pwd" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c71c0c79b9701efe4e1e4b563b2016dd4ee789eb99badcb09d61ac4b92e4a2" +dependencies = [ + "libc", + "thiserror 1.0.69", +] + [[package]] name = "qorb" version = "0.4.1" @@ -12393,7 +12521,7 @@ dependencies = [ "indoc 2.0.7", "instability", "itertools 0.13.0", - "lru", + "lru 0.12.5", "paste", "strum 0.26.3", "unicode-segmentation", @@ -12874,6 +13002,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rlimit" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" +dependencies = [ + "libc", +] + [[package]] name = "ron" version = "0.8.1" @@ -12978,6 +13115,31 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rumors" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rumors?rev=be1d76947f75f27e7f53be438a93d350019dd026#be1d76947f75f27e7f53be438a93d350019dd026" +dependencies = [ + "async-stream", + "before", + "blake3", + "bytes", + "ciborium", + "futures", + "futures-util", + "hex", + "itertools 0.14.0", + "rand 0.8.6", + "seq-macro", + "serde", + "smallvec 1.15.1", + "static_assertions", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tokio-stream", +] + [[package]] name = "russh" version = "0.62.4" @@ -13018,9 +13180,9 @@ dependencies = [ "ml-kem", "module-lattice", "num-bigint", - "p256", + "p256 0.14.0", "p384 0.14.0", - "p521", + "p521 0.14.0", "pageant", "pbkdf2", "pkcs1 0.8.0-rc.4", @@ -13040,8 +13202,8 @@ dependencies = [ "sha3 0.12.0", "signature 3.0.0", "spki 0.8.0", - "ssh-encoding", - "ssh-key", + "ssh-encoding 0.3.0", + "ssh-key 0.7.0-rc.11", "subtle", "thiserror 2.0.18", "tokio", @@ -13058,7 +13220,7 @@ checksum = "3aec6cb630dbe85d72ffd7bcd95f07e1bd69f9f270ee8adfa1afe443a6331438" dependencies = [ "log", "nix 0.31.2", - "ssh-encoding", + "ssh-encoding 0.3.0", "windows-sys 0.61.2", ] @@ -15021,7 +15183,7 @@ checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" [[package]] name = "sprockets-tls" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sprockets.git?rev=a233079e04d9688454486c452b694587a28c5257#a233079e04d9688454486c452b694587a28c5257" +source = "git+https://github.com/oxidecomputer/sprockets.git?rev=98615b86d51f74ce220198eb16c57932fd24e7cb#98615b86d51f74ce220198eb16c57932fd24e7cb" dependencies = [ "anyhow", "attest-data 0.5.0 (git+https://github.com/oxidecomputer/dice-util?rev=4a39ef08d81e5177edee0bddb1146032aa21074d)", @@ -15054,7 +15216,7 @@ dependencies = [ [[package]] name = "sprockets-tls-test-utils" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sprockets.git?rev=a233079e04d9688454486c452b694587a28c5257#a233079e04d9688454486c452b694587a28c5257" +source = "git+https://github.com/oxidecomputer/sprockets.git?rev=98615b86d51f74ce220198eb16c57932fd24e7cb#98615b86d51f74ce220198eb16c57932fd24e7cb" dependencies = [ "camino", "pki-playground 0.2.0 (git+https://github.com/oxidecomputer/pki-playground?rev=7600756029ce046a02c6234aa84ce230cc5eaa04)", @@ -15092,6 +15254,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ssh-cipher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caac132742f0d33c3af65bfcde7f6aa8f62f0e991d80db99149eb9d44708784f" +dependencies = [ + "cipher 0.4.4", + "ssh-encoding 0.2.0", +] + [[package]] name = "ssh-cipher" version = "0.3.0" @@ -15106,10 +15278,21 @@ dependencies = [ "ctutils", "des", "poly1305 0.9.0", - "ssh-encoding", + "ssh-encoding 0.3.0", "zeroize", ] +[[package]] +name = "ssh-encoding" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9242b9ef4108a78e8cd1a2c98e193ef372437f8c22be363075233321dd4a15" +dependencies = [ + "base64ct", + "pem-rfc7468 0.7.0", + "sha2 0.10.9", +] + [[package]] name = "ssh-encoding" version = "0.3.0" @@ -15125,6 +15308,28 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ssh-key" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b86f5297f0f04d08cabaa0f6bff7cb6aec4d9c3b49d87990d63da9d9156a8c3" +dependencies = [ + "ed25519-dalek 2.2.0", + "p256 0.13.2", + "p384 0.13.1", + "p521 0.13.3", + "rand_core 0.6.4", + "rsa 0.9.10", + "sec1 0.7.3", + "serde", + "sha2 0.10.9", + "signature 2.2.0", + "ssh-cipher 0.2.0", + "ssh-encoding 0.2.0", + "subtle", + "zeroize", +] + [[package]] name = "ssh-key" version = "0.7.0-rc.11" @@ -15137,17 +15342,17 @@ dependencies = [ "ed25519-dalek 3.0.0", "hex", "hmac 0.13.0", - "p256", + "p256 0.14.0", "p384 0.14.0", - "p521", + "p521 0.14.0", "rand_core 0.10.0", "rsa 0.10.0-rc.18", "sec1 0.8.1", "sha1 0.11.0", "sha2 0.11.0", "signature 3.0.0", - "ssh-cipher", - "ssh-encoding", + "ssh-cipher 0.3.0", + "ssh-encoding 0.3.0", "zeroize", ] @@ -15328,6 +15533,14 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "suanpan" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rumors?rev=be1d76947f75f27e7f53be438a93d350019dd026#be1d76947f75f27e7f53be438a93d350019dd026" +dependencies = [ + "dashu-int", +] + [[package]] name = "subprocess" version = "0.2.15" @@ -15447,6 +15660,110 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" +[[package]] +name = "sush-api" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +dependencies = [ + "chrono", + "dropshot", + "http-range-header", + "hyper", + "schemars 0.8.22", + "serde", + "sled-hardware-types", + "sush-common", + "thiserror 1.0.69", +] + +[[package]] +name = "sush-common" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +dependencies = [ + "bytes", + "bytesize", + "chrono", + "crypto-bigint 0.5.5", + "ed25519-dalek 2.2.0", + "hex", + "http", + "http-range-header", + "p256 0.13.2", + "pem-rfc7468 0.7.0", + "rand 0.8.6", + "rand_core 0.6.4", + "rlimit", + "rustix 1.1.4", + "schemars 0.8.22", + "serde", + "serde_json", + "sha2 0.10.9", + "sha3 0.10.8", + "signature 2.2.0", + "sled-hardware-types", + "slog", + "ssh-key 0.6.7", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite 0.28.0", + "x509-cert", +] + +[[package]] +name = "sush-server" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +dependencies = [ + "async-trait", + "atomicwrites", + "bytes", + "bytesize", + "camino", + "chrono", + "ciborium", + "clap", + "dropshot", + "ed25519-dalek 2.2.0", + "futures", + "http", + "http-body-util", + "http-range-header", + "hyper", + "hyper-util", + "libc", + "lru 0.18.3", + "memmap2", + "p256 0.13.2", + "percent-encoding", + "pwd", + "qorb", + "rand_core 0.6.4", + "rumors", + "rustix 1.1.4", + "rustls", + "schemars 0.8.22", + "serde", + "serde_json", + "sha2 0.10.9", + "sled-hardware-types", + "slog", + "slog-term", + "sprockets-tls", + "ssh-key 0.6.7", + "sush-api", + "sush-common", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-rustls", + "tokio-stream", + "tokio-tungstenite 0.28.0", + "tokio-util", + "x509-cert", +] + [[package]] name = "swrite" version = "0.1.0" @@ -15903,9 +16220,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -16110,6 +16427,18 @@ dependencies = [ "tungstenite 0.23.0", ] +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.28.0", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -16894,6 +17223,23 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.2", + "sha1 0.10.6", + "thiserror 2.0.18", + "utf-8", +] + [[package]] name = "twox-hash" version = "2.1.2" diff --git a/Cargo.toml b/Cargo.toml index 304268defd3..bdd036dd8e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -846,8 +846,8 @@ slog-term = "2.9.1" smf = "0.2" socket2 = { version = "0.5", features = ["all"] } sp-sim = { path = "sp-sim" } -sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "a233079e04d9688454486c452b694587a28c5257" } -sprockets-tls-test-utils = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "a233079e04d9688454486c452b694587a28c5257" } +sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb", default-features = false } +sprockets-tls-test-utils = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb" } sqlformat = "0.3.5" sqlparser = { version = "0.61.0", features = [ "visitor" ] } static_assertions = "1.1.0" @@ -861,6 +861,9 @@ subtle = "2.6.1" supports-color = "3.0.2" support-bundle-collection = { path = "support-bundle-collection" } support-bundle-viewer = "0.1.2" +sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6" } +sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6" } +sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6", features = ["embedded"] } swrite = "0.1.0" sync-ptr = "0.1.4" libsw = { version = "3.5.0", features = ["tokio"] } @@ -1160,20 +1163,25 @@ branch = "oxide/omicron" [patch.crates-io.omicron-workspace-hack] path = "workspace-hack" -# Several crates such as crucible and propolis have have a Git dependency on +# Several crates such as crucible and propolis have a Git dependency on # this repo. Omicron itself depends on these crates, which can lead to two -# copies of these crates in the dependency graph. (As a Git dependency, and as -# a path dependency.) The goal of omicron-uuid-kinds and +# copies of these crates in the dependency graph. (As a Git dependency, +# and as a path dependency.) The goal of omicron-uuid-kinds and # omicron-generation-kinds is to provide unified registries of UUID and # generation kinds. Two copies of the same kinds floating around is unnecessary # and painful. # -# These directives ensure that whenever we see omicron-uuid-kinds or -# omicron-generation-kinds as a Git dependency, we'll use the path dependency -# version of the crate instead. +# Similarly, sush has a Git dependency on this repo's sled-hardware-types, +# and the BaseboardId in its API must be the same type sled-agent derives +# from HardwareManager::baseboard(). +# +# These directives ensure that whenever we see omicron-uuid-kinds, +# omicron-generation-kinds, or sled-hardware-types as a Git dependency, +# we'll use the path dependency version of the crate instead. # # See also: uuid-kinds/README.adoc and generation-kinds/README.adoc. [patch."https://github.com/oxidecomputer/omicron"] omicron-common = { path = "common" } omicron-generation-kinds = { path = "generation-kinds" } omicron-uuid-kinds = { path = "uuid-kinds" } +sled-hardware-types = { path = "sled-hardware/types" } diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index c9450ecefaa..462ba47ebea 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -14,6 +14,8 @@ use either::Either; use omicron_common::address::BOOTSTRAP_PREFIX; use omicron_common::address::BOOTSTRAP_SLED_SUBNET_PREFIX_LENGTH; use omicron_common::address::DDMD_PORT; +use omicron_common::address::Ipv6Subnet; +use omicron_common::address::SLED_PREFIX_LENGTH; use oxnet::Ipv6Net; use sled_hardware_types::underlay::BootstrapInterface; use slog::Logger; @@ -103,6 +105,25 @@ impl Client { self.inner.enable_stats(request).await.map(|resp| resp.into_inner()) } + /// Returns the underlay subnets DDM advertises. Callers must probe + /// each subnet for what they expect to find, because sleds also + /// advertise internal DNS subnets, and RFD 63 reserves a services + /// prefix. + pub async fn derive_underlay_subnets_from_prefixes( + &self, + ) -> Result< + impl Iterator> + use<>, + DdmError, + > { + let prefixes = self.inner.get_prefixes().await?.into_inner(); + Ok(prefixes.into_values().flatten().filter_map(|prefix| { + let addr = prefix.destination.addr(); + (prefix.destination.width() == SLED_PREFIX_LENGTH + && addr.segments()[0] != BOOTSTRAP_PREFIX) + .then(|| Ipv6Subnet::new(addr)) + })) + } + /// Returns the addresses of connected sleds. /// /// Note: These sleds have not yet been verified. diff --git a/common/src/address.rs b/common/src/address.rs index 953ccd40df2..d8c36c0f196 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -232,6 +232,8 @@ pub const BOOTSTRAP_AGENT_RACK_INIT_PORT: u16 = 12346; pub const BOOTSTORE_PORT: u16 = 12347; pub const REPO_DEPOT_PORT: u16 = 12348; pub const TRUST_QUORUM_PORT: u16 = 12349; +pub const SUSH_API_PORT: u16 = 12350; +pub const SUSH_GOSSIP_PORT: u16 = 12351; pub const BOOTSTRAP_AGENT_LOCKSTEP_PORT: u16 = 8080; diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index 637c0a97484..60c7a46bd59 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -49,7 +49,7 @@ Management Gateway Service (client: gateway-client) consumed by: lldpd (lldp/lldpd) via 1 path consumed by: mgd (maghemite/mgd) via 1 path consumed by: omicron-nexus (omicron/nexus) via 6 paths - consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path + consumed by: omicron-sled-agent (omicron/sled-agent) via 2 paths consumed by: wicketd (omicron/wicketd) via 3 paths Wicketd Installinator (client: installinator-client) diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 47577484c32..3514f8dd2c9 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -35,6 +35,8 @@ dropshot.workspace = true flate2.workspace = true flume.workspace = true futures.workspace = true +gateway-client.workspace = true +gateway-types.workspace = true glob.workspace = true hex.workspace = true http.workspace = true @@ -106,8 +108,11 @@ slog-dtrace.workspace = true slog-error-chain.workspace = true slog-term.workspace = true smf.workspace = true -sprockets-tls.workspace = true +sprockets-tls = { workspace = true, features = ["ipcc"] } strum.workspace = true +sush-api.workspace = true +sush-common.workspace = true +sush-server.workspace = true tar.workspace = true thiserror.workspace = true tofino.workspace = true diff --git a/sled-agent/config-reconciler/src/handle.rs b/sled-agent/config-reconciler/src/handle.rs index d37ea02360f..947f30bfef6 100644 --- a/sled-agent/config-reconciler/src/handle.rs +++ b/sled-agent/config-reconciler/src/handle.rs @@ -645,6 +645,28 @@ impl AvailableDatasetsReceiver { } } + /// Wait until the set of available datasets may have changed. + /// + /// Returns immediately if the set has changed since this receiver last + /// observed it. The test variants' datasets never change, so for them this + /// never returns. + pub async fn changed(&mut self) { + match &mut self.inner { + AvailableDatasetsReceiverInner::Real(receiver) => { + // An error means the reconciler task is gone, in which case the + // set can never change again. + if receiver.changed().await.is_err() { + std::future::pending().await + } + } + #[cfg(feature = "testing")] + AvailableDatasetsReceiverInner::FakeTempDir { .. } + | AvailableDatasetsReceiverInner::FakeStatic(_) => { + std::future::pending().await + } + } + } + pub fn all_mounted_debug_datasets(&self) -> Vec { match &self.inner { AvailableDatasetsReceiverInner::Real(receiver) => { diff --git a/sled-agent/src/bootstrap/bootstore_setup.rs b/sled-agent/src/bootstrap/bootstore_setup.rs index 020f7506761..dd4508feaf9 100644 --- a/sled-agent/src/bootstrap/bootstore_setup.rs +++ b/sled-agent/src/bootstrap/bootstore_setup.rs @@ -11,6 +11,7 @@ use super::server::StartError; use bootstore::schemes::v0 as bootstore; use camino::Utf8PathBuf; use omicron_common::address::BOOTSTORE_PORT; +use omicron_common::address::SUSH_GOSSIP_PORT; use omicron_common::address::TRUST_QUORUM_PORT; use omicron_ddm_admin_client::Client as DdmAdminClient; use sled_hardware_types::Baseboard; @@ -22,6 +23,7 @@ use std::collections::BTreeSet; use std::net::Ipv6Addr; use std::net::SocketAddrV6; use std::time::Duration; +use tokio::sync::watch; const BOOTSTORE_FSM_STATE_FILE: &str = "bootstore-fsm-state.json"; const BOOTSTORE_NETWORK_CONFIG_FILE: &str = "bootstore-network-config.json"; @@ -75,10 +77,11 @@ fn bootstore_network_config_paths( Ok(paths) } -pub async fn poll_ddmd_for_bootstore_and_tq_peer_update( +pub async fn poll_ddmd_for_peer_updates( log: Logger, bootstore_node_handle: bootstore::NodeHandle, trust_quorum_handle: trust_quorum::NodeTaskHandle, + sush_gossip_tx: watch::Sender>, ) { let mut current_bootstore_peers: BTreeSet = BTreeSet::new(); let mut current_tq_peers: BTreeSet = BTreeSet::new(); @@ -137,6 +140,18 @@ pub async fn poll_ddmd_for_bootstore_and_tq_peer_update( return; } } + // Inform sush gossip of all known peer addresses. + let sush_peers: BTreeSet<_> = addrs + .iter() + .map(|ip| SocketAddrV6::new(*ip, SUSH_GOSSIP_PORT, 0, 0)) + .collect(); + sush_gossip_tx.send_if_modified(|current| { + let changed = *current != sush_peers; + if changed { + *current = sush_peers; + } + changed + }); } Err(err) => { warn!( diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 5f4d2d83be3..af4683292bb 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -120,6 +120,56 @@ pub struct Config { /// root certificates and whether to use local certificate chain or /// one over IPCC pub sprockets: SprocketsConfig, + + /// Settings for the Support Shell server (RFD 620). If this is absent, no + /// Support Shell server runs on this sled. + #[serde(default)] + pub sush: Option, +} + +/// Configuration for the Support Shell (`sush`) server that runs in the global +/// zone. See RFD 620. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SushConfig { + /// PEM files holding the trusted root certificates for job requests, + /// one certificate per file. A job whose signature does not chain to + /// one of these is refused, so an empty list means no job can ever run. + pub roots: Vec, + + /// Where to record job output before an encrypted dataset is available. + /// This is on the ramdisk, so it costs global zone memory and does not + /// survive a reboot. + #[serde(default = "SushConfig::default_ramdisk_dir")] + pub ramdisk_dir: Utf8PathBuf, + + /// Maximum output a single job may record before an encrypted dataset is + /// available. The default is small because the ramdisk is global zone + /// memory. + #[serde(default = "SushConfig::default_ramdisk_max_output_mb")] + pub ramdisk_max_output_mb: u32, + + /// Maximum output a single job may record once an encrypted dataset is + /// available. + #[serde(default = "SushConfig::default_max_output_mb")] + pub max_output_mb: u32, +} + +impl SushConfig { + // TODO: check that this is the right default. `/var/run/oxide` is tmpfs + // and is created during bootstrap, but it exists to hold ZFS key files + // (see `illumos_utils::zfs::KEYPATH_ROOT`), not bulk job output. + fn default_ramdisk_dir() -> Utf8PathBuf { + "/var/run/oxide/sush".into() + } + + fn default_ramdisk_max_output_mb() -> u32 { + 64 + } + + fn default_max_output_mb() -> u32 { + 10 * 1024 + } } #[derive(Debug, thiserror::Error)] @@ -210,4 +260,32 @@ mod test { } assert!(configs_seen > 0, "No sled-agent configs found"); } + + #[test] + fn test_sush_config() { + // Only `roots` is required. The rest have defaults. + let sush: SushConfig = + toml::from_str(r#"roots = ["/pkg/sush-root.pem"]"#).unwrap(); + assert_eq!(sush.roots, vec![Utf8PathBuf::from("/pkg/sush-root.pem")]); + assert_eq!(sush.ramdisk_dir, SushConfig::default_ramdisk_dir()); + assert_eq!( + sush.ramdisk_max_output_mb, + SushConfig::default_ramdisk_max_output_mb() + ); + assert_eq!(sush.max_output_mb, SushConfig::default_max_output_mb()); + + let sush: SushConfig = toml::from_str( + r#" + roots = [] + ramdisk_dir = "/var/run/sush" + ramdisk_max_output_mb = 8 + max_output_mb = 128 + "#, + ) + .unwrap(); + assert!(sush.roots.is_empty()); + assert_eq!(sush.ramdisk_dir, "/var/run/sush"); + assert_eq!(sush.ramdisk_max_output_mb, 8); + assert_eq!(sush.max_output_mb, 128); + } } diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index a30026d0134..f41bc1d4949 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -35,6 +35,7 @@ pub mod server; pub mod services; mod sled_agent; mod support_bundle; +pub mod sush; mod swap_device; mod vmm_reservoir; mod zone_bundle; diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs index bb34420cdb2..c0b74859bb9 100644 --- a/sled-agent/src/long_running_tasks.rs +++ b/sled-agent/src/long_running_tasks.rs @@ -14,7 +14,7 @@ use crate::artifact_store::{ArtifactStore, SledAgentArtifactStoreWrapper}; use crate::bootstrap::bootstore_setup::{ - new_bootstore_config, poll_ddmd_for_bootstore_and_tq_peer_update, + new_bootstore_config, poll_ddmd_for_peer_updates, }; use crate::bootstrap::secret_retriever::{ ConfigurableSecretRetriever, ConfigurableSecretRetrieverHandle, @@ -24,6 +24,7 @@ use crate::config::Config; use crate::hardware_monitor::{HardwareMonitor, HardwareMonitorHandle}; use crate::services::ServiceManager; use crate::sled_agent::SledAgent; +use crate::sush::{GossipInputs, SushHandles, spawn_sush_tasks}; use crate::zone_bundle::ZoneBundler; use bootstore::schemes::v0 as bootstore; use key_manager::{KeyManager, StorageKeyRequester}; @@ -41,9 +42,10 @@ use sled_storage::config::MountConfig; use sled_storage::disk::RawSyntheticDisk; use slog::{Logger, info}; use sprockets_tls::keys::SprocketsConfig; -use std::net::Ipv6Addr; +use std::collections::BTreeSet; +use std::net::{Ipv6Addr, SocketAddrV6}; use std::sync::Arc; -use tokio::sync::oneshot; +use tokio::sync::{oneshot, watch}; use trust_quorum; /// A mechanism for interacting with all long running tasks that can be shared @@ -91,6 +93,9 @@ pub struct LongRunningTaskHandles { /// Handle to access the set of reference measurements pub measurements: Arc, + + /// Handles to the Support Shell server, if it is running + pub sush: Option, } pub struct LongRunningTaskResult { @@ -196,17 +201,51 @@ pub async fn spawn_all_longrunning_tasks( ) .await; + let (sush_gossip_peers_tx, sush_gossip_peers_rx) = + watch::channel(BTreeSet::new()); let bootstore = spawn_bootstore_tasks( log, &config_reconciler, &hardware_manager, global_zone_bootstrap_ip, trust_quorum.clone(), + sush_gossip_peers_tx, ) .await; let health_monitor = spawn_health_monitor_tasks(log).await; + // sush must work when the control plane doesn't, so it starts here rather + // than waiting for rack membership, and serves its API on the bootstrap + // network. Once this sled is told its underlay address, + // `crate::server::Server::start` serves the same API there too. + let sush = match &config.sush { + Some(sush_config) => { + spawn_sush_tasks( + log, + sush_config, + hardware_manager.baseboard().into(), + config_reconciler + .internal_disks_rx() + .current() + .all_cluster_datasets() + .collect(), + GossipInputs { + sprockets: config.sprockets.clone(), + measurements: measurements.clone(), + bootstrap_ip: global_zone_bootstrap_ip, + peers: sush_gossip_peers_rx, + }, + config_reconciler.available_datasets_rx(), + ) + .await + } + None => { + info!(log, "sush not configured"); + None + } + }; + LongRunningTaskResult { long_running_task_handles: LongRunningTaskHandles { config_reconciler, @@ -221,6 +260,7 @@ pub async fn spawn_all_longrunning_tasks( secret_retriever: secret_retriever_config, artifact_store, measurements, + sush, }, config_reconciler_spawn_token, sled_agent_started_tx, @@ -324,6 +364,7 @@ async fn spawn_bootstore_tasks( hardware_manager: &HardwareManager, global_zone_bootstrap_ip: Ipv6Addr, tq_handle: trust_quorum::NodeTaskHandle, + sush_gossip_tx: watch::Sender>, ) -> bootstore::NodeHandle { let config = new_bootstore_config( &config_reconciler @@ -346,7 +387,7 @@ async fn spawn_bootstore_tasks( let log = log.new(o!("component" => "bootstore_ddmd_poller")); let node_handle2 = node_handle.clone(); tokio::spawn(async move { - poll_ddmd_for_bootstore_and_tq_peer_update(log, node_handle2, tq_handle) + poll_ddmd_for_peer_updates(log, node_handle2, tq_handle, sush_gossip_tx) .await }); diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index c23e7033150..c4538d47c45 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -10,6 +10,7 @@ use super::sled_agent::SledAgent; use crate::long_running_tasks::LongRunningTaskHandles; use crate::nexus::make_nexus_client; use crate::services::ServiceManager; +use crate::sush::SushHandles; use internal_dns_resolver::Resolver; use omicron_uuid_kinds::SledUuid; use sled_agent_config_reconciler::ConfigReconcilerSpawnToken; @@ -18,12 +19,16 @@ use slog::Logger; use slog_error_chain::InlineErrorChain; use std::net::SocketAddr; use std::sync::Arc; +use sush_server::JobManager; /// Packages up a [`SledAgent`], running the sled agent API under a Dropshot /// server wired up to the sled agent pub struct Server { /// Dropshot server for the API. http_server: dropshot::HttpServer, + + /// The Support Shell's task handles and API server, if it is running. + sush: Option<(SushHandles, dropshot::HttpServer>)>, } impl Server { @@ -57,6 +62,7 @@ impl Server { let nexus_client = make_nexus_client(&log, resolver); + let sush_handles = long_running_tasks_handles.sush.clone(); let sled_agent = SledAgent::new( &config, log.clone(), @@ -86,7 +92,20 @@ impl Server { ))) .start() .map_err(|error| format!("initializing server: {}", error))?; - Ok(Server { http_server }) + + // Now that this sled knows its underlay address, sush can serve + // its API there, too. + let sush = sush_handles.and_then(|handles| { + match handles.start_api(*sled_address.ip()) { + Ok(server) => Some((handles, server)), + Err(error) => { + warn!(log, "failed to start sush server"; "error" => error); + None + } + } + }); + + Ok(Server { http_server, sush }) } pub(crate) fn sled_agent(&self) -> &SledAgent { @@ -103,6 +122,13 @@ impl Server { } pub async fn close(self) -> Result<(), String> { - self.http_server.close().await + let sush = match self.sush { + Some((handles, server)) => { + handles.shutdown(); + server.close().await + } + None => Ok(()), + }; + self.http_server.close().await.and(sush) } } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs new file mode 100644 index 00000000000..21fc5c440fb --- /dev/null +++ b/sled-agent/src/sush.rs @@ -0,0 +1,427 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The Support Shell (`sush`) server that runs in the global zone. +//! +//! See RFD 620. This server runs jobs targeted at this sled on behalf of +//! Oxide support. A job arrives signed, and this sled runs it only when +//! the signature chains to one of the root certificates in its config. +//! The server is deliberately independent of the rest of the control plane, +//! because it is one of the tools you reach for when the control plane is +//! the thing that is broken. It therefore starts during bootstrap, before +//! this sled knows whether it is part of a rack at all. +//! +//! Startup is in two parts, because its inputs become available at different +//! times: +//! +//! * [`spawn_sush_tasks`] builds the job manager as soon as we know our +//! baseboard, and is called from `long_running_tasks`. It serves the +//! API on the bootstrap network, so the switch zone proxy can reach +//! this sled before RSS runs. +//! * [`SushHandles::start_api`] serves the API again on the underlay, +//! and is called from `server` once this sled has been told its +//! underlay address. +//! +//! Gossip runs over sprockets on the bootstrap network, so jobs and +//! sessions are shared across sleds. A universe is a shared gossip +//! history; peers that meet merge into one by a dominance rule. +//! A restarted sled re-seeds its universe, rejoins the rack's, then +//! replays its history without re-executing it. Its gossip identity +//! is stored in a _bookmark_. +//! +//! The bookmark, like every record sush must trust across reboots, +//! lives in the [sush locker](https://github.com/oxidecomputer/sush/blob/main/server/src/locker.rs). +//! Each locker record is one file on each M.2. Stores write every +//! copy, and loads adopt a record only when the copies show it cannot +//! be stale. The locker is not a small bootstore: the bootstore holds +//! rack-wide facts that sleds may recover from their peers, but a locker +//! record says what this sled itself did or committed to do. These +//! guarantees are critical to the correctness of the +//! [rumors](https://github.com/oxidecomputer/rumors) gossip algorithm, +//! which sush uses to synchronize job and event sets. + +use crate::config::SushConfig; +use camino::Utf8PathBuf; +use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer, ServerBuilder}; +use gateway_client::Client as MgsClient; +use gateway_types::component::SpType; +use omicron_common::address::{ + MGS_PORT, SUSH_API_PORT, SUSH_GOSSIP_PORT, get_switch_zone_address, +}; +use omicron_ddm_admin_client::Client as DdmClient; +use sled_agent_config_reconciler::AvailableDatasetsReceiver; +use sled_agent_measurements::MeasurementsHandle; +use sled_hardware_types::BaseboardId; +use slog::{Logger, debug, error, info, o, warn}; +use slog_error_chain::InlineErrorChain; +use sprockets_tls::keys::SprocketsConfig; +use std::collections::BTreeSet; +use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; +use std::sync::Arc; +use std::time::Duration; +use tokio::fs::create_dir_all; +use tokio::spawn; +use tokio::sync::watch; +use tokio::time::sleep; +use tokio_util::sync::CancellationToken; + +use sush_common::targets::{Cubbies, MAX_CUBBY}; +use sush_server::executor::PathIsolation; +use sush_server::gossip::{GossipConfig, isolated, lonely, spawn_gossip}; +use sush_server::link::CorpusSource; +use sush_server::locker::Locker; +use sush_server::output::{JobOutputDir, OutputDirs}; +use sush_server::server::ApiServer; +use sush_server::{JobManager, seed_gossip}; + +/// Subdirectory of an encrypted dataset that job output is recorded in. +const SUSH_OUTPUT_SUBDIR: &str = "sush"; + +/// How often to refresh the cubby map from MGS. +const MGS_POLL_INTERVAL: Duration = Duration::from_secs(30); + +/// How long to wait for an MGS candidate to answer. +const MGS_PROBE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Maximum size of a request body the API will accept. The largest thing a +/// client sends is a signed job request or a certificate, both small. +const REQUEST_MAX_BODY_BYTES: usize = 0xFFFF; + +/// Handles to the Support Shell server's tasks. +#[derive(Clone)] +pub struct SushHandles { + log: Logger, + manager: Arc, + shutdown: CancellationToken, +} + +impl SushHandles { + /// Start serving the Support Shell API at `ip`. + pub fn start_api( + &self, + ip: Ipv6Addr, + ) -> Result>, String> { + let bind_address = + SocketAddr::V6(SocketAddrV6::new(ip, SUSH_API_PORT, 0, 0)); + let api = sush_api::sush_api_mod::api_description::() + .map_err(|err| format!("failed to describe sush API: {err}"))?; + let server = ServerBuilder::new( + api, + Arc::clone(&self.manager), + self.log.new(o!("component" => "dropshot (sush)")), + ) + .config(ConfigDropshot { + bind_address, + default_request_body_max_bytes: REQUEST_MAX_BODY_BYTES, + // An interactive job holds a websocket open for as long as it runs, + // so a handler must outlive the request that created it. + default_handler_task_mode: HandlerTaskMode::Detached, + log_headers: vec![], + compression: Default::default(), + }) + .start() + .map_err(|err| err.to_string())?; + info!( + self.log, "started sush server"; + "address" => %bind_address, + ); + Ok(server) + } + + /// Stop the Support Shell server's tasks. + pub fn shutdown(&self) { + self.shutdown.cancel(); + } +} + +/// What gossip needs from the sled: its sprockets identity, its reference +/// measurements, the bootstrap address to listen on, and where to find its +/// peers. +pub struct GossipInputs { + pub sprockets: SprocketsConfig, + pub measurements: Arc, + pub bootstrap_ip: Ipv6Addr, + pub peers: watch::Receiver>, +} + +/// Start the Support Shell server's tasks, or log the reason and return +/// `None` when they cannot start. The locker gets one slot on each of +/// `cluster_datasets`. +pub async fn spawn_sush_tasks( + log: &Logger, + config: &SushConfig, + own_baseboard: BaseboardId, + cluster_datasets: Vec, + gossip: GossipInputs, + available_datasets_rx: AvailableDatasetsReceiver, +) -> Option { + let log = log.new(o!("component" => "sush")); + + // Job output starts on the ramdisk, because an encrypted dataset cannot be + // mounted until trust quorum is established, and sush must be useful + // before then. + if let Err(err) = create_dir_all(&config.ramdisk_dir).await { + error!( + log, + "not starting sush server: could not create job output directory"; + "directory" => %config.ramdisk_dir, + "error" => InlineErrorChain::new(&err), + ); + return None; + } + let (output_dirs_tx, output_dirs_rx) = watch::channel(OutputDirs::new( + config.ramdisk_dir.as_std_path(), + mb_to_bytes(config.ramdisk_max_output_mb), + )); + + if config.roots.is_empty() { + warn!(log, "sush has no root certificates, so no job will ever run"); + } + let shutdown = CancellationToken::new(); + + // A slot that cannot be created is still handed to the locker, which + // decides what a failed slot means for each load and store. + let slots: Vec = + cluster_datasets.iter().map(|dataset| dataset.join("sush")).collect(); + for slot in &slots { + if let Err(err) = create_dir_all(slot).await { + warn!( + log, "could not create a locker slot"; + "slot" => %slot, + "error" => InlineErrorChain::new(&err), + ); + } + } + let locker = Locker::new(&log, slots); + + // Gossip re-reads the attestation corpus on every handshake, because a + // software update changes it. A sled that cannot gossip still serves local + // jobs. + let GossipInputs { sprockets, measurements, bootstrap_ip, peers } = gossip; + let corpus: CorpusSource = Arc::new({ + let log = log.clone(); + move || match measurements.current_measurements() { + Ok(corpus) => corpus, + Err(e) => { + error!(log, "measurement error"; e); + vec![] + } + } + }); + let listen_addr = SocketAddrV6::new(bootstrap_ip, SUSH_GOSSIP_PORT, 0, 0); + let (universe, linked) = match spawn_gossip( + &log, + GossipConfig::default(), + sprockets, + corpus, + listen_addr, + peers, + seed_gossip(&log, &locker).await, + shutdown.clone(), + ) + .await + { + Ok((_, universe, linked)) => (universe, linked), + Err(err) => { + warn!( + log, + "gossip disabled, this sled serves local jobs only"; + "error" => InlineErrorChain::new(&err), + ); + // This seed never reaches a peer, so it is not worth + // storing. The null locker stores nothing, leaving the + // bookmark on the M.2s untouched for the next boot. + let seed = seed_gossip(&log, &Locker::null()).await; + (isolated(seed.into_rumors()), lonely()) + } + }; + + let (tx_cubbies, rx_cubbies) = watch::channel(Cubbies::new()); + let mut manager = match JobManager::new( + log.clone(), + PathIsolation::Enable, + JobOutputDir::new(output_dirs_rx), + own_baseboard, + rx_cubbies, + universe, + linked, + &locker, + &config.roots, + shutdown.clone(), + ) + .await + { + Ok(manager) => manager, + Err(err) => { + error!(log, "not starting sush server"; "error" => InlineErrorChain::new(&err)); + return None; + } + }; + spawn(poll_mgs_for_cubbies( + log.new(o!("component" => "cubby map")), + tx_cubbies, + )); + + // The state manager runs until shutdown and nothing waits on it, + // so all we can usefully do with its handle is notice when it stops. + if let Some(join) = manager.take_join_handle() { + let log = log.clone(); + spawn(async move { + match join.await { + Ok(()) => info!(log, "sush state manager stopped"), + Err(err) => error!( + log, + "sush state manager failed"; + "error" => InlineErrorChain::new(&err), + ), + } + }); + } + + spawn(promote_output_dir( + log.clone(), + available_datasets_rx, + output_dirs_tx, + mb_to_bytes(config.max_output_mb), + )); + + info!(log, "started sush job manager"); + let handles = SushHandles { log, manager: Arc::new(manager), shutdown }; + + match handles.start_api(bootstrap_ip) { + Ok(server) => { + let shutdown = handles.shutdown.clone(); + spawn(async move { + shutdown.cancelled().await; + server.close().await.ok(); + }); + } + Err(err) => warn!( + handles.log, + "sush is not serving on the bootstrap network"; + "error" => err, + ), + } + + Some(handles) +} + +/// Start recording new job output on an encrypted debug dataset as soon +/// as one is mounted, with a raised size limit. Output already recorded +/// on the ramdisk stays there, readable until reboot. +async fn promote_output_dir( + log: Logger, + mut available_datasets_rx: AvailableDatasetsReceiver, + output_dirs_tx: watch::Sender, + max_output_bytes: u64, +) { + loop { + if let Some(dataset) = + available_datasets_rx.all_mounted_debug_datasets().first() + { + let dir = dataset.path.join(SUSH_OUTPUT_SUBDIR); + match create_dir_all(&dir).await { + Ok(()) => { + output_dirs_tx.send_modify(|dirs| { + *dirs = + dirs.moved_to(dir.as_std_path(), max_output_bytes) + }); + info!( + log, "recording job output on encrypted dataset"; + "directory" => %dir, + ); + return; + } + Err(err) => error!( + log, + "could not create job output directory on encrypted dataset, leaving it on the ramdisk"; + "directory" => %dir, + "error" => InlineErrorChain::new(&err), + ), + } + } + available_datasets_rx.changed().await; + } +} + +fn mb_to_bytes(mb: u32) -> u64 { + u64::from(mb) * 1024 * 1024 +} + +/// Keep the cubby map current from MGS's view of the SPs. A job may +/// name its target sled by cubby, and the map says which baseboard +/// is in each cubby. +/// +/// DDM's advertised subnets are only candidates (see +/// [`DdmClient::derive_underlay_subnets_from_prefixes`]), so we must +/// probe for MGS. Each round's answers merge into the existing map, +/// so a probe outage never erases it. +async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { + let ddm = match DdmClient::localhost(&log) { + Ok(ddm) => ddm, + Err(err) => { + error!( + log, "not polling MGS, cubby-targeted jobs will not run here"; + "error" => InlineErrorChain::new(&err), + ); + return; + } + }; + let client = reqwest::ClientBuilder::new() + .connect_timeout(MGS_PROBE_TIMEOUT) + .timeout(MGS_PROBE_TIMEOUT) + .build() + .expect("failed to build an HTTP client"); + loop { + match ddm.derive_underlay_subnets_from_prefixes().await { + Ok(subnets) => { + let mut map = Cubbies::new(); + for subnet in subnets { + let addr = SocketAddrV6::new( + get_switch_zone_address(subnet), + MGS_PORT, + 0, + 0, + ); + let mgs = MgsClient::new_with_client( + &format!("http://{addr}"), + client.clone(), + log.clone(), + ); + if mgs.sp_local_switch_id().await.is_err() { + continue; + } + for cubby in 0..=MAX_CUBBY { + match mgs.sp_get(&SpType::Sled, cubby.into()).await { + Ok(state) => { + let state = state.into_inner(); + map.insert( + cubby, + BaseboardId { + part_number: state.model, + serial_number: state.serial_number, + }, + ); + } + Err(err) => { + debug!( + log, "no SP state for cubby"; + "cubby" => cubby, "error" => %err, + ); + } + } + } + } + cubbies.send_modify(|current| current.extend(map)); + } + Err(err) => { + warn!( + log, "unable to fetch prefixes"; + "error" => InlineErrorChain::new(&err), + ); + } + } + sleep(MGS_POLL_INTERVAL).await; + } +} diff --git a/smf/sled-agent/gimlet-standalone/config.toml b/smf/sled-agent/gimlet-standalone/config.toml index 547ea950c11..34b46bbd7e4 100644 --- a/smf/sled-agent/gimlet-standalone/config.toml +++ b/smf/sled-agent/gimlet-standalone/config.toml @@ -84,3 +84,8 @@ resolve = { which = "ipcc" } attest = { which = "ipcc" } roots = ["/usr/share/oxide/idcerts/staging.pem", "/usr/share/oxide/idcerts/production.pem"] enforce = "Enforced" + +# Support Shell (RFD 620). TODO: replace the sandbox root with the +# production certificate. +[sush] +roots = ["/opt/oxide/sled-agent/pkg/sush-sandbox.pem"] diff --git a/smf/sled-agent/gimlet-standalone/sush-sandbox.pem b/smf/sled-agent/gimlet-standalone/sush-sandbox.pem new file mode 100644 index 00000000000..17b6a761ef2 --- /dev/null +++ b/smf/sled-agent/gimlet-standalone/sush-sandbox.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBpTCCAUqgAwIBAgIUcaflT3yMc6uPJUFaRV5NZtUdXi0wCgYIKoZIzj0EAwIw +UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwMTI3MTk0 +NzU2WhgPOTk5OTEyMzEyMzU5NTlaMFAxCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZP +eGlkZSBDb21wdXRlciBDb21wYW55MSAwHgYDVQQDDBdTdXBwb3J0IFNoZWxsIFBy +b3RvdHlwZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP5+eaTZnpF4tghwJolg +m8te/yAfAnkgzOV1FJiH0nauoM8gsyoiVAu9CwlHNnivju6TslFFyxBlW5h3eMnJ +ULwwCgYIKoZIzj0EAwIDSQAwRgIhAIqvomz/EXcZGCti2QwHkdPOMfidDw02/Rwk +t3x+5DsfAiEAlEeWgVPiLcseslP86MJpLvVPFFO3S08IODCNgOUh7tM= +-----END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet/config.toml b/smf/sled-agent/gimlet/config.toml index 92a9cda8392..eca0e7ef7a9 100644 --- a/smf/sled-agent/gimlet/config.toml +++ b/smf/sled-agent/gimlet/config.toml @@ -79,3 +79,8 @@ resolve = { which = "ipcc" } attest = { which = "ipcc" } roots = ["/usr/share/oxide/idcerts/staging.pem", "/usr/share/oxide/idcerts/production.pem"] enforce = "Enforced" + +# Support Shell (RFD 620). TODO: replace the sandbox root with the +# production certificate. +[sush] +roots = ["/opt/oxide/sled-agent/pkg/sush-sandbox.pem"] diff --git a/smf/sled-agent/gimlet/sush-sandbox.pem b/smf/sled-agent/gimlet/sush-sandbox.pem new file mode 100644 index 00000000000..17b6a761ef2 --- /dev/null +++ b/smf/sled-agent/gimlet/sush-sandbox.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBpTCCAUqgAwIBAgIUcaflT3yMc6uPJUFaRV5NZtUdXi0wCgYIKoZIzj0EAwIw +UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwMTI3MTk0 +NzU2WhgPOTk5OTEyMzEyMzU5NTlaMFAxCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZP +eGlkZSBDb21wdXRlciBDb21wYW55MSAwHgYDVQQDDBdTdXBwb3J0IFNoZWxsIFBy +b3RvdHlwZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP5+eaTZnpF4tghwJolg +m8te/yAfAnkgzOV1FJiH0nauoM8gsyoiVAu9CwlHNnivju6TslFFyxBlW5h3eMnJ +ULwwCgYIKoZIzj0EAwIDSQAwRgIhAIqvomz/EXcZGCti2QwHkdPOMfidDw02/Rwk +t3x+5DsfAiEAlEeWgVPiLcseslP86MJpLvVPFFO3S08IODCNgOUh7tM= +-----END CERTIFICATE----- diff --git a/smf/sled-agent/non-gimlet/config.toml b/smf/sled-agent/non-gimlet/config.toml index 4f46cd56cb7..3342db3e333 100644 --- a/smf/sled-agent/non-gimlet/config.toml +++ b/smf/sled-agent/non-gimlet/config.toml @@ -129,3 +129,8 @@ resolve = { which = "local", priv_key = "/opt/oxide/sled-agent/pkg/test-sprocket attest = { which = "local", priv_key = "/opt/oxide/sled-agent/pkg/test-alias-1.key.pem", cert_chain = "/opt/oxide/sled-agent/pkg/test-alias-1.certlist.pem", log = "/opt/oxide/sled-agent/pkg/sprockets-log.bin", test_corpus = [ "/opt/oxide/sled-agent/pkg/testing-measurements/corim-rot.cbor", "/opt/oxide/sled-agent/pkg/testing-measurements/corim-sp.cbor"] } roots = ["/opt/oxide/sled-agent/pkg/test-root-a.cert.pem"] enforce = "Enforced" + +# Support Shell (RFD 620). TODO: replace the sandbox root with the +# production certificate. +[sush] +roots = ["/opt/oxide/sled-agent/pkg/sush-sandbox.pem"] diff --git a/smf/sled-agent/non-gimlet/sush-sandbox.pem b/smf/sled-agent/non-gimlet/sush-sandbox.pem new file mode 100644 index 00000000000..17b6a761ef2 --- /dev/null +++ b/smf/sled-agent/non-gimlet/sush-sandbox.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBpTCCAUqgAwIBAgIUcaflT3yMc6uPJUFaRV5NZtUdXi0wCgYIKoZIzj0EAwIw +UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwMTI3MTk0 +NzU2WhgPOTk5OTEyMzEyMzU5NTlaMFAxCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZP +eGlkZSBDb21wdXRlciBDb21wYW55MSAwHgYDVQQDDBdTdXBwb3J0IFNoZWxsIFBy +b3RvdHlwZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP5+eaTZnpF4tghwJolg +m8te/yAfAnkgzOV1FJiH0nauoM8gsyoiVAu9CwlHNnivju6TslFFyxBlW5h3eMnJ +ULwwCgYIKoZIzj0EAwIDSQAwRgIhAIqvomz/EXcZGCti2QwHkdPOMfidDw02/Rwk +t3x+5DsfAiEAlEeWgVPiLcseslP86MJpLvVPFFO3S08IODCNgOUh7tM= +-----END CERTIFICATE----- diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index d9d151cc286..ddcf58c17f6 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -28,6 +28,7 @@ base64 = { version = "0.22.1" } base64ct = { version = "1.8.3", default-features = false, features = ["std"] } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.11.0", default-features = false, features = ["serde", "std"] } +blake3 = { version = "1.8.5", features = ["mmap", "rayon"] } bstr = { version = "1.12.1" } buf-list = { version = "1.1.2", default-features = false, features = ["tokio1"] } byte-wrapper = { version = "0.1.0", features = ["schemars08", "serde"] } @@ -40,6 +41,7 @@ const-oid = { version = "0.9.6", default-features = false, features = ["db", "st crossbeam-epoch = { version = "0.9.18" } crossbeam-utils = { version = "0.8.21" } crossterm = { version = "0.28.1", features = ["event-stream", "serde"] } +crypto-bigint = { version = "0.5.5", features = ["generic-array", "zeroize"] } crypto-common-6f8ce4dd05d13bba = { package = "crypto-common", version = "0.2.2", default-features = false, features = ["getrandom"] } crypto-common-c65f7effa3be6d31 = { package = "crypto-common", version = "0.1.7", default-features = false, features = ["getrandom", "std"] } ctutils = { version = "0.4.2", default-features = false, features = ["subtle"] } @@ -48,6 +50,7 @@ daft = { version = "0.1.8", features = ["derive", "newtype-uuid1", "oxnet01", "u data-encoding = { version = "2.10.0" } der = { version = "0.7.10", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] } derive_more = { version = "2.1.1", features = ["as_ref", "is_variant"] } +dice-verifier = { git = "https://github.com/oxidecomputer/dice-util", rev = "4a39ef08d81e5177edee0bddb1146032aa21074d", default-features = false, features = ["ipcc", "mock"] } digest-93f6ce9d446188ac = { package = "digest", version = "0.10.7", features = ["mac", "oid", "std"] } digest-a6292c17cd707f01 = { package = "digest", version = "0.11.3", features = ["alloc", "mac", "oid", "rand_core"] } ed25519-dalek = { version = "2.2.0", features = ["digest", "pem", "rand_core"] } @@ -71,6 +74,7 @@ getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.17", defaul getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4.3", default-features = false, features = ["std", "sys_rng", "wasm_js"] } group = { version = "0.13.0", default-features = false, features = ["alloc"] } hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15.5" } +hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17.0" } hashbrown-986da7b5efc2b80e = { package = "hashbrown", version = "0.16.1" } hex = { version = "0.4.3", features = ["serde"] } hickory-proto = { version = "0.25.2", features = ["serde", "text-parsing"] } @@ -120,7 +124,7 @@ regex-syntax = { version = "0.8.10" } reqwest-594e8ee84c453af0 = { package = "reqwest", version = "0.13.2", features = ["blocking", "cookies", "json", "query", "stream"] } reqwest-5ef9efb8ec2df382 = { package = "reqwest", version = "0.12.28", features = ["blocking", "json", "stream"] } rsa = { version = "0.9.10", features = ["serde", "sha2"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["fs"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["process", "pty", "termios"] } rustls = { version = "0.23.41" } schemars = { version = "0.8.22", features = ["bytes", "chrono", "semver", "url", "uuid1"] } scopeguard = { version = "1.2.0" } @@ -135,9 +139,10 @@ simd-adler32 = { version = "0.3.8", default-features = false, features = ["std"] similar = { version = "2.7.0", features = ["bytes", "inline", "unicode"] } slab = { version = "0.4.12" } slog = { version = "2.8.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } -smallvec = { version = "1.15.1", default-features = false, features = ["const_new"] } +smallvec = { version = "1.15.1", default-features = false, features = ["const_new", "union"] } spin = { version = "0.9.8" } spki = { version = "0.7.3", default-features = false, features = ["pem", "std"] } +sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb", default-features = false, features = ["ipcc"] } string_cache = { version = "0.8.9" } strum-2f80eeee3b1b6c7e = { package = "strum", version = "0.26.3", features = ["derive"] } strum-754bda37e0fb3874 = { package = "strum", version = "0.27.2", features = ["derive"] } @@ -180,6 +185,7 @@ base64 = { version = "0.22.1" } base64ct = { version = "1.8.3", default-features = false, features = ["std"] } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.11.0", default-features = false, features = ["serde", "std"] } +blake3 = { version = "1.8.5", features = ["mmap", "rayon"] } bstr = { version = "1.12.1" } buf-list = { version = "1.1.2", default-features = false, features = ["tokio1"] } byte-wrapper = { version = "0.1.0", features = ["schemars08", "serde"] } @@ -193,6 +199,7 @@ const-oid = { version = "0.9.6", default-features = false, features = ["db", "st crossbeam-epoch = { version = "0.9.18" } crossbeam-utils = { version = "0.8.21" } crossterm = { version = "0.28.1", features = ["event-stream", "serde"] } +crypto-bigint = { version = "0.5.5", features = ["generic-array", "zeroize"] } crypto-common-6f8ce4dd05d13bba = { package = "crypto-common", version = "0.2.2", default-features = false, features = ["getrandom"] } crypto-common-c65f7effa3be6d31 = { package = "crypto-common", version = "0.1.7", default-features = false, features = ["getrandom", "std"] } ctutils = { version = "0.4.2", default-features = false, features = ["subtle"] } @@ -202,6 +209,7 @@ data-encoding = { version = "2.10.0" } der = { version = "0.7.10", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] } derive_more = { version = "2.1.1", features = ["as_ref", "is_variant"] } derive_more-impl = { version = "2.1.1", features = ["as_ref", "is_variant"] } +dice-verifier = { git = "https://github.com/oxidecomputer/dice-util", rev = "4a39ef08d81e5177edee0bddb1146032aa21074d", default-features = false, features = ["ipcc", "mock"] } digest-93f6ce9d446188ac = { package = "digest", version = "0.10.7", features = ["mac", "oid", "std"] } digest-a6292c17cd707f01 = { package = "digest", version = "0.11.3", features = ["alloc", "mac", "oid", "rand_core"] } ed25519-dalek = { version = "2.2.0", features = ["digest", "pem", "rand_core"] } @@ -225,6 +233,7 @@ getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.17", defaul getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4.3", default-features = false, features = ["std", "sys_rng", "wasm_js"] } group = { version = "0.13.0", default-features = false, features = ["alloc"] } hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15.5" } +hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17.0" } hashbrown-986da7b5efc2b80e = { package = "hashbrown", version = "0.16.1" } heck = { version = "0.4.1", features = ["unicode"] } hex = { version = "0.4.3", features = ["serde"] } @@ -275,7 +284,7 @@ regex-syntax = { version = "0.8.10" } reqwest-594e8ee84c453af0 = { package = "reqwest", version = "0.13.2", features = ["blocking", "cookies", "json", "query", "stream"] } reqwest-5ef9efb8ec2df382 = { package = "reqwest", version = "0.12.28", features = ["blocking", "json", "stream"] } rsa = { version = "0.9.10", features = ["serde", "sha2"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["fs"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["process", "pty", "termios"] } rustls = { version = "0.23.41" } schemars = { version = "0.8.22", features = ["bytes", "chrono", "semver", "url", "uuid1"] } scopeguard = { version = "1.2.0" } @@ -291,9 +300,10 @@ simd-adler32 = { version = "0.3.8", default-features = false, features = ["std"] similar = { version = "2.7.0", features = ["bytes", "inline", "unicode"] } slab = { version = "0.4.12" } slog = { version = "2.8.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } -smallvec = { version = "1.15.1", default-features = false, features = ["const_new"] } +smallvec = { version = "1.15.1", default-features = false, features = ["const_new", "union"] } spin = { version = "0.9.8" } spki = { version = "0.7.3", default-features = false, features = ["pem", "std"] } +sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb", default-features = false, features = ["ipcc"] } string_cache = { version = "0.8.9" } strum-2f80eeee3b1b6c7e = { package = "strum", version = "0.26.3", features = ["derive"] } strum-754bda37e0fb3874 = { package = "strum", version = "0.27.2", features = ["derive"] } @@ -335,13 +345,14 @@ dof-468e82937335b1c9 = { package = "dof", version = "0.3.0", default-features = dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4.0", default-features = false, features = ["des"] } hyper-rustls = { version = "0.27.7", features = ["http2"] } hyper-util = { version = "0.1.20", features = ["full"] } -linux-raw-sys = { version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } +linux-raw-sys-5ef9efb8ec2df382 = { package = "linux-raw-sys", version = "0.12.1", default-features = false, features = ["auxvec", "elf", "errno", "general", "ioctl", "no_std", "prctl"] } +linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } miniz_oxide = { version = "0.8.9", default-features = false, features = ["simd", "with-alloc"] } mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-unknown-linux-gnu.build-dependencies] @@ -350,13 +361,14 @@ dof-468e82937335b1c9 = { package = "dof", version = "0.3.0", default-features = dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4.0", default-features = false, features = ["des"] } hyper-rustls = { version = "0.27.7", features = ["http2"] } hyper-util = { version = "0.1.20", features = ["full"] } -linux-raw-sys = { version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } +linux-raw-sys-5ef9efb8ec2df382 = { package = "linux-raw-sys", version = "0.12.1", default-features = false, features = ["auxvec", "elf", "errno", "general", "ioctl", "no_std", "prctl"] } +linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } miniz_oxide = { version = "0.8.9", default-features = false, features = ["simd", "with-alloc"] } mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-apple-darwin.dependencies] @@ -369,7 +381,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-apple-darwin.build-dependencies] @@ -382,7 +394,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.aarch64-apple-darwin.dependencies] @@ -395,7 +407,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.aarch64-apple-darwin.build-dependencies] @@ -408,7 +420,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-unknown-illumos.dependencies] @@ -426,7 +438,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } winnow = { version = "1.0.3" } @@ -446,7 +458,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } winnow = { version = "1.0.3" } From 552801fb0fce02ff41e7431ba1f447d4fa509689 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 18:01:36 -0600 Subject: [PATCH 02/18] Log the start of sush tasks Co-Authored-By: Claude Mythos 5 --- sled-agent/src/long_running_tasks.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs index c0b74859bb9..27549ec91c2 100644 --- a/sled-agent/src/long_running_tasks.rs +++ b/sled-agent/src/long_running_tasks.rs @@ -221,6 +221,7 @@ pub async fn spawn_all_longrunning_tasks( // `crate::server::Server::start` serves the same API there too. let sush = match &config.sush { Some(sush_config) => { + info!(log, "Starting sush tasks"); spawn_sush_tasks( log, sush_config, From 1b65c0fa80bc66ddc8cdccb662e48b1d6234c075 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 18:25:43 -0600 Subject: [PATCH 03/18] Fix SushConfig::ramdisk_dir doc Co-Authored-By: Claude Mythos 5 --- sled-agent/src/config.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index af4683292bb..4b92545cdd1 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -138,8 +138,7 @@ pub struct SushConfig { pub roots: Vec, /// Where to record job output before an encrypted dataset is available. - /// This is on the ramdisk, so it costs global zone memory and does not - /// survive a reboot. + /// Must be on a ramdisk so that it is never persisted unencrypted. #[serde(default = "SushConfig::default_ramdisk_dir")] pub ramdisk_dir: Utf8PathBuf, From 7f810f4c2dae97f0c54c2a7284c0cdd679616308 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 18:41:24 -0600 Subject: [PATCH 04/18] Fix DDM subnet and cubby map docs Also rephrase a `bool.then()` as `if`. Co-Authored-By: Claude Mythos 5 --- clients/ddm-admin-client/src/lib.rs | 18 +++++++++++------- sled-agent/src/sush.rs | 15 +++++++-------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index 462ba47ebea..84cee697204 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -105,10 +105,10 @@ impl Client { self.inner.enable_stats(request).await.map(|resp| resp.into_inner()) } - /// Returns the underlay subnets DDM advertises. Callers must probe - /// each subnet for what they expect to find, because sleds also - /// advertise internal DNS subnets, and RFD 63 reserves a services - /// prefix. + /// Returns every non-bootstrap /64 prefix ddmd has learned. These + /// are not all sled subnets: an internal DNS zone is reachable at + /// its own reserved /64, advertised by the sled hosting it, and + /// RFD 63 reserves others. pub async fn derive_underlay_subnets_from_prefixes( &self, ) -> Result< @@ -118,9 +118,13 @@ impl Client { let prefixes = self.inner.get_prefixes().await?.into_inner(); Ok(prefixes.into_values().flatten().filter_map(|prefix| { let addr = prefix.destination.addr(); - (prefix.destination.width() == SLED_PREFIX_LENGTH - && addr.segments()[0] != BOOTSTRAP_PREFIX) - .then(|| Ipv6Subnet::new(addr)) + if prefix.destination.width() == SLED_PREFIX_LENGTH + && addr.segments()[0] != BOOTSTRAP_PREFIX + { + Some(Ipv6Subnet::new(addr)) + } else { + None + } })) } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 21fc5c440fb..2097afeda0e 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -349,14 +349,13 @@ fn mb_to_bytes(mb: u32) -> u64 { u64::from(mb) * 1024 * 1024 } -/// Keep the cubby map current from MGS's view of the SPs. A job may -/// name its target sled by cubby, and the map says which baseboard -/// is in each cubby. -/// -/// DDM's advertised subnets are only candidates (see -/// [`DdmClient::derive_underlay_subnets_from_prefixes`]), so we must -/// probe for MGS. Each round's answers merge into the existing map, -/// so a probe outage never erases it. +/// Periodically ask MGS which baseboard sits in each cubby, and +/// publish the map. A job may name its target sled by cubby; +/// this map resolves it. MGS answers at a fixed address within its +/// switch zone's subnet, so each round we ask at that address in every +/// /64 ddmd has learned, and take answers from any that respond. The +/// answers merge into the map, so a round that goes unanswered never +/// erases it. async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { let ddm = match DdmClient::localhost(&log) { Ok(ddm) => ddm, From d44a8383a4afc73285470ba6e66e32f1e9f79cde Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 18:57:56 -0600 Subject: [PATCH 05/18] Document poll_ddmd_for_peer_updates Co-Authored-By: Claude Mythos 5 --- sled-agent/src/bootstrap/bootstore_setup.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sled-agent/src/bootstrap/bootstore_setup.rs b/sled-agent/src/bootstrap/bootstore_setup.rs index dd4508feaf9..e833dcf40b2 100644 --- a/sled-agent/src/bootstrap/bootstore_setup.rs +++ b/sled-agent/src/bootstrap/bootstore_setup.rs @@ -77,6 +77,9 @@ fn bootstore_network_config_paths( Ok(paths) } +/// Poll ddmd for the set of peer sleds on the bootstrap network, and +/// publish it to the three consumers that each need it at their own +/// port: the bootstore, the trust quorum node, and sush gossip. pub async fn poll_ddmd_for_peer_updates( log: Logger, bootstore_node_handle: bootstore::NodeHandle, From 3dc9c683c31b6220ba46b1eead96bd7264e49a63 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:02:15 -0600 Subject: [PATCH 06/18] Use ByteCount for byte counts Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 2097afeda0e..a4b7171844f 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -49,6 +49,7 @@ use gateway_types::component::SpType; use omicron_common::address::{ MGS_PORT, SUSH_API_PORT, SUSH_GOSSIP_PORT, get_switch_zone_address, }; +use omicron_common::api::external::ByteCount; use omicron_ddm_admin_client::Client as DdmClient; use sled_agent_config_reconciler::AvailableDatasetsReceiver; use sled_agent_measurements::MeasurementsHandle; @@ -172,7 +173,7 @@ pub async fn spawn_sush_tasks( } let (output_dirs_tx, output_dirs_rx) = watch::channel(OutputDirs::new( config.ramdisk_dir.as_std_path(), - mb_to_bytes(config.ramdisk_max_output_mb), + ByteCount::from_mebibytes_u32(config.ramdisk_max_output_mb).to_bytes(), )); if config.roots.is_empty() { @@ -283,7 +284,7 @@ pub async fn spawn_sush_tasks( log.clone(), available_datasets_rx, output_dirs_tx, - mb_to_bytes(config.max_output_mb), + ByteCount::from_mebibytes_u32(config.max_output_mb).to_bytes(), )); info!(log, "started sush job manager"); @@ -345,10 +346,6 @@ async fn promote_output_dir( } } -fn mb_to_bytes(mb: u32) -> u64 { - u64::from(mb) * 1024 * 1024 -} - /// Periodically ask MGS which baseboard sits in each cubby, and /// publish the map. A job may name its target sled by cubby; /// this map resolves it. MGS answers at a fixed address within its From ae91da078135a9cbd6506d1bb4dece2872e8e815 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:19:16 -0600 Subject: [PATCH 07/18] Return an anyhow::Result from SushHandles::start_api Co-Authored-By: Claude Mythos 5 --- sled-agent/src/server.rs | 2 +- sled-agent/src/sush.rs | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index c4538d47c45..6b9cd6de56c 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -99,7 +99,7 @@ impl Server { match handles.start_api(*sled_address.ip()) { Ok(server) => Some((handles, server)), Err(error) => { - warn!(log, "failed to start sush server"; "error" => error); + warn!(log, "failed to start sush server"; "error" => #%error); None } } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index a4b7171844f..8c625b8a3fe 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -42,6 +42,7 @@ //! which sush uses to synchronize job and event sets. use crate::config::SushConfig; +use anyhow::Context; use camino::Utf8PathBuf; use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer, ServerBuilder}; use gateway_client::Client as MgsClient; @@ -102,11 +103,11 @@ impl SushHandles { pub fn start_api( &self, ip: Ipv6Addr, - ) -> Result>, String> { + ) -> anyhow::Result>> { let bind_address = SocketAddr::V6(SocketAddrV6::new(ip, SUSH_API_PORT, 0, 0)); let api = sush_api::sush_api_mod::api_description::() - .map_err(|err| format!("failed to describe sush API: {err}"))?; + .context("describing the sush API")?; let server = ServerBuilder::new( api, Arc::clone(&self.manager), @@ -122,7 +123,7 @@ impl SushHandles { compression: Default::default(), }) .start() - .map_err(|err| err.to_string())?; + .context("starting the sush API server")?; info!( self.log, "started sush server"; "address" => %bind_address, @@ -301,7 +302,7 @@ pub async fn spawn_sush_tasks( Err(err) => warn!( handles.log, "sush is not serving on the bootstrap network"; - "error" => err, + "error" => #%err, ), } From 301e9d59ebe7b4a435ddf43f5c740cccd220d8da Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:19:52 -0600 Subject: [PATCH 08/18] Kill useless GossipInputs doc --- sled-agent/src/sush.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 8c625b8a3fe..e172adf04f5 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -137,9 +137,6 @@ impl SushHandles { } } -/// What gossip needs from the sled: its sprockets identity, its reference -/// measurements, the bootstrap address to listen on, and where to find its -/// peers. pub struct GossipInputs { pub sprockets: SprocketsConfig, pub measurements: Arc, From 59f30ff40a119a0a0f1e861af29dd0f63bb3f207 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:27:00 -0600 Subject: [PATCH 09/18] Wrap long log line --- sled-agent/src/sush.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index e172adf04f5..8de9c3a0939 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -334,7 +334,8 @@ async fn promote_output_dir( } Err(err) => error!( log, - "could not create job output directory on encrypted dataset, leaving it on the ramdisk"; + "could not create job output directory on encrypted dataset, \ + leaving it on the ramdisk"; "directory" => %dir, "error" => InlineErrorChain::new(&err), ), From 4e9c78ed6f483cec7aa61791bbb2543e6a840052 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:43:06 -0600 Subject: [PATCH 10/18] Warn if the config reconciler task goes away Co-Authored-By: Claude Mythos 5 --- sled-agent/config-reconciler/src/handle.rs | 10 +++++++--- sled-agent/src/sush.rs | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sled-agent/config-reconciler/src/handle.rs b/sled-agent/config-reconciler/src/handle.rs index 947f30bfef6..3c2f71572e2 100644 --- a/sled-agent/config-reconciler/src/handle.rs +++ b/sled-agent/config-reconciler/src/handle.rs @@ -20,6 +20,7 @@ use sled_storage::nested_dataset::NestedDatasetConfig; use sled_storage::nested_dataset::NestedDatasetListOptions; use sled_storage::nested_dataset::NestedDatasetLocation; use slog::Logger; +use slog::warn; use std::collections::HashSet; use std::sync::Arc; use std::sync::OnceLock; @@ -650,12 +651,15 @@ impl AvailableDatasetsReceiver { /// Returns immediately if the set has changed since this receiver last /// observed it. The test variants' datasets never change, so for them this /// never returns. - pub async fn changed(&mut self) { + pub async fn changed(&mut self, log: &Logger) { match &mut self.inner { AvailableDatasetsReceiverInner::Real(receiver) => { - // An error means the reconciler task is gone, in which case the - // set can never change again. if receiver.changed().await.is_err() { + warn!( + log, + "config reconciler task is gone; the set of available \ + datasets will never change again", + ); std::future::pending().await } } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 8de9c3a0939..83192941cbf 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -341,7 +341,7 @@ async fn promote_output_dir( ), } } - available_datasets_rx.changed().await; + available_datasets_rx.changed(&log).await; } } From 5a652270de63002fc3966601f1940230f69b8b72 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:47:14 -0600 Subject: [PATCH 11/18] fix locker link Co-authored-by: Eliza Weisman --- sled-agent/src/sush.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 83192941cbf..e759292abac 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -31,7 +31,7 @@ //! is stored in a _bookmark_. //! //! The bookmark, like every record sush must trust across reboots, -//! lives in the [sush locker](https://github.com/oxidecomputer/sush/blob/main/server/src/locker.rs). +//! lives in the [sush locker]. //! Each locker record is one file on each M.2. Stores write every //! copy, and loads adopt a record only when the copies show it cannot //! be stale. The locker is not a small bootstore: the bootstore holds @@ -40,6 +40,9 @@ //! guarantees are critical to the correctness of the //! [rumors](https://github.com/oxidecomputer/rumors) gossip algorithm, //! which sush uses to synchronize job and event sets. +//! +//! [sush locker]: +//! https://github.com/oxidecomputer/sush/blob/main/server/src/locker.rs use crate::config::SushConfig; use anyhow::Context; From 3c7792c50267455128009e05090fedfd856c6dce Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:48:04 -0600 Subject: [PATCH 12/18] Declaude Co-authored-by: Eliza Weisman --- sled-agent/src/long_running_tasks.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs index 27549ec91c2..beca56ac781 100644 --- a/sled-agent/src/long_running_tasks.rs +++ b/sled-agent/src/long_running_tasks.rs @@ -215,10 +215,11 @@ pub async fn spawn_all_longrunning_tasks( let health_monitor = spawn_health_monitor_tasks(log).await; - // sush must work when the control plane doesn't, so it starts here rather - // than waiting for rack membership, and serves its API on the bootstrap - // network. Once this sled is told its underlay address, - // `crate::server::Server::start` serves the same API there too. + // Because sush must work even when the control plane doesn't, we start it + // before waiting for rack membership. Initially, the sush API is served on + // the bootstrap network. Once the sled is told its underlay address, + // `crate::server::Server::start` will also serve the sush API on the + // bootstrap network. let sush = match &config.sush { Some(sush_config) => { info!(log, "Starting sush tasks"); From b1eefa9dbe7a2683b2b0ab0062aceff19088f4c0 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:53:50 -0600 Subject: [PATCH 13/18] better corpus closure Co-authored-by: Eliza Weisman --- sled-agent/src/sush.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index e759292abac..f0077d44b83 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -197,15 +197,19 @@ pub async fn spawn_sush_tasks( } let locker = Locker::new(&log, slots); - // Gossip re-reads the attestation corpus on every handshake, because a - // software update changes it. A sled that cannot gossip still serves local - // jobs. let GossipInputs { sprockets, measurements, bootstrap_ip, peers } = gossip; + + // On every gossip protocol handshake, we must re-read the attestation + // corpus, because a software update may have changed it. This closure + // is invoked in order to do that. let corpus: CorpusSource = Arc::new({ let log = log.clone(); move || match measurements.current_measurements() { Ok(corpus) => corpus, Err(e) => { + // If reading the measurements fails, this sled cannot + // participate in the gossip protocol, but can still serve + // local jobs. error!(log, "measurement error"; e); vec![] } From 4c7ccb146bf135ee8c4ca40ea342ecf3383a6265 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 20:05:52 -0600 Subject: [PATCH 14/18] Fix up applied suggestions Co-Authored-By: Claude Mythos 5 --- sled-agent/src/long_running_tasks.rs | 2 +- sled-agent/src/sush.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs index beca56ac781..cbd1671cdba 100644 --- a/sled-agent/src/long_running_tasks.rs +++ b/sled-agent/src/long_running_tasks.rs @@ -219,7 +219,7 @@ pub async fn spawn_all_longrunning_tasks( // before waiting for rack membership. Initially, the sush API is served on // the bootstrap network. Once the sled is told its underlay address, // `crate::server::Server::start` will also serve the sush API on the - // bootstrap network. + // underlay network. let sush = match &config.sush { Some(sush_config) => { info!(log, "Starting sush tasks"); diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index f0077d44b83..0e5736bee1b 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -198,7 +198,7 @@ pub async fn spawn_sush_tasks( let locker = Locker::new(&log, slots); let GossipInputs { sprockets, measurements, bootstrap_ip, peers } = gossip; - + // On every gossip protocol handshake, we must re-read the attestation // corpus, because a software update may have changed it. This closure // is invoked in order to do that. From ecaf96d4b949ab286f6e8c77ae233b702451c4a0 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 20:25:36 -0600 Subject: [PATCH 15/18] Improve sush module doc Co-Authored-By: finch Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 43 ++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 0e5736bee1b..75cc0da40a4 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -23,23 +23,34 @@ //! and is called from `server` once this sled has been told its //! underlay address. //! -//! Gossip runs over sprockets on the bootstrap network, so jobs and -//! sessions are shared across sleds. A universe is a shared gossip -//! history; peers that meet merge into one by a dominance rule. -//! A restarted sled re-seeds its universe, rejoins the rack's, then -//! replays its history without re-executing it. Its gossip identity -//! is stored in a _bookmark_. +//! Gossip runs over sprockets on the bootstrap network. A job may be +//! addressed to a single sled, but the messages carrying jobs, +//! sessions, and events spread to every sled. The set of causally +//! related messages is called a _universe_: peers in the same universe +//! can gossip and converge on its contents; peers in different +//! universes cannot gossip at all. Every identity in a universe +//! descends from a single seed. When sleds meet, a deterministic rule +//! picks whose seed wins, and the others bootstrap from it, each +//! taking a slice of the winner's identity space. A restarted sled +//! therefore rejoins the rack's universe and replays what it missed +//! without re-executing it. Each sled stores its identity in a +//! _bookmark_, so that a restart resumes it rather than growing a new +//! one on every boot. //! -//! The bookmark, like every record sush must trust across reboots, -//! lives in the [sush locker]. -//! Each locker record is one file on each M.2. Stores write every -//! copy, and loads adopt a record only when the copies show it cannot -//! be stale. The locker is not a small bootstore: the bootstore holds -//! rack-wide facts that sleds may recover from their peers, but a locker -//! record says what this sled itself did or committed to do. These -//! guarantees are critical to the correctness of the -//! [rumors](https://github.com/oxidecomputer/rumors) gossip algorithm, -//! which sush uses to synchronize job and event sets. +//! The bookmark, like every record sush must trust across reboots, lives +//! in the [sush locker]. When we write a record to the locker, we actually +//! write two copies, one to each M.2. When we load a record, it only +//! succeeds if both copies are exact matches, or if one of them is +//! entirely missing (the latter to handle the case of M.2 hardware +//! replacement in the field). A load fails on mismatched copies, so that +//! a torn write or corruption on one drive cannot induce the reader to +//! load stale or invalid information from the other drive. This turns the +//! pair of M.2 drives into a single mirrored storage container that fails +//! closed on any disagreement. We accept this because an M.2 failure is +//! considered a non-user-replaceable part failure, for which the solution +//! is an RMA. If this occurs in the field, sush may refuse to run jobs on +//! the sled containing the failed M.2; we report this error to the user, +//! who should replace the sled. //! //! [sush locker]: //! https://github.com/oxidecomputer/sush/blob/main/server/src/locker.rs From 6ef1b0662dabaaf81c8b953bfe949a290c80062a Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 21:40:28 -0600 Subject: [PATCH 16/18] Update sush: `isolated` & `lonely` are now associated functions Co-Authored-By: Claude Mythos 5 --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- sled-agent/src/sush.rs | 6 ++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69566a16986..9f81aca3fb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15663,7 +15663,7 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "sush-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +source = "git+https://github.com/oxidecomputer/sush?rev=2ceb9d92ba2d607018d044ce709cc49b94bf040b#2ceb9d92ba2d607018d044ce709cc49b94bf040b" dependencies = [ "chrono", "dropshot", @@ -15679,7 +15679,7 @@ dependencies = [ [[package]] name = "sush-common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +source = "git+https://github.com/oxidecomputer/sush?rev=2ceb9d92ba2d607018d044ce709cc49b94bf040b#2ceb9d92ba2d607018d044ce709cc49b94bf040b" dependencies = [ "bytes", "bytesize", @@ -15714,7 +15714,7 @@ dependencies = [ [[package]] name = "sush-server" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +source = "git+https://github.com/oxidecomputer/sush?rev=2ceb9d92ba2d607018d044ce709cc49b94bf040b#2ceb9d92ba2d607018d044ce709cc49b94bf040b" dependencies = [ "async-trait", "atomicwrites", diff --git a/Cargo.toml b/Cargo.toml index bdd036dd8e8..f52c6434a5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -861,9 +861,9 @@ subtle = "2.6.1" supports-color = "3.0.2" support-bundle-collection = { path = "support-bundle-collection" } support-bundle-viewer = "0.1.2" -sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6" } -sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6" } -sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6", features = ["embedded"] } +sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b" } +sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b" } +sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b", features = ["embedded"] } swrite = "0.1.0" sync-ptr = "0.1.4" libsw = { version = "3.5.0", features = ["tokio"] } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 75cc0da40a4..6bdbd788452 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -84,7 +84,9 @@ use tokio_util::sync::CancellationToken; use sush_common::targets::{Cubbies, MAX_CUBBY}; use sush_server::executor::PathIsolation; -use sush_server::gossip::{GossipConfig, isolated, lonely, spawn_gossip}; +use sush_server::gossip::{ + GossipConfig, LinkedBaseboards, Universe, spawn_gossip, +}; use sush_server::link::CorpusSource; use sush_server::locker::Locker; use sush_server::output::{JobOutputDir, OutputDirs}; @@ -250,7 +252,7 @@ pub async fn spawn_sush_tasks( // storing. The null locker stores nothing, leaving the // bookmark on the M.2s untouched for the next boot. let seed = seed_gossip(&log, &Locker::null()).await; - (isolated(seed.into_rumors()), lonely()) + (Universe::isolated(seed.into_rumors()), LinkedBaseboards::lonely()) } }; From e50003de55ff32420b0e89264f55690055a3f4a0 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 21:49:39 -0600 Subject: [PATCH 17/18] Improve REQUEST_MAX_BODY_BYTES doc Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 6bdbd788452..1f750aaaea8 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -102,8 +102,14 @@ const MGS_POLL_INTERVAL: Duration = Duration::from_secs(30); /// How long to wait for an MGS candidate to answer. const MGS_PROBE_TIMEOUT: Duration = Duration::from_secs(5); -/// Maximum size of a request body the API will accept. The largest thing a -/// client sends is a signed job request or a certificate, both small. +/// Maximum size of a request body the API will accept. The protocol +/// defines no message size limits, so we chose this cap rather than +/// deriving it. The largest bodies a client sends are the command in +/// a signed job request and a PEM-encoded certificate. Neither has +/// exceeded a few KB in practice, so this leaves an order of magnitude +/// of headroom while bounding how much the server needs to buffer for +/// any one request. We should raise it if we start seeing significantly +/// larger requests or certs in the wild. const REQUEST_MAX_BODY_BYTES: usize = 0xFFFF; /// Handles to the Support Shell server's tasks. From 48ada88ead76e5ebc28b8c230ae06f4d5980b8b1 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 22:00:05 -0600 Subject: [PATCH 18/18] Reject empty roots at config parsing time Co-Authored-By: Claude Mythos 5 --- sled-agent/src/config.rs | 26 +++++++++++++++++++++++--- sled-agent/src/sush.rs | 3 --- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 4b92545cdd1..b69ea5475c0 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -134,7 +134,10 @@ pub struct Config { pub struct SushConfig { /// PEM files holding the trusted root certificates for job requests, /// one certificate per file. A job whose signature does not chain to - /// one of these is refused, so an empty list means no job can ever run. + /// one of these is refused, so an empty list means no job could ever + /// run, and config parsing rejects it. To run a sled with no Support + /// Shell server, omit the `[sush]` section instead. + #[serde(deserialize_with = "SushConfig::nonempty_roots")] pub roots: Vec, /// Where to record job output before an encrypted dataset is available. @@ -169,6 +172,18 @@ impl SushConfig { fn default_max_output_mb() -> u32 { 10 * 1024 } + + fn nonempty_roots<'de, D: serde::Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + let roots = Vec::deserialize(deserializer)?; + if roots.is_empty() { + return Err(serde::de::Error::custom( + "sush needs at least one root certificate to run", + )); + } + Ok(roots) + } } #[derive(Debug, thiserror::Error)] @@ -275,16 +290,21 @@ mod test { let sush: SushConfig = toml::from_str( r#" - roots = [] + roots = ["/pkg/sush-root.pem"] ramdisk_dir = "/var/run/sush" ramdisk_max_output_mb = 8 max_output_mb = 128 "#, ) .unwrap(); - assert!(sush.roots.is_empty()); assert_eq!(sush.ramdisk_dir, "/var/run/sush"); assert_eq!(sush.ramdisk_max_output_mb, 8); assert_eq!(sush.max_output_mb, 128); + + // No roots would mean no jobs can ever run, so reject at parse time. + let err = toml::from_str::("roots = []").unwrap_err(); + assert!( + err.to_string().contains("needs at least one root certificate") + ); } } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 1f750aaaea8..52fdd845266 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -196,9 +196,6 @@ pub async fn spawn_sush_tasks( ByteCount::from_mebibytes_u32(config.ramdisk_max_output_mb).to_bytes(), )); - if config.roots.is_empty() { - warn!(log, "sush has no root certificates, so no job will ever run"); - } let shutdown = CancellationToken::new(); // A slot that cannot be created is still handed to the locker, which