From 7dc9f6db0380d3ab2e9359928f23b10df72a1f1b Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sun, 6 Sep 2026 16:41:27 -0600 Subject: [PATCH] Add the sush proxy to the switch zone This continues the integration of the [Support Shell](https://github.com/oxidecomputer/sush) ([RFD 620](https://rfd.shared.oxide.computer/rfd/0620)), building on the sled-agent embedding: a new switch zone service, `sush-proxy`, terminates technician-port connections and routes each request to a sled's sush server. The proxy finds sush servers by probing the bootstrap and underlay prefixes given by DDM, and polls MGS for the cubby map so a request may name its target sled by cubby. **For security review:** On real hardware the proxy serves TLS backed by the sled's platform identity. At zone startup, sled-agent generates an ephemeral key and has the RoT sign its certificate with the TQ key (Ed25519 over the SHA3-256 digest of the TBS certificate), and the sush client verifies that the chain ends at a platform identity root. Simulated and emulated environments have no RoT, so their proxies serve plaintext. Co-Authored-By: Claude Mythos 5 --- .cargo/xtask.toml | 3 + Cargo.lock | 31 ++- Cargo.toml | 2 + clients/ddm-admin-client/src/lib.rs | 12 ++ common/src/address.rs | 1 + dev-tools/ls-apis/api-manifest.toml | 25 +++ dev-tools/ls-apis/tests/api_dependencies.out | 2 + package-manifest.toml | 13 ++ sled-agent/Cargo.toml | 2 +- sled-agent/src/services.rs | 97 ++++++++- sled-agent/src/sush.rs | 108 +++++++++- smf/sush-proxy/manifest.xml | 49 +++++ sush-proxy/Cargo.toml | 30 +++ sush-proxy/src/lib.rs | 213 +++++++++++++++++++ sush-proxy/src/main.rs | 85 ++++++++ workspace-hack/Cargo.toml | 20 +- 16 files changed, 677 insertions(+), 16 deletions(-) create mode 100644 smf/sush-proxy/manifest.xml create mode 100644 sush-proxy/Cargo.toml create mode 100644 sush-proxy/src/lib.rs create mode 100644 sush-proxy/src/main.rs diff --git a/.cargo/xtask.toml b/.cargo/xtask.toml index 76d7b342d94..2c491e54005 100644 --- a/.cargo/xtask.toml +++ b/.cargo/xtask.toml @@ -28,6 +28,8 @@ [libraries."libxmlsec1.so.1"] # libipcc should only be depended on by binaries that communicate with ipcc. +# sush-proxy does not, but the workspace builds it alongside sled-agent, whose +# ipcc feature unifies onto their shared sprockets-tls. [libraries."libipcc.so.1"] binary_allow_list = [ "installinator", @@ -36,6 +38,7 @@ binary_allow_list = [ "omicron-dev", "sled-agent", "sled-agent-sim", + "sush-proxy", ] # libnvme is a global zone only library and therefore we must be sure that only diff --git a/Cargo.lock b/Cargo.lock index 69566a16986..62b1ce8f6b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9974,6 +9974,7 @@ dependencies = [ "cc", "chacha20 0.10.0", "chrono", + "cipher 0.4.4", "cipher 0.5.2", "clap", "clap_builder", @@ -10067,10 +10068,9 @@ dependencies = [ "ppv-lite86", "predicates", "proc-macro2", + "qorb", "quote", - "rand 0.8.6", "rand 0.9.2", - "rand_chacha 0.3.1", "rand_chacha 0.9.0", "regex", "regex-automata", @@ -15711,6 +15711,33 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "sush-proxy" +version = "0.1.0" +dependencies = [ + "anyhow", + "camino", + "clap", + "futures", + "gateway-client", + "gateway-types", + "omicron-common", + "omicron-ddm-admin-client", + "omicron-workspace-hack", + "oxide-tokio-rt", + "reqwest 0.13.2", + "sled-hardware-types", + "slog", + "slog-async", + "slog-dtrace", + "slog-term", + "sprockets-tls", + "sush-common", + "sush-server", + "tokio", + "tokio-util", +] + [[package]] name = "sush-server" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index bdd036dd8e8..0451a62be92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,6 +171,7 @@ members = [ "sled-storage/zfs-test-harness", "sp-sim", "support-bundle-collection", + "sush-proxy", "test-utils", "trust-quorum", "trust-quorum/gfss", @@ -368,6 +369,7 @@ default-members = [ "sled-storage/zfs-test-harness", "sp-sim", "support-bundle-collection", + "sush-proxy", "trust-quorum", "trust-quorum/gfss", "trust-quorum/protocol", diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index 462ba47ebea..4ddb0e4a9a2 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -16,6 +16,7 @@ use omicron_common::address::BOOTSTRAP_SLED_SUBNET_PREFIX_LENGTH; use omicron_common::address::DDMD_PORT; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX_LENGTH; +use omicron_common::address::get_sled_address; use oxnet::Ipv6Net; use sled_hardware_types::underlay::BootstrapInterface; use slog::Logger; @@ -105,6 +106,17 @@ impl Client { self.inner.enable_stats(request).await.map(|resp| resp.into_inner()) } + /// Returns the sled address behind each subnet + /// [`Self::derive_underlay_subnets_from_prefixes`] returns, and the + /// same caveat applies: callers must probe for the sleds they + /// expect to find. + pub async fn derive_sled_addrs_from_prefixes( + &self, + ) -> Result + use<>, DdmError> { + let subnets = self.derive_underlay_subnets_from_prefixes().await?; + Ok(subnets.map(get_sled_address)) + } + /// Returns the underlay subnets DDM advertises. Callers must probe /// each subnet for what they expect to find, because sleds also /// advertise internal DNS subnets, and RFD 63 reserves a services diff --git a/common/src/address.rs b/common/src/address.rs index d8c36c0f196..bb815401587 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -234,6 +234,7 @@ pub const REPO_DEPOT_PORT: u16 = 12348; pub const TRUST_QUORUM_PORT: u16 = 12349; pub const SUSH_API_PORT: u16 = 12350; pub const SUSH_GOSSIP_PORT: u16 = 12351; +pub const SUSH_PROXY_PORT: u16 = 12352; pub const BOOTSTRAP_AGENT_LOCKSTEP_PORT: u16 = 8080; diff --git a/dev-tools/ls-apis/api-manifest.toml b/dev-tools/ls-apis/api-manifest.toml index 34685d5a1f6..38b0a44b7f0 100644 --- a/dev-tools/ls-apis/api-manifest.toml +++ b/dev-tools/ls-apis/api-manifest.toml @@ -58,6 +58,7 @@ packages = [ "lldpd", "mgd", "omicron-gateway", + "sush-proxy", "tfportd", "wicketd", ] @@ -774,6 +775,30 @@ permalinks = [ "https://github.com/oxidecomputer/omicron/blob/6cef874/sled-agent/src/instance.rs#L2283", ] +[[intra_deployment_unit_only_edges]] +server = "sush-proxy" +client = "ddm-admin-client" +note = """ +sush-proxy discovers sleds through its own switch zone's ddmd, always +via Client::localhost (sush-proxy/src/lib.rs). +""" +permalinks = [ + "https://github.com/oxidecomputer/omicron/blob/main/sush-proxy/src/lib.rs", +] + +[[intra_deployment_unit_only_edges]] +server = "sush-proxy" +client = "gateway-client" +note = """ +sush-proxy reads the cubby map from its own switch zone's MGS. +sled-agent sets the mgs-address SMF property to [::1] (services.rs), +and the manifest passes it via --mgs-address. +""" +permalinks = [ + "https://github.com/oxidecomputer/omicron/blob/main/sled-agent/src/services.rs", + "https://github.com/oxidecomputer/omicron/blob/main/smf/sush-proxy/manifest.xml", +] + [[intra_deployment_unit_only_edges]] server = "propolis-server" client = "sled-agent-client" diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index 60c7a46bd59..61c4ee92ebd 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -27,6 +27,7 @@ Maghemite DDM Admin (client: ddm-admin-client) consumed by: mgd (maghemite/mgd) via 1 path consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path consumed by: sled-agent-rack-setup (omicron/sled-agent/rack-setup) via 1 path [embedded in omicron-sled-agent; rack-init only] + consumed by: sush-proxy (omicron/sush-proxy) via 1 path consumed by: wicketd (omicron/wicketd) via 1 path DNS Server (client: dns-service-client) @@ -50,6 +51,7 @@ Management Gateway Service (client: gateway-client) consumed by: mgd (maghemite/mgd) via 1 path consumed by: omicron-nexus (omicron/nexus) via 6 paths consumed by: omicron-sled-agent (omicron/sled-agent) via 2 paths + consumed by: sush-proxy (omicron/sush-proxy) via 1 path consumed by: wicketd (omicron/wicketd) via 3 paths Wicketd Installinator (client: installinator-client) diff --git a/package-manifest.toml b/package-manifest.toml index edb753576ce..5bb3c7fd3e9 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -525,6 +525,16 @@ source.paths = [{ from = "smf/wicketd", to = "/var/svc/manifest/site/wicketd" }] output.type = "zone" output.intermediate_only = true +[package.sush-proxy] +service_name = "sush-proxy" +only_for_targets.image = "standard" +source.type = "local" +source.rust.binary_names = ["sush-proxy"] +source.rust.release = true +source.paths = [{ from = "smf/sush-proxy", to = "/var/svc/manifest/site/sush-proxy" }] +output.type = "zone" +output.intermediate_only = true + [package.wicket] service_name = "wicket" only_for_targets.image = "standard" @@ -856,6 +866,7 @@ source.packages = [ "pumpkind.tar.gz", "wicketd.tar.gz", "wicket.tar.gz", + "sush-proxy.tar.gz", "mg-ddm.tar.gz", "mgd.tar.gz", "switch_zone_setup.tar.gz", @@ -883,6 +894,7 @@ source.packages = [ "lldp.tar.gz", "wicketd.tar.gz", "wicket.tar.gz", + "sush-proxy.tar.gz", "mg-ddm.tar.gz", "mgd.tar.gz", "switch_zone_setup.tar.gz", @@ -910,6 +922,7 @@ source.packages = [ "lldp.tar.gz", "wicketd.tar.gz", "wicket.tar.gz", + "sush-proxy.tar.gz", "mg-ddm.tar.gz", "mgd.tar.gz", "switch_zone_setup.tar.gz", diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 3514f8dd2c9..71361c65da4 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -129,7 +129,7 @@ tufaceous-brand-metadata.workspace = true usdt.workspace = true uuid.workspace = true walkdir.workspace = true -x509-cert.workspace = true +x509-cert = { workspace = true, features = ["std"] } zeroize.workspace = true zip.workspace = true zone.workspace = true diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 53278932687..6ddc5fc109a 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -27,6 +27,9 @@ use crate::config::SidecarRevision; use crate::ddm_reconciler::DdmReconciler; use crate::metrics::MetricsRequestQueue; use crate::profile::*; +use crate::sush::{ + SUSH_PROXY_CERT_CHAIN_PATH, SUSH_PROXY_KEY_PATH, generate_proxy_identity, +}; use camino::{Utf8Path, Utf8PathBuf}; use clickhouse_admin_types::CLICKHOUSE_KEEPER_CONFIG_DIR; use clickhouse_admin_types::CLICKHOUSE_KEEPER_CONFIG_FILE; @@ -64,6 +67,7 @@ use omicron_common::address::MGS_PORT; use omicron_common::address::NTP_ADMIN_PORT; use omicron_common::address::RACK_PREFIX_LENGTH; use omicron_common::address::SLED_PREFIX_LENGTH; +use omicron_common::address::SUSH_PROXY_PORT; use omicron_common::address::TFPORTD_PORT; use omicron_common::address::WICKETD_COMMISSION_PORT; use omicron_common::address::WICKETD_NEXUS_PROXY_PORT; @@ -462,6 +466,25 @@ enum SwitchService { MgDdm { mode: String }, Mgd, SpSim, + SushProxy { tls: SushProxyTls, baseboard: Baseboard }, +} + +/// How the sush proxy authenticates itself to clients. +#[derive(Clone, Copy, Debug, PartialEq)] +enum SushProxyTls { + /// An ephemeral key with a certificate signed by the RoT at zone startup. + Platform, + /// No authentication. For development images only, which have no RoT. + Insecure, +} + +impl SushProxyTls { + fn as_str(&self) -> &'static str { + match self { + SushProxyTls::Platform => "platform", + SushProxyTls::Insecure => "insecure", + } + } } impl illumos_utils::smf_helper::Service for SwitchService { @@ -477,6 +500,7 @@ impl illumos_utils::smf_helper::Service for SwitchService { SwitchService::MgDdm { .. } => "mg-ddm", SwitchService::Mgd => "mgd", SwitchService::SpSim => "sp-sim", + SwitchService::SushProxy { .. } => "sush-proxy", } } fn smf_name(&self) -> String { @@ -2407,6 +2431,7 @@ impl ServiceManager { let mut mgd_service = ServiceBuilder::new("oxide/mgd"); let mut mg_ddm_service = ServiceBuilder::new("oxide/mg-ddm"); let mut uplink_service = ServiceBuilder::new("oxide/uplink"); + let mut sush_proxy_service = ServiceBuilder::new("oxide/sush-proxy"); let mut switch_zone_setup_config = PropertyGroupBuilder::new("config") .add_property( @@ -2482,6 +2507,60 @@ impl ServiceManager { SwitchService::SpSim => { info!(self.inner.log, "Setting up Simulated SP service"); } + SwitchService::SushProxy { tls, baseboard } => { + info!(self.inner.log, "Setting up sush-proxy service"); + if let SushProxyTls::Platform = tls { + if let Err(err) = generate_proxy_identity( + &self.inner.log, + &installed_zone.root(), + ) + .await + { + error!( + self.inner.log, + "failed to generate the sush proxy TLS identity"; + "error" => #%err, + ); + } + } + let config = PropertyGroupBuilder::new("config") + // Bind `::` so the proxy serves all interfaces, + // particularly the tech ports. + .add_property( + "address", + "astring", + &format!("[::]:{SUSH_PROXY_PORT}"), + ) + .add_property( + "mgs-address", + "astring", + &format!("[::1]:{MGS_PORT}"), + ) + .add_property("tls", "astring", tls.as_str()) + .add_property( + "home", + "astring", + &format!( + "{}:{}", + baseboard.model(), + baseboard.identifier() + ), + ) + .add_property( + "priv-key", + "astring", + SUSH_PROXY_KEY_PATH, + ) + .add_property( + "cert-chain", + "astring", + SUSH_PROXY_CERT_CHAIN_PATH, + ); + sush_proxy_service = sush_proxy_service.add_instance( + ServiceInstanceBuilder::new("default") + .add_property_group(config), + ); + } SwitchService::Wicketd { baseboard } => { info!(self.inner.log, "Setting up wicketd service"); // If we're launching the switch zone, we'll have a @@ -3000,7 +3079,8 @@ impl ServiceManager { .add_service(pumpkind_service) .add_service(mgd_service) .add_service(mg_ddm_service) - .add_service(uplink_service); + .add_service(uplink_service) + .add_service(sush_proxy_service); // If we have the rack subnet, also set up /etc/resolv.conf. if let Some(info) = info { @@ -3131,6 +3211,10 @@ impl ServiceManager { SwitchService::Wicketd { baseboard: baseboard.clone() }, SwitchService::Mgd, SwitchService::MgDdm { mode: "transit".to_string() }, + SwitchService::SushProxy { + tls: SushProxyTls::Platform, + baseboard: baseboard.clone(), + }, ] } @@ -3151,6 +3235,10 @@ impl ServiceManager { asic, }, SwitchService::SpSim, + SwitchService::SushProxy { + tls: SushProxyTls::Insecure, + baseboard: baseboard.clone(), + }, ] } @@ -3182,6 +3270,10 @@ impl ServiceManager { asic, }, SwitchService::SpSim, + SwitchService::SushProxy { + tls: SushProxyTls::Insecure, + baseboard: baseboard.clone(), + }, ] } }; @@ -3553,6 +3645,9 @@ impl ServiceManager { SwitchService::SpSim => { // nothing to configure } + SwitchService::SushProxy { .. } => { + // nothing to configure + } SwitchService::Mgd => { info!(self.inner.log, "configuring mgd service"); smfh.delpropvalue_default_instance( diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 21fc5c440fb..1581f4856fd 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -42,7 +42,8 @@ //! which sush uses to synchronize job and event sets. use crate::config::SushConfig; -use camino::Utf8PathBuf; +use anyhow::Context; +use camino::{Utf8Path, Utf8PathBuf}; use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer, ServerBuilder}; use gateway_client::Client as MgsClient; use gateway_types::component::SpType; @@ -50,22 +51,34 @@ use omicron_common::address::{ MGS_PORT, SUSH_API_PORT, SUSH_GOSSIP_PORT, get_switch_zone_address, }; use omicron_ddm_admin_client::Client as DdmClient; +use sha3::{Digest as _, Sha3_256}; use sled_agent_config_reconciler::AvailableDatasetsReceiver; use sled_agent_measurements::MeasurementsHandle; use sled_hardware_types::BaseboardId; use slog::{Logger, debug, error, info, o, warn}; use slog_error_chain::InlineErrorChain; +use sprockets_tls::ipcc::Ipcc; use sprockets_tls::keys::SprocketsConfig; use std::collections::BTreeSet; +use std::io; +use std::iter::once; use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use std::sync::Arc; use std::time::Duration; -use tokio::fs::create_dir_all; +use tokio::fs::{OpenOptions, create_dir_all}; +use tokio::io::AsyncWriteExt as _; use tokio::spawn; use tokio::sync::watch; +use tokio::task::spawn_blocking; use tokio::time::sleep; use tokio_util::sync::CancellationToken; +use x509_cert::Certificate; +use x509_cert::der::oid::db::rfc8410::ID_ED_25519; +use x509_cert::der::{Decode as _, Reader as _, SliceReader}; +use x509_cert::spki::AlgorithmIdentifierOwned; +use x509_cert::time::Validity; +use sush_common::keys::{EphemeralKey, KeyType, pem_cert_chain}; use sush_common::targets::{Cubbies, MAX_CUBBY}; use sush_server::executor::PathIsolation; use sush_server::gossip::{GossipConfig, isolated, lonely, spawn_gossip}; @@ -78,6 +91,17 @@ use sush_server::{JobManager, seed_gossip}; /// Subdirectory of an encrypted dataset that job output is recorded in. const SUSH_OUTPUT_SUBDIR: &str = "sush"; +/// Path inside the switch zone to the sush proxy's TLS private key. +pub const SUSH_PROXY_KEY_PATH: &str = "/etc/sush-proxy/key.pem"; + +/// Path inside the switch zone to the sush proxy's TLS certificate chain. +pub const SUSH_PROXY_CERT_CHAIN_PATH: &str = "/etc/sush-proxy/chain.pem"; + +/// How long a generated proxy identity claims to be valid. Nothing checks +/// expiry today, and every zone startup generates a fresh identity. +const SUSH_PROXY_CERT_VALIDITY: Duration = + Duration::from_secs(365 * 24 * 60 * 60); + /// How often to refresh the cubby map from MGS. const MGS_POLL_INTERVAL: Duration = Duration::from_secs(30); @@ -425,3 +449,83 @@ async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { sleep(MGS_POLL_INTERVAL).await; } } + +/// Generate the switch zone proxy's TLS identity: an ephemeral key whose +/// certificate the RoT signs once, in its signing convention (Ed25519 +/// over the SHA3-256 digest of the TBS certificate). The key and chain +/// are written as PEM under `zone_root` for the proxy to serve with. +pub async fn generate_proxy_identity( + log: &Logger, + zone_root: &Utf8Path, +) -> anyhow::Result<()> { + // IPCC requests are ioctls, i.e., blocking I/O. + let (key_pem, chain_pem) = spawn_blocking(generate_proxy_pems).await??; + let key_path = format!("{zone_root}{SUSH_PROXY_KEY_PATH}"); + let chain_path = format!("{zone_root}{SUSH_PROXY_CERT_CHAIN_PATH}"); + let dir = Utf8Path::new(&key_path).parent().expect("key path has a parent"); + create_dir_all(dir).await.with_context(|| format!("creating {dir}"))?; + write_private(&key_path, key_pem.as_bytes()) + .await + .with_context(|| format!("writing {key_path}"))?; + tokio::fs::write(&chain_path, chain_pem.as_bytes()) + .await + .with_context(|| format!("writing {chain_path}"))?; + info!(log, "generated sush proxy TLS identity"; "key" => key_path); + Ok(()) +} + +/// The proxy's private key and certificate chain, PEM-encoded. +fn generate_proxy_pems() -> anyhow::Result<(String, String)> { + let ipcc = Ipcc::new().context("opening IPCC")?; + let chain_der = + ipcc.rot_get_tq_cert_chain().context("fetching the TQ cert chain")?; + // The RoT returns the chain leaf first, as sprockets assumes too. + let platform = der_cert_chain(&chain_der)?; + let issuer = platform + .first() + .context("the TQ cert chain is empty")? + .tbs_certificate + .subject + .clone(); + let leaf = EphemeralKey::new_delegated( + KeyType::Ed25519, + "CN=sush-proxy".parse().context("parsing the subject")?, + issuer, + Validity::from_now(SUSH_PROXY_CERT_VALIDITY) + .context("computing validity")?, + AlgorithmIdentifierOwned { oid: ID_ED_25519, parameters: None }, + |tbs| ipcc.rot_tq_sign(&Sha3_256::digest(tbs)), + ) + .context("generating the proxy key")?; + let key_pem = leaf.private_key_pem().context("encoding the proxy key")?; + let chain = once(leaf.cert().clone()).chain(platform).collect::>(); + let chain_pem = pem_cert_chain(chain).context("encoding the chain")?; + Ok((key_pem, chain_pem)) +} + +/// Parse a concatenated series of DER certs, as the RoT returns. +fn der_cert_chain(bytes: &[u8]) -> anyhow::Result> { + let mut chain = Vec::new(); + let mut reader = + SliceReader::new(bytes).context("reading the TQ cert chain")?; + while !reader.is_finished() { + chain.push( + Certificate::decode(&mut reader) + .context("parsing the TQ cert chain")?, + ); + } + Ok(chain) +} + +/// Write a file readable only by the owner. +async fn write_private(path: &str, contents: &[u8]) -> io::Result<()> { + let mut file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .mode(0o600) + .open(path) + .await?; + file.write_all(contents).await?; + file.flush().await +} diff --git a/smf/sush-proxy/manifest.xml b/smf/sush-proxy/manifest.xml new file mode 100644 index 00000000000..9778cc57dd1 --- /dev/null +++ b/smf/sush-proxy/manifest.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sush-proxy/Cargo.toml b/sush-proxy/Cargo.toml new file mode 100644 index 00000000000..fa64641b505 --- /dev/null +++ b/sush-proxy/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "sush-proxy" +version = "0.1.0" +edition.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +camino.workspace = true +clap.workspace = true +futures.workspace = true +gateway-client.workspace = true +gateway-types.workspace = true +omicron-common.workspace = true +omicron-ddm-admin-client.workspace = true +oxide-tokio-rt.workspace = true +reqwest = { workspace = true, features = ["json", "rustls"] } +sled-hardware-types.workspace = true +slog.workspace = true +slog-async.workspace = true +slog-dtrace.workspace = true +slog-term.workspace = true +sprockets-tls.workspace = true +sush-common.workspace = true +sush-server.workspace = true +tokio.workspace = true +tokio-util.workspace = true +omicron-workspace-hack.workspace = true diff --git a/sush-proxy/src/lib.rs b/sush-proxy/src/lib.rs new file mode 100644 index 00000000000..7810774026f --- /dev/null +++ b/sush-proxy/src/lib.rs @@ -0,0 +1,213 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The Support Shell proxy for the switch zone (RFD 620). +//! +//! The proxy terminates technician-port connections and routes each +//! request to a sled's `sush` server. It discovers sleds by probing +//! the addresses behind the bootstrap and underlay prefixes DDM +//! advertises, and learns the cubby numbering from MGS. +//! +//! The code that verifies the proxy's TLS identity lives in sush's +//! `client/src/tls.rs`. + +use std::collections::BTreeMap; +use std::net::{SocketAddr, SocketAddrV6}; +use std::time::Duration; + +use anyhow::{Context, Result}; +use camino::Utf8PathBuf; +use futures::future::join_all; +use gateway_client::Client as MgsClient; +use gateway_types::component::SpType; +use omicron_common::address::SUSH_API_PORT; +use omicron_ddm_admin_client::Client as DdmClient; +use sled_hardware_types::BaseboardId; +use sled_hardware_types::underlay::BootstrapInterface; +use slog::{Logger, debug, warn}; +use sprockets_tls::keys::ResolveSetting; +use sush_common::targets::{Cubbies, MAX_CUBBY}; +use sush_server::ProxyServer; +use sush_server::proxy::{Targets, platform_tls}; +use tokio::sync::watch; +use tokio::time::sleep; +use tokio_util::sync::CancellationToken; + +const POLL_INTERVAL: Duration = Duration::from_secs(30); +const PROBE_TIMEOUT: Duration = Duration::from_secs(5); + +/// How the proxy authenticates itself to clients. +#[derive(Clone, Debug)] +pub enum Tls { + /// The sled's platform identity: an ephemeral key that sled-agent + /// generates and the RoT signs once, served from local files. + Platform { priv_key: Utf8PathBuf, cert_chain: Utf8PathBuf }, + /// None. For development images, which have no RoT to sign an + /// identity with. + Insecure, +} + +pub struct Config { + pub address: SocketAddr, + pub mgs_address: SocketAddrV6, + pub tls: Tls, + pub home: Option, +} + +/// Start the proxy and run its discovery loops; only a startup +/// failure returns. +pub async fn run(log: &Logger, config: Config) -> Result<()> { + let tls = match config.tls { + Tls::Platform { priv_key, cert_chain } => Some( + platform_tls(log, ResolveSetting::Local { priv_key, cert_chain }) + .context("loading the TLS identity")?, + ), + Tls::Insecure => None, + }; + let (tx_targets, rx_targets) = watch::channel(Targets::default()); + let _proxy = ProxyServer::start( + log, + config.address, + tls, + rx_targets, + config.home, + CancellationToken::new(), + ) + .await + .context("starting the proxy")?; + + let ddm = DdmClient::localhost(log).context("reaching ddmd")?; + let mgs = mgs_client(log, config.mgs_address); + tokio::join!(sleds(log, ddm, &tx_targets), cubbies(log, mgs, &tx_targets),); + unreachable!("discovery loops never return"); +} + +fn mgs_client(log: &Logger, address: SocketAddrV6) -> MgsClient { + let client = reqwest::ClientBuilder::new() + .connect_timeout(PROBE_TIMEOUT) + .timeout(PROBE_TIMEOUT) + .build() + .expect("failed to build an HTTP client"); + MgsClient::new_with_client( + &format!("http://{address}"), + client, + log.clone(), + ) +} + +/// Keep `Targets::sleds` current. Every DDM-advertised address that +/// answers `/target` is routable. Sleds answer on their bootstrap +/// addresses from boot and on their underlay addresses once RSS +/// assigns them. We probe both, and underlay wins. Each round's +/// answers merge into the existing map, so a missed probe never +/// evicts a sled; a stale address fails at forwarding time instead. +async fn sleds(log: &Logger, ddm: DdmClient, targets: &watch::Sender) { + let probe = reqwest::ClientBuilder::new() + .connect_timeout(PROBE_TIMEOUT) + .timeout(PROBE_TIMEOUT) + .build() + .expect("failed to build an HTTP client"); + loop { + let mut sleds = BTreeMap::new(); + match ddm + .derive_bootstrap_addrs_from_prefixes(&[ + BootstrapInterface::GlobalZone, + ]) + .await + { + Ok(addrs) => { + let addrs = + addrs.map(|ip| SocketAddrV6::new(ip, SUSH_API_PORT, 0, 0)); + discover(log, &probe, addrs, &mut sleds).await; + } + Err(err) => { + warn!(log, "unable to fetch bootstrap prefixes"; "error" => %err) + } + } + match ddm.derive_sled_addrs_from_prefixes().await { + Ok(addrs) => { + let addrs = addrs + .map(|a| SocketAddrV6::new(*a.ip(), SUSH_API_PORT, 0, 0)); + discover(log, &probe, addrs, &mut sleds).await; + } + Err(err) => { + warn!(log, "unable to fetch underlay prefixes"; "error" => %err) + } + } + targets.send_modify(|t| t.sleds.extend(sleds)); + sleep(POLL_INTERVAL).await; + } +} + +/// Probe candidate addresses concurrently and record the sleds that +/// answer. +async fn discover( + log: &Logger, + probe: &reqwest::Client, + addrs: impl Iterator, + sleds: &mut BTreeMap, +) { + let probes = + addrs.map(|addr| async move { (addr, target(probe, addr).await) }); + for (addr, result) in join_all(probes).await { + match result { + Ok(baseboard) => { + sleds.insert(baseboard, SocketAddr::V6(addr)); + } + Err(err) => { + debug!(log, "sled did not answer"; "addr" => %addr, "error" => %err); + } + } + } +} + +/// Ask a sush server which baseboard it serves. +async fn target( + probe: &reqwest::Client, + addr: SocketAddrV6, +) -> Result { + probe + .get(format!("http://{addr}/target")) + .send() + .await? + .error_for_status()? + .json() + .await +} + +/// Keep `Targets::cubbies` current from MGS's view of the SPs. Each +/// round's answers merge into the existing map, so a probe outage +/// never erases it. +async fn cubbies( + log: &Logger, + mgs: MgsClient, + targets: &watch::Sender, +) { + loop { + let polls = (0..=MAX_CUBBY).map(|cubby| { + let mgs = &mgs; + async move { (cubby, mgs.sp_get(&SpType::Sled, cubby.into()).await) } + }); + let mut cubbies = Cubbies::new(); + for (cubby, result) in join_all(polls).await { + match result { + Ok(state) => { + let state = state.into_inner(); + cubbies.insert( + cubby, + BaseboardId { + part_number: state.model, + serial_number: state.serial_number, + }, + ); + } + Err(err) => { + debug!(log, "no SP state for cubby"; "cubby" => cubby, "error" => %err); + } + } + } + targets.send_modify(|t| t.cubbies.extend(cubbies)); + sleep(POLL_INTERVAL).await; + } +} diff --git a/sush-proxy/src/main.rs b/sush-proxy/src/main.rs new file mode 100644 index 00000000000..4d63859f22c --- /dev/null +++ b/sush-proxy/src/main.rs @@ -0,0 +1,85 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Executable for the sush switch zone proxy. + +use std::net::{SocketAddr, SocketAddrV6}; + +use anyhow::Context; +use camino::Utf8PathBuf; +use clap::Parser; +use slog::{Drain, o}; +use sush_proxy::{Config, Tls, run}; + +#[derive(Debug, Parser)] +#[clap(name = "sush-proxy", about = "sush switch zone proxy")] +struct Args { + /// The address to listen on for sush clients + #[clap(long)] + address: SocketAddr, + + /// The address (expected to be on localhost) for MGS + #[clap(long)] + mgs_address: SocketAddrV6, + + /// How to authenticate to clients + #[clap(long, value_enum)] + tls: TlsArg, + + /// The TLS private key (PEM), for `--tls platform` + #[clap(long, required_if_eq("tls", "platform"))] + priv_key: Option, + + /// The TLS certificate chain (PEM), for `--tls platform` + #[clap(long, required_if_eq("tls", "platform"))] + cert_chain: Option, + + /// The baseboard (part:serial) of the sled hosting the proxy, + /// preferred for requests that name no target + #[clap(long)] + home: Option, +} + +/// How the proxy authenticates itself to clients. +#[derive(Clone, Copy, Debug, clap::ValueEnum)] +enum TlsArg { + /// The sled's platform identity, from a local key and chain + Platform, + /// None. For development images only + Insecure, +} + +fn main() -> anyhow::Result<()> { + oxide_tokio_rt::run(run_proxy()) +} + +async fn run_proxy() -> anyhow::Result<()> { + let Args { address, mgs_address, tls, priv_key, cert_chain, home } = + Args::parse(); + let home = home + .map(|home| { + home.parse() + .map_err(|err| anyhow::anyhow!("bad --home `{home}`: {err}")) + }) + .transpose()?; + let tls = match tls { + TlsArg::Platform => Tls::Platform { + priv_key: priv_key.unwrap(), + cert_chain: cert_chain.unwrap(), + }, + TlsArg::Insecure => Tls::Insecure, + }; + let decorator = slog_term::TermDecorator::new().build(); + let drain = slog_term::FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + let log = slog::Logger::root(drain, o!("component" => "sush-proxy")); + let (drain, registration) = slog_dtrace::with_drain(log); + let log = slog::Logger::root(drain.fuse(), o!()); + if let slog_dtrace::ProbeRegistration::Failed(err) = registration { + anyhow::bail!("failed to register DTrace probes: {err}"); + } + run(&log, Config { address, mgs_address, tls, home }) + .await + .context("running the proxy") +} diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index ddcf58c17f6..5d5af216db1 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -35,6 +35,7 @@ byte-wrapper = { version = "0.1.0", features = ["schemars08", "serde"] } bytes = { version = "1.11.1", features = ["serde"] } camino = { version = "1.2.5", default-features = false, features = ["serde1"] } chrono = { version = "0.4.45", features = ["serde"] } +cipher-9fbad63c4bcf4a8f = { package = "cipher", version = "0.4.4", default-features = false, features = ["zeroize"] } clap = { version = "4.6.1", features = ["cargo", "derive", "env", "wrap_help"] } clap_builder = { version = "4.6.0", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] } const-oid = { version = "0.9.6", default-features = false, features = ["db", "std"] } @@ -113,11 +114,10 @@ postgres-types = { version = "0.2.12", default-features = false, features = ["wi ppv-lite86 = { version = "0.2.21", default-features = false, features = ["simd", "std"] } predicates = { version = "3.1.4" } proc-macro2 = { version = "1.0.106" } +qorb = { version = "0.4.1" } quote = { version = "1.0.45" } -rand-274715c4dabd11b0 = { package = "rand", version = "0.9.2" } -rand-c38e5c1d305a1b54 = { package = "rand", version = "0.8.6" } -rand_chacha-274715c4dabd11b0 = { package = "rand_chacha", version = "0.9.0", default-features = false, features = ["std"] } -rand_chacha-468e82937335b1c9 = { package = "rand_chacha", version = "0.3.1", default-features = false, features = ["std"] } +rand = { version = "0.9.2" } +rand_chacha = { version = "0.9.0", default-features = false, features = ["std"] } regex = { version = "1.12.3" } regex-automata = { version = "0.4.14", default-features = false, features = ["dfa", "hybrid", "meta", "nfa", "perf", "std", "unicode"] } regex-syntax = { version = "0.8.10" } @@ -193,6 +193,7 @@ bytes = { version = "1.11.1", features = ["serde"] } camino = { version = "1.2.5", default-features = false, features = ["serde1"] } cc = { version = "1.2.56", default-features = false, features = ["parallel"] } chrono = { version = "0.4.45", features = ["serde"] } +cipher-9fbad63c4bcf4a8f = { package = "cipher", version = "0.4.4", default-features = false, features = ["zeroize"] } clap = { version = "4.6.1", features = ["cargo", "derive", "env", "wrap_help"] } clap_builder = { version = "4.6.0", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] } const-oid = { version = "0.9.6", default-features = false, features = ["db", "std"] } @@ -273,11 +274,10 @@ postgres-types = { version = "0.2.12", default-features = false, features = ["wi ppv-lite86 = { version = "0.2.21", default-features = false, features = ["simd", "std"] } predicates = { version = "3.1.4" } proc-macro2 = { version = "1.0.106" } +qorb = { version = "0.4.1" } quote = { version = "1.0.45" } -rand-274715c4dabd11b0 = { package = "rand", version = "0.9.2" } -rand-c38e5c1d305a1b54 = { package = "rand", version = "0.8.6" } -rand_chacha-274715c4dabd11b0 = { package = "rand_chacha", version = "0.9.0", default-features = false, features = ["std"] } -rand_chacha-468e82937335b1c9 = { package = "rand_chacha", version = "0.3.1", default-features = false, features = ["std"] } +rand = { version = "0.9.2" } +rand_chacha = { version = "0.9.0", default-features = false, features = ["std"] } regex = { version = "1.12.3" } regex-automata = { version = "0.4.14", default-features = false, features = ["dfa", "hybrid", "meta", "nfa", "perf", "std", "unicode"] } regex-syntax = { version = "0.8.10" } @@ -425,7 +425,7 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws- [target.x86_64-unknown-illumos.dependencies] chacha20 = { version = "0.10.0", default-features = false, features = ["legacy", "rng", "zeroize"] } -cipher = { version = "0.5.2", default-features = false, features = ["block-padding", "rand_core", "stream-wrapper"] } +cipher-d8f496e17d97b5cb = { package = "cipher", version = "0.5.2", default-features = false, features = ["block-padding", "rand_core", "stream-wrapper"] } cookie = { version = "0.18.1", default-features = false, features = ["percent-encode"] } dof-468e82937335b1c9 = { package = "dof", version = "0.3.0", default-features = false, features = ["des"] } dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4.0", default-features = false, features = ["des"] } @@ -445,7 +445,7 @@ winnow = { version = "1.0.3" } [target.x86_64-unknown-illumos.build-dependencies] chacha20 = { version = "0.10.0", default-features = false, features = ["legacy", "rng", "zeroize"] } -cipher = { version = "0.5.2", default-features = false, features = ["block-padding", "rand_core", "stream-wrapper"] } +cipher-d8f496e17d97b5cb = { package = "cipher", version = "0.5.2", default-features = false, features = ["block-padding", "rand_core", "stream-wrapper"] } cookie = { version = "0.18.1", default-features = false, features = ["percent-encode"] } dof-468e82937335b1c9 = { package = "dof", version = "0.3.0", default-features = false, features = ["des"] } dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4.0", default-features = false, features = ["des"] }