Add the sush proxy to the switch zone

.cargo/xtask.toml

@@ -28,6 +28,8 @@
 [libraries."libxmlsec1.so.1"]
 
 # libipcc should only be depended on by binaries that communicate with ipcc.
+# sush-proxy does not, but the workspace builds it alongside sled-agent, whose
+# ipcc feature unifies onto their shared sprockets-tls.
 [libraries."libipcc.so.1"]
 binary_allow_list = [
   "installinator",
@@ -36,6 +38,7 @@ binary_allow_list = [
   "omicron-dev",
   "sled-agent",
   "sled-agent-sim",
+  "sush-proxy",
 ]
 
 # libnvme is a global zone only library and therefore we must be sure that only

Cargo.toml

@@ -171,6 +171,7 @@ members = [
     "sled-storage/zfs-test-harness",
     "sp-sim",
     "support-bundle-collection",
+    "sush-proxy",
     "test-utils",
     "trust-quorum",
     "trust-quorum/gfss",
@@ -368,6 +369,7 @@ default-members = [
     "sled-storage/zfs-test-harness",
     "sp-sim",
     "support-bundle-collection",
+    "sush-proxy",
     "trust-quorum",
     "trust-quorum/gfss",
     "trust-quorum/protocol",

clients/ddm-admin-client/src/lib.rs

@@ -16,6 +16,7 @@ use omicron_common::address::BOOTSTRAP_SLED_SUBNET_PREFIX_LENGTH;
 use omicron_common::address::DDMD_PORT;
 use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::SLED_PREFIX_LENGTH;
+use omicron_common::address::get_sled_address;
 use oxnet::Ipv6Net;
 use sled_hardware_types::underlay::BootstrapInterface;
 use slog::Logger;
@@ -105,6 +106,17 @@ impl Client {
         self.inner.enable_stats(request).await.map(|resp| resp.into_inner())
     }
 
+    /// Returns the sled address within each subnet
+    /// [`Self::derive_underlay_subnets_from_prefixes`] returns. Since
+    /// those are not all sled subnets, not every address answers as a
+    /// sled.
+    pub async fn derive_sled_addrs_from_prefixes(
+        &self,
+    ) -> Result<impl Iterator<Item = SocketAddrV6> + use<>, DdmError> {
+        let subnets = self.derive_underlay_subnets_from_prefixes().await?;
+        Ok(subnets.map(get_sled_address))
+    }
+
     /// Returns every non-bootstrap /64 prefix ddmd has learned. These
     /// are not all sled subnets: an internal DNS zone is reachable at
     /// its own reserved /64, advertised by the sled hosting it, and

common/src/address.rs

@@ -234,6 +234,7 @@ pub const REPO_DEPOT_PORT: u16 = 12348;
 pub const TRUST_QUORUM_PORT: u16 = 12349;
 pub const SUSH_API_PORT: u16 = 12350;
 pub const SUSH_GOSSIP_PORT: u16 = 12351;
+pub const SUSH_PROXY_PORT: u16 = 12352;
 
 pub const BOOTSTRAP_AGENT_LOCKSTEP_PORT: u16 = 8080;
 

dev-tools/ls-apis/api-manifest.toml

@@ -58,6 +58,7 @@ packages = [
     "lldpd",
     "mgd",
     "omicron-gateway",
+    "sush-proxy",
     "tfportd",
     "wicketd",
 ]
@@ -774,6 +775,30 @@ permalinks = [
     "https://github.com/oxidecomputer/omicron/blob/6cef874/sled-agent/src/instance.rs#L2283",
 ]
 
+[[intra_deployment_unit_only_edges]]
+server = "sush-proxy"
+client = "ddm-admin-client"
+note = """
+sush-proxy discovers sleds through its own switch zone's ddmd, always
+via Client::localhost (sush-proxy/src/lib.rs).
+"""
+permalinks = [
+    "https://github.com/oxidecomputer/omicron/blob/main/sush-proxy/src/lib.rs",
+]
+
+[[intra_deployment_unit_only_edges]]
+server = "sush-proxy"
+client = "gateway-client"
+note = """
+sush-proxy reads the cubby map from its own switch zone's MGS.
+sled-agent sets the mgs-address SMF property to [::1] (services.rs),
+and the manifest passes it via --mgs-address.
+"""
+permalinks = [
+    "https://github.com/oxidecomputer/omicron/blob/main/sled-agent/src/services.rs",
+    "https://github.com/oxidecomputer/omicron/blob/main/smf/sush-proxy/manifest.xml",
+]
+
 [[intra_deployment_unit_only_edges]]
 server = "propolis-server"
 client = "sled-agent-client"

dev-tools/ls-apis/tests/api_dependencies.out

@@ -27,6 +27,7 @@ Maghemite DDM Admin (client: ddm-admin-client)
     consumed by: mgd (maghemite/mgd) via 1 path
     consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path
     consumed by: sled-agent-rack-setup (omicron/sled-agent/rack-setup) via 1 path [embedded in omicron-sled-agent; rack-init only]
+    consumed by: sush-proxy (omicron/sush-proxy) via 1 path
     consumed by: wicketd (omicron/wicketd) via 1 path
 
 DNS Server (client: dns-service-client)
@@ -50,6 +51,7 @@ Management Gateway Service (client: gateway-client)
     consumed by: mgd (maghemite/mgd) via 1 path
     consumed by: omicron-nexus (omicron/nexus) via 6 paths
     consumed by: omicron-sled-agent (omicron/sled-agent) via 2 paths
+    consumed by: sush-proxy (omicron/sush-proxy) via 1 path
     consumed by: wicketd (omicron/wicketd) via 3 paths
 
 Wicketd Installinator (client: installinator-client)

package-manifest.toml

@@ -525,6 +525,16 @@ source.paths = [{ from = "smf/wicketd", to = "/var/svc/manifest/site/wicketd" }]
 output.type = "zone"
 output.intermediate_only = true
 
+[package.sush-proxy]
+service_name = "sush-proxy"
+only_for_targets.image = "standard"
+source.type = "local"
+source.rust.binary_names = ["sush-proxy"]
+source.rust.release = true
+source.paths = [{ from = "smf/sush-proxy", to = "/var/svc/manifest/site/sush-proxy" }]
+output.type = "zone"
+output.intermediate_only = true
+
 [package.wicket]
 service_name = "wicket"
 only_for_targets.image = "standard"
@@ -856,6 +866,7 @@ source.packages = [
   "pumpkind.tar.gz",
   "wicketd.tar.gz",
   "wicket.tar.gz",
+  "sush-proxy.tar.gz",
   "mg-ddm.tar.gz",
   "mgd.tar.gz",
   "switch_zone_setup.tar.gz",
@@ -883,6 +894,7 @@ source.packages = [
   "lldp.tar.gz",
   "wicketd.tar.gz",
   "wicket.tar.gz",
+  "sush-proxy.tar.gz",
   "mg-ddm.tar.gz",
   "mgd.tar.gz",
   "switch_zone_setup.tar.gz",
@@ -910,6 +922,7 @@ source.packages = [
   "lldp.tar.gz",
   "wicketd.tar.gz",
   "wicket.tar.gz",
+  "sush-proxy.tar.gz",
   "mg-ddm.tar.gz",
   "mgd.tar.gz",
   "switch_zone_setup.tar.gz",

sled-agent/Cargo.toml

@@ -129,7 +129,7 @@ tufaceous-brand-metadata.workspace = true
 usdt.workspace = true
 uuid.workspace = true
 walkdir.workspace = true
-x509-cert.workspace = true
+x509-cert = { workspace = true, features = ["std"] }
 zeroize.workspace = true
 zip.workspace = true
 zone.workspace = true

sled-agent/src/services.rs

@@ -27,6 +27,9 @@ use crate::config::SidecarRevision;
 use crate::ddm_reconciler::DdmReconciler;
 use crate::metrics::MetricsRequestQueue;
 use crate::profile::*;
+use crate::sush::{
+    SUSH_PROXY_CERT_CHAIN_PATH, SUSH_PROXY_KEY_PATH, generate_proxy_identity,
+};
 use camino::{Utf8Path, Utf8PathBuf};
 use clickhouse_admin_types::CLICKHOUSE_KEEPER_CONFIG_DIR;
 use clickhouse_admin_types::CLICKHOUSE_KEEPER_CONFIG_FILE;
@@ -64,6 +67,7 @@ use omicron_common::address::MGS_PORT;
 use omicron_common::address::NTP_ADMIN_PORT;
 use omicron_common::address::RACK_PREFIX_LENGTH;
 use omicron_common::address::SLED_PREFIX_LENGTH;
+use omicron_common::address::SUSH_PROXY_PORT;
 use omicron_common::address::TFPORTD_PORT;
 use omicron_common::address::WICKETD_COMMISSION_PORT;
 use omicron_common::address::WICKETD_NEXUS_PROXY_PORT;
@@ -462,6 +466,25 @@ enum SwitchService {
     MgDdm { mode: String },
     Mgd,
     SpSim,
+    SushProxy { tls: SushProxyTls, baseboard: Baseboard },
+}
+
+/// How the sush proxy authenticates itself to clients.
+#[derive(Clone, Copy, Debug, PartialEq)]
+enum SushProxyTls {
+    /// An ephemeral key with a certificate signed by the RoT at zone startup.
+    Platform,
+    /// No authentication. For development images only, which have no RoT.
+    Insecure,
+}
+
+impl SushProxyTls {
+    fn as_str(&self) -> &'static str {
+        match self {
+            SushProxyTls::Platform => "platform",
+            SushProxyTls::Insecure => "insecure",
+        }
+    }
 }
 
 impl illumos_utils::smf_helper::Service for SwitchService {
@@ -477,6 +500,7 @@ impl illumos_utils::smf_helper::Service for SwitchService {
             SwitchService::MgDdm { .. } => "mg-ddm",
             SwitchService::Mgd => "mgd",
             SwitchService::SpSim => "sp-sim",
+            SwitchService::SushProxy { .. } => "sush-proxy",
         }
     }
     fn smf_name(&self) -> String {
@@ -2407,6 +2431,7 @@ impl ServiceManager {
         let mut mgd_service = ServiceBuilder::new("oxide/mgd");
         let mut mg_ddm_service = ServiceBuilder::new("oxide/mg-ddm");
         let mut uplink_service = ServiceBuilder::new("oxide/uplink");
+        let mut sush_proxy_service = ServiceBuilder::new("oxide/sush-proxy");
 
         let mut switch_zone_setup_config = PropertyGroupBuilder::new("config")
             .add_property(
@@ -2482,6 +2507,60 @@ impl ServiceManager {
                 SwitchService::SpSim => {
                     info!(self.inner.log, "Setting up Simulated SP service");
                 }
+                SwitchService::SushProxy { tls, baseboard } => {
+                    info!(self.inner.log, "Setting up sush-proxy service");
+                    if let SushProxyTls::Platform = tls {
+                        if let Err(err) = generate_proxy_identity(
+                            &self.inner.log,
+                            &installed_zone.root(),
+                        )
+                        .await
+                        {
+                            error!(
+                                self.inner.log,
+                                "failed to generate the sush proxy TLS identity";
+                                "error" => #%err,
+                            );
+                        }
+                    }
+                    let config = PropertyGroupBuilder::new("config")
+                        // Bind `::` so the proxy serves all interfaces,
+                        // particularly the tech ports.
+                        .add_property(
+                            "address",
+                            "astring",
+                            &format!("[::]:{SUSH_PROXY_PORT}"),
+                        )
+                        .add_property(
+                            "mgs-address",
+                            "astring",
+                            &format!("[::1]:{MGS_PORT}"),
+                        )
+                        .add_property("tls", "astring", tls.as_str())
+                        .add_property(
+                            "home",
+                            "astring",
+                            &format!(
+                                "{}:{}",
+                                baseboard.model(),
+                                baseboard.identifier()
+                            ),
+                        )
+                        .add_property(
+                            "priv-key",
+                            "astring",
+                            SUSH_PROXY_KEY_PATH,
+                        )
+                        .add_property(
+                            "cert-chain",
+                            "astring",
+                            SUSH_PROXY_CERT_CHAIN_PATH,
+                        );
+                    sush_proxy_service = sush_proxy_service.add_instance(
+                        ServiceInstanceBuilder::new("default")
+                            .add_property_group(config),
+                    );
+                }
                 SwitchService::Wicketd { baseboard } => {
                     info!(self.inner.log, "Setting up wicketd service");
                     // If we're launching the switch zone, we'll have a
@@ -3000,7 +3079,8 @@ impl ServiceManager {
             .add_service(pumpkind_service)
             .add_service(mgd_service)
             .add_service(mg_ddm_service)
-            .add_service(uplink_service);
+            .add_service(uplink_service)
+            .add_service(sush_proxy_service);
 
         // If we have the rack subnet, also set up /etc/resolv.conf.
         if let Some(info) = info {
@@ -3131,6 +3211,10 @@ impl ServiceManager {
                     SwitchService::Wicketd { baseboard: baseboard.clone() },
                     SwitchService::Mgd,
                     SwitchService::MgDdm { mode: "transit".to_string() },
+                    SwitchService::SushProxy {
+                        tls: SushProxyTls::Platform,
+                        baseboard: baseboard.clone(),
+                    },
                 ]
             }
 
@@ -3151,6 +3235,10 @@ impl ServiceManager {
                         asic,
                     },
                     SwitchService::SpSim,
+                    SwitchService::SushProxy {
+                        tls: SushProxyTls::Insecure,
+                        baseboard: baseboard.clone(),
+                    },
                 ]
             }
 
@@ -3182,6 +3270,10 @@ impl ServiceManager {
                         asic,
                     },
                     SwitchService::SpSim,
+                    SwitchService::SushProxy {
+                        tls: SushProxyTls::Insecure,
+                        baseboard: baseboard.clone(),
+                    },
                 ]
             }
         };
@@ -3553,6 +3645,9 @@ impl ServiceManager {
                         SwitchService::SpSim => {
                             // nothing to configure
                         }
+                        SwitchService::SushProxy { .. } => {
+                            // nothing to configure
+                        }
                         SwitchService::Mgd => {
                             info!(self.inner.log, "configuring mgd service");
                             smfh.delpropvalue_default_instance(