From 8b2a020c14468d50f9f03fa3a8c62556141e4cd5 Mon Sep 17 00:00:00 2001 From: "Christian S.J. Peron" Date: Thu, 8 Oct 2026 12:29:32 -0500 Subject: [PATCH] Add a bunch of volume handling fixes --- README.md | 80 +++++++++++++++++++++++ src/shell/stage_launch.sh | 130 +++++++++++++++++++++++--------------- 2 files changed, 158 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 6efc74a..31174f7 100644 --- a/README.md +++ b/README.md @@ -430,3 +430,83 @@ a5ec8053ea freebsd-13_4 /dev/pts/2 2374 assembled 276s % sudo cblock console --name a5ec8053ea root@a5ec8053ea:/ # ``` + +### Volumes + +A volume mounts a host file system into a cellblock when it launches: + +``` +--volume TYPE:HOST:CONTAINER:ro|rw +``` + +* `TYPE` is a file system type for `mount -t`: `nullfs` for a host + directory, or `ufs` for a device such as a zvol. +* `HOST` is the host directory or device. +* `CONTAINER` is where it appears inside the cellblock. It is created if it + does not exist. It may not contain `..`, or pass through a symlink in the + image that points outside the cellblock. +* `ro` mounts it read-only, `rw` read-write. + +Paths may not contain `:` or `,`. `--tmpfs`, `--procfs` and `--fdescfs` +mount an in-memory `/tmp`, `/proc` and `/dev/fd`. + +``` +% sudo cblock launch --name nginx --network natnet \ + --volume nullfs:/storage/www:/usr/local/www:ro +``` + +In the `cblock_warden` manifest: + +```yaml +cellblocks: + - image: nginx + network: natnet + volumes: + - type: nullfs + origin: /storage/www + mountpoint: /usr/local/www + perms: ro +``` + +Anyone who can launch a cellblock can mount any host path into it. This is +one reason access to cblockd is root-equivalent (see Permissions). + +### Secrets + +Keep secrets such as keys and passwords out of images and build contexts: +anything in an image is readable by everyone who can launch it. Instead, +keep them in a host directory and give each cellblock its own, mounted +read-only: + +``` +% sudo install -d -m 0700 /usr/local/cblocks/secrets +% sudo install -d -m 0750 -g 80 /usr/local/cblocks/secrets/nginx +% sudo install -m 0400 -o 80 -g 80 tls.key /usr/local/cblocks/secrets/nginx/ +% sudo cblock launch --name nginx --network natnet \ + --volume nullfs:/usr/local/cblocks/secrets/nginx:/run/secrets:ro +``` + +* The top directory keeps users on the host out. Inside the cellblock, + the per-cellblock directory appears as `/run/secrets` with its own + owner and mode, so give its group to the service. +* Cellblocks share user and group IDs with the host, so use the numeric + IDs the service runs as inside the image (80 is `www` on FreeBSD). +* Point the service at the file, for example `ssl_certificate_key + /run/secrets/tls.key;` for nginx. Many programs also accept a + `*_FILE` setting naming a file to read a password from. Avoid + environment variables for secrets: `ps -e` shows them, and every child + process inherits them. +* nullfs shows the host directory as it is now, so a secret replaced on + the host is visible in the running cellblock straight away. The + service may need a restart or reload to read it again. +* The files are on the host's disk. To keep them in memory only, mount a + tmpfs on the secrets directory at boot. + +Secrets from HashiCorp Vault or OpenBao work the same way: run Vault Agent +(or OpenBao Agent) on the host, have its templates write each cellblock's +secrets into that cellblock's directory, and mount it as above. The agent +handles authentication, renewal and rotation, and nothing in the +cellblock needs a Vault token. + +Secrets are not available during `cblock build`. Do not put them in the +build directory: the whole build context is copied to the cblockd host. diff --git a/src/shell/stage_launch.sh b/src/shell/stage_launch.sh index 1e71318..c7e67f9 100755 --- a/src/shell/stage_launch.sh +++ b/src/shell/stage_launch.sh @@ -169,62 +169,93 @@ setup_port_redirects() done } -emit_mount_specification() -{ - _all_fields=$1 - _root="${data_root}/instances/${instance_id}/root" - - echo "$1" | grep '[;&<>|]' - if [ $? -eq 0 ]; then +# Print the host path of the mount point for container path $1, creating +# it if it doesn't exist. Fail if it would land outside the instance root, +# for example through "..", or a symlink in the image that points outside. +mount_point() +{ + _root=$(realpath "${data_root}/instances/${instance_id}/root") + case "/$1/" in + */../*) + echo "$1: volume path may not contain .." >&2 return 1 - fi - for spec in $(echo "${_all_fields}" | sed "s/,/ /g"); do - case $spec in - tmpfs) - echo mount -t tmpfs tmpfs ${_root}/tmp\; - ;; + ;; + esac + # Resolve the part that exists, and keep the rest to create. + _existing="${_root}/$1" + _rest="" + while [ ! -e "$_existing" ]; do + _rest="/$(basename "$_existing")${_rest}" + _existing=$(dirname "$_existing") + done + _real=$(realpath "$_existing") + case "${_real}/" in + "${_root}"/*) + ;; + *) + echo "$1: volume path leaves the instance root" >&2 + return 1 + ;; + esac + mkdir -p "${_real}${_rest}" || return 1 + echo "${_real}${_rest}" +} + +# Mount the comma separated volume list $1: devfs (mounted elsewhere), +# tmpfs, procfs, fdescfs, or fs_type:host_path:container_path:ro|rw. +# The specs come from the client, so they are only ever passed to mount +# as arguments, never evaluated by the shell. +mount_volumes() +{ + _ifs="$IFS" + set -f + IFS=, + set -- $1 + IFS="$_ifs" + set +f + for spec; do + case "$spec" in devfs) + continue + ;; + tmpfs) + _mnt=$(mount_point /tmp) && mount -t tmpfs tmpfs "$_mnt" ;; procfs) - echo mount -t procfs procfs ${_root}/proc\; + _mnt=$(mount_point /proc) && mount -t procfs procfs "$_mnt" ;; fdescfs) - echo mount -t fdescfs fdescfs ${_root}/dev/fd\; - ;; - *:*:*:*) - for field in $(jot 4); do - case $field in - 1) - fs_type=`echo "$spec" | cut -f $field -d:` - ;; - 2) - fs_host=`echo "$spec" | cut -f $field -d:` - ;; - 3) - container_mount=`echo "$spec" | cut -f $field -d:` - ;; - 4) - perms=`echo "$spec" | cut -f $field -d:` - ;; - esac - done - if [ -z $fs_type ] || [ -z $fs_host ] || [ -z $container_mount ] || [ -z $perm ]; then - echo must follow fs:local:container:perms >&2 - echo exit 1 - return - fi - echo -n "mount -t $fs_type " - if [ "$perms" = "RO" ] || [ "$perms" = "ro" ]; then - echo -n "-o ro " - fi - echo $fs_host ${_root}/$container_mount\; + _mnt=$(mount_point /dev/fd) && mount -t fdescfs fdescfs "$_mnt" ;; *) - echo must follow fs:local:container:perms >&2 - echo exit 1 - return + IFS=: read -r fs_type fs_host container perms extra <&2 + return 1 + fi + case "$perms" in + ro|RO) + _opts="-o ro" + ;; + rw|RW) + _opts="" + ;; + *) + echo "$spec: perms must be ro or rw" >&2 + return 1 + ;; + esac + _mnt=$(mount_point "$container") && \ + mount -t "$fs_type" $_opts "$fs_host" "$_mnt" ;; esac + if [ $? -ne 0 ]; then + echo "$spec: mount failed" >&2 + return 1 + fi done } @@ -419,12 +450,7 @@ do_launch() esac mount -t devfs devfs "${instance_root}/dev" config_devfs - # if mount_spec is *just* devfs skip over mount operations since - # devfs is handled elsewhere. - if [ "$mount_spec" != "devfs" ]; then - mnt_cmd=$(emit_mount_specification "$mount_spec") - eval $mnt_cmd - fi + mount_volumes "$mount_spec" || exit 1 net_type=$(network_type) set $(emit_entrypoint) if [ "$net_type" = "bridge" ]; then