diff --git a/Cblockfile b/Cblockfile index ba133e7..4f2dae6 100644 --- a/Cblockfile +++ b/Cblockfile @@ -2,7 +2,7 @@ FROM base ENV ASSUME_ALWAYS_YES = "YES" ENV LC_ALL = "C" -RUN "pkg install go125 git" +RUN "pkg install go git" ENV PATH = "/usr/local/go125/bin:$PATH" RUN "go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest" RUN "mkdir /build" diff --git a/src/shell/stage_launch.sh b/src/shell/stage_launch.sh index c7e67f9..1f91da2 100755 --- a/src/shell/stage_launch.sh +++ b/src/shell/stage_launch.sh @@ -24,6 +24,13 @@ # SUCH DAMAGE. # . "$(dirname "$0")/common.sh" +# do_launch re-runs this script with --setup-network, holding the network +# lock, to allocate the instance's interface or address. See setup_network. +mode=launch +if [ "$1" = "--setup-network" ]; then + mode=setup_network + shift +fi data_root="$1" image_name="$2" instance_id="$3" @@ -394,13 +401,14 @@ network_to_ip6() for ip in $(subcalc inet6 $net_addr print | grep -v "^;"); do if [ $(is_assigned $ip 6) = "no" ] && \ [ $(is_broadcast $net_addr $ip 6) = "no" ]; then - ifconfig "$network" inet6 "${ip}/128" alias + ifconfig "$network" inet6 "${ip}/128" alias || exit 1 echo "${ip}" echo "nat,${instance_id},${ip},$network,6" >> \ $data_root/networks/cur - return + return 0 fi done + echo "No free addresses left on NAT interface $network" >&2 exit 1 } @@ -410,20 +418,62 @@ network_to_ip() for ip in $(subcalc inet $net_addr print | grep -v "^;"); do if [ $(is_assigned $ip 4) = "no" ] && \ [ $(is_broadcast $net_addr $ip 4) = "no" ]; then - ifconfig "$network" inet "${ip}/32" alias + ifconfig "$network" inet "${ip}/32" alias || exit 1 echo "nat,${instance_id},${ip},$network,4" >> \ $data_root/networks/cur echo "${ip}" rdr_rules=$(setup_port_redirects "$ports" "$ip" "$(nat_outif)") - if [ "$rdr_rules" ]; then - echo "$rdr_rules" | pfctl -a cblock-rdr/${instance_id} -f - + if [ "$rdr_rules" ] && ! echo "$rdr_rules" | \ + pfctl -a cblock-rdr/${instance_id} -f -; then + echo "Failed to load port redirects for ${ip}" >&2 + exit 1 fi - return + return 0 fi done + echo "No free addresses left on NAT interface $network" >&2 exit 1 } +# Create the instance's epair (bridge) or pick its address and load its port +# redirects (NAT), recording them in networks/cur, and print the interface or +# address. This runs under the network lock, which stage_launch_cleanup.sh +# also takes, because concurrent launches would otherwise: +# - pick the same free NAT address, +# - fail to load redirects with EBUSY: pf stages rule addresses in a single, +# global buffer, so concurrent rdr loads invalidate each other's ticket, +# - lose their networks/cur entry while a cleanup rewrites the file. +setup_network() +{ + case $(network_type) in + bridge) + get_jail_interface + ;; + nat) + if [ "$(nat_ip_version)" = "6" ]; then + network_to_ip6 + else + network_to_ip + fi + ;; + esac +} + +# Run setup_network in a separate invocation of this script, holding the +# network lock. +setup_network_locked() +{ + _flags="" + case $- in + *x*) + _flags=-x + ;; + esac + lockf -k "${data_root}/networks/lock" /bin/sh $_flags "$0" \ + --setup-network "$data_root" "$image_name" "$instance_id" \ + "$mount_spec" "$network" "$tag" "$ports" +} + do_launch() { img_tag="${image_name}:${tag}" @@ -454,7 +504,7 @@ do_launch() net_type=$(network_type) set $(emit_entrypoint) if [ "$net_type" = "bridge" ]; then - netif=$(get_jail_interface) || exit 1 + netif=$(setup_network_locked) || exit 1 jail -c \ "host.hostname=${instance_hostname}" \ "vnet" \ @@ -467,16 +517,13 @@ do_launch() else if [ "$net_type" = "host" ]; then netspec="ip4.addr=$(get_default_ip)" - elif [ $(nat_ip_version) = "6" ]; then - ip=$(network_to_ip6) - netspec="ip6.addr=$ip" else - ip=$(network_to_ip) - netspec="ip4.addr=$ip" - fi - if [ "$net_type" = "nat" ] && [ ! "$ip" ]; then - echo "No free addresses left on NAT interface $network" - exit 1 + ip=$(setup_network_locked) || exit 1 + if [ "$(nat_ip_version)" = "6" ]; then + netspec="ip6.addr=$ip" + else + netspec="ip4.addr=$ip" + fi fi jailcmd="" if [ -f "${image_dir}/AUDITCFG" ]; then @@ -496,4 +543,8 @@ launch_block() do_launch } +if [ "$mode" = "setup_network" ]; then + setup_network + exit +fi launch_block diff --git a/src/shell/stage_launch_cleanup.sh b/src/shell/stage_launch_cleanup.sh index 8c03b50..85047fd 100644 --- a/src/shell/stage_launch_cleanup.sh +++ b/src/shell/stage_launch_cleanup.sh @@ -25,6 +25,14 @@ # SUCH DAMAGE. # . "$(dirname "$0")/common.sh" +# The script re-runs itself with --network-cleanup, holding the network lock, +# to release the instance's network resources. See setup_network in +# stage_launch.sh. +mode=cleanup +if [ "$1" = "--network-cleanup" ]; then + mode=network_cleanup + shift +fi data_root="$1" instance="$2" type=$3 @@ -34,11 +42,12 @@ network_cleanup() if [ ! -f "$data_root"/networks/cur ]; then return 0 fi - newdb=$(mktemp) + # Same directory, so the mv below is an atomic rename. + newdb=$(mktemp "$data_root"/networks/cur.XXXXXX) || return 1 while read ln; do ins=$(echo "$ln" | awk -F, '{ print $2 }') if [ "$ins" != "$instance" ]; then - echo $ln >> $newdb + echo "$ln" >> "$newdb" continue fi type=$(echo "$ln" | awk -F, '{ print $1 }') @@ -62,8 +71,7 @@ network_cleanup() ;; esac done < "$data_root"/networks/cur - rm "$data_root"/networks/cur - mv $newdb "$data_root"/networks/cur + mv "$newdb" "$data_root"/networks/cur } umount_reverse_order() @@ -125,5 +133,16 @@ cleanup() esac } +if [ "$mode" = "network_cleanup" ]; then + network_cleanup + exit +fi cleanup -network_cleanup +_flags="" +case $- in +*x*) + _flags=-x + ;; +esac +lockf -k "${data_root}/networks/lock" /bin/sh $_flags "$0" \ + --network-cleanup "$data_root" "$instance"