From 38a377f0eb3800a607d7db3911a63e19b185ab80 Mon Sep 17 00:00:00 2001
From: "Christian S.J. Peron" <csjayp@gmail.com>
Date: Wed, 11 Nov 2026 16:42:18 -0500
Subject: [PATCH] control concurrency

do not run multiple pfctl isntances at once which could happen
when we are starting up many cblocks at once. The constraint lives
in the kernel and we end up getting EBUSY back when trying to
install the nat rules
---
 Cblockfile                        |  2 +-
 src/shell/stage_launch.sh         | 83 +++++++++++++++++++++++++------
 src/shell/stage_launch_cleanup.sh | 29 +++++++++--
 3 files changed, 92 insertions(+), 22 deletions(-)

diff --git a/Cblockfile b/Cblockfile
index ba133e7..4f2dae6 100644
--- a/Cblockfile
+++ b/Cblockfile
@@ -2,7 +2,7 @@ FROM base
 
 ENV ASSUME_ALWAYS_YES = "YES"
 ENV LC_ALL = "C"
-RUN "pkg install go125 git"
+RUN "pkg install go git"
 ENV PATH = "/usr/local/go125/bin:$PATH"
 RUN "go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest"
 RUN "mkdir /build"
diff --git a/src/shell/stage_launch.sh b/src/shell/stage_launch.sh
index c7e67f9..1f91da2 100755
--- a/src/shell/stage_launch.sh
+++ b/src/shell/stage_launch.sh
@@ -24,6 +24,13 @@
 # SUCH DAMAGE.
 #
 . "$(dirname "$0")/common.sh"
+# do_launch re-runs this script with --setup-network, holding the network
+# lock, to allocate the instance's interface or address. See setup_network.
+mode=launch
+if [ "$1" = "--setup-network" ]; then
+    mode=setup_network
+    shift
+fi
 data_root="$1"
 image_name="$2"
 instance_id="$3"
@@ -394,13 +401,14 @@ network_to_ip6()
     for ip in $(subcalc inet6 $net_addr print | grep -v "^;"); do
         if [ $(is_assigned $ip 6) = "no" ] && \
            [ $(is_broadcast $net_addr $ip 6) = "no" ]; then
-            ifconfig "$network" inet6 "${ip}/128" alias
+            ifconfig "$network" inet6 "${ip}/128" alias || exit 1
             echo "${ip}"
             echo "nat,${instance_id},${ip},$network,6" >> \
               $data_root/networks/cur
-            return
+            return 0
         fi
     done
+    echo "No free addresses left on NAT interface $network" >&2
     exit 1
 }
 
@@ -410,20 +418,62 @@ network_to_ip()
     for ip in $(subcalc inet $net_addr print | grep -v "^;"); do
         if [ $(is_assigned $ip 4) = "no" ] && \
            [ $(is_broadcast $net_addr $ip 4) = "no" ]; then
-            ifconfig "$network" inet "${ip}/32" alias
+            ifconfig "$network" inet "${ip}/32" alias || exit 1
             echo "nat,${instance_id},${ip},$network,4" >> \
               $data_root/networks/cur
             echo "${ip}"
             rdr_rules=$(setup_port_redirects "$ports" "$ip" "$(nat_outif)")
-            if [ "$rdr_rules" ]; then
-                echo "$rdr_rules" | pfctl -a cblock-rdr/${instance_id} -f -
+            if [ "$rdr_rules" ] && ! echo "$rdr_rules" | \
+              pfctl -a cblock-rdr/${instance_id} -f -; then
+                echo "Failed to load port redirects for ${ip}" >&2
+                exit 1
             fi
-            return
+            return 0
         fi
     done
+    echo "No free addresses left on NAT interface $network" >&2
     exit 1
 }
 
+# Create the instance's epair (bridge) or pick its address and load its port
+# redirects (NAT), recording them in networks/cur, and print the interface or
+# address. This runs under the network lock, which stage_launch_cleanup.sh
+# also takes, because concurrent launches would otherwise:
+#  - pick the same free NAT address,
+#  - fail to load redirects with EBUSY: pf stages rule addresses in a single,
+#    global buffer, so concurrent rdr loads invalidate each other's ticket,
+#  - lose their networks/cur entry while a cleanup rewrites the file.
+setup_network()
+{
+    case $(network_type) in
+    bridge)
+        get_jail_interface
+        ;;
+    nat)
+        if [ "$(nat_ip_version)" = "6" ]; then
+            network_to_ip6
+        else
+            network_to_ip
+        fi
+        ;;
+    esac
+}
+
+# Run setup_network in a separate invocation of this script, holding the
+# network lock.
+setup_network_locked()
+{
+    _flags=""
+    case $- in
+    *x*)
+        _flags=-x
+        ;;
+    esac
+    lockf -k "${data_root}/networks/lock" /bin/sh $_flags "$0" \
+      --setup-network "$data_root" "$image_name" "$instance_id" \
+      "$mount_spec" "$network" "$tag" "$ports"
+}
+
 do_launch()
 {
     img_tag="${image_name}:${tag}"
@@ -454,7 +504,7 @@ do_launch()
     net_type=$(network_type)
     set $(emit_entrypoint)
     if [ "$net_type" = "bridge" ]; then
-       netif=$(get_jail_interface) || exit 1
+       netif=$(setup_network_locked) || exit 1
        jail -c \
           "host.hostname=${instance_hostname}" \
           "vnet" \
@@ -467,16 +517,13 @@ do_launch()
     else
         if [ "$net_type" = "host" ]; then
             netspec="ip4.addr=$(get_default_ip)"
-        elif [ $(nat_ip_version) = "6" ]; then
-            ip=$(network_to_ip6)
-            netspec="ip6.addr=$ip"
         else
-            ip=$(network_to_ip)
-            netspec="ip4.addr=$ip"
-        fi
-        if [ "$net_type" = "nat" ] && [ ! "$ip" ]; then
-            echo "No free addresses left on NAT interface $network"
-            exit 1
+            ip=$(setup_network_locked) || exit 1
+            if [ "$(nat_ip_version)" = "6" ]; then
+                netspec="ip6.addr=$ip"
+            else
+                netspec="ip4.addr=$ip"
+            fi
         fi
         jailcmd=""
         if [ -f "${image_dir}/AUDITCFG" ]; then
@@ -496,4 +543,8 @@ launch_block()
     do_launch
 }
 
+if [ "$mode" = "setup_network" ]; then
+    setup_network
+    exit
+fi
 launch_block
diff --git a/src/shell/stage_launch_cleanup.sh b/src/shell/stage_launch_cleanup.sh
index 8c03b50..85047fd 100644
--- a/src/shell/stage_launch_cleanup.sh
+++ b/src/shell/stage_launch_cleanup.sh
@@ -25,6 +25,14 @@
 # SUCH DAMAGE.
 #
 . "$(dirname "$0")/common.sh"
+# The script re-runs itself with --network-cleanup, holding the network lock,
+# to release the instance's network resources. See setup_network in
+# stage_launch.sh.
+mode=cleanup
+if [ "$1" = "--network-cleanup" ]; then
+    mode=network_cleanup
+    shift
+fi
 data_root="$1"
 instance="$2"
 type=$3
@@ -34,11 +42,12 @@ network_cleanup()
     if [ ! -f "$data_root"/networks/cur ]; then
         return 0
     fi
-    newdb=$(mktemp)
+    # Same directory, so the mv below is an atomic rename.
+    newdb=$(mktemp "$data_root"/networks/cur.XXXXXX) || return 1
     while read ln; do
         ins=$(echo "$ln" | awk -F, '{ print $2 }')
         if [ "$ins" != "$instance" ]; then
-            echo $ln >> $newdb
+            echo "$ln" >> "$newdb"
             continue
         fi
         type=$(echo "$ln" | awk -F, '{ print $1 }')
@@ -62,8 +71,7 @@ network_cleanup()
             ;;
         esac
     done < "$data_root"/networks/cur
-    rm "$data_root"/networks/cur
-    mv $newdb "$data_root"/networks/cur
+    mv "$newdb" "$data_root"/networks/cur
 }
 
 umount_reverse_order()
@@ -125,5 +133,16 @@ cleanup()
     esac
 }
 
+if [ "$mode" = "network_cleanup" ]; then
+    network_cleanup
+    exit
+fi
 cleanup
-network_cleanup
+_flags=""
+case $- in
+*x*)
+    _flags=-x
+    ;;
+esac
+lockf -k "${data_root}/networks/lock" /bin/sh $_flags "$0" \
+  --network-cleanup "$data_root" "$instance"