fix(worker): stop 4 s DNS stalls on the isolated bridge by giving its containers their own resolvers

CLAUDE.md

@@ -153,6 +153,7 @@ Two separate Docker images:
 | `CASHPILOT_EGRESS_IP` | No | -- | State the public IP directly (validated; a LAN/tailnet address is rejected) |
 | `CASHPILOT_EGRESS_IP_URL` | No | -- | Custom IP-echo endpoint returning a bare IP |
 | `CASHPILOT_CONTAINER_NETWORK` | No | -- | A bridge you created; bridge-networked containers join it instead of Docker's default, so host firewall rules can confine them (docs/home-network-security.md). Missing = deploy refused |
+| `CASHPILOT_CONTAINER_DNS` | No | `1.1.1.1,9.9.9.9` | Resolvers for containers on that bridge only, so their lookups skip a LAN/tailnet resolver the firewall rejects. Empty = the daemon's list |
 
 ### Egress IP awareness (CashPilot-5qc)
 

app/orchestrator.py

@@ -8,6 +8,7 @@
 from __future__ import annotations
 
 import contextlib
+import ipaddress
 import logging
 import os
 import time
@@ -287,6 +288,36 @@ def container_network(network_mode: str | None) -> str | None:
     return _CONTAINER_NETWORK
 
 
+#: Resolvers for containers on CASHPILOT_CONTAINER_NETWORK. Docker's embedded
+#: resolver forwards their lookups from inside that network, so a daemon whose
+#: list starts with a LAN or tailnet resolver gets refused by the very rules
+#: that isolate the bridge, and every lookup waits about 4 s before Docker
+#: falls through to the next server. Empty keeps the daemon's own list.
+_CONTAINER_DNS = os.getenv("CASHPILOT_CONTAINER_DNS", "1.1.1.1,9.9.9.9")
+
+
+def container_dns(network: str | None) -> list[str] | None:
+    """The resolvers a new container gets, or None for the daemon's own.
+
+    Only a container joined to the operator's isolated bridge gets them; every
+    other container, and the host, keep Docker's default. A setting that is not
+    a list of IP addresses is refused here, before the running container is
+    touched, for the same reason as container_network.
+    """
+    if not network:
+        return None
+    servers = [entry.strip() for entry in _CONTAINER_DNS.split(",") if entry.strip()]
+    for server in servers:
+        try:
+            ipaddress.ip_address(server)
+        except ValueError:
+            raise ContainerNetworkError(
+                f"CASHPILOT_CONTAINER_DNS has {server!r}; it takes IP addresses separated by commas, "
+                f"for example 1.1.1.1,9.9.9.9, or nothing to keep the Docker daemon's resolvers."
+            ) from None
+    return servers or None
+
+
 #: The runtime for containers on bridge networking when the deploy request
 #: names none, e.g. a gVisor runtime registered in daemon.json. Empty means
 #: Docker's default, as before. CashPilot never picks one for the operator.
@@ -395,8 +426,10 @@ def deploy_raw(
 
     # Decided before the old container is touched: a network or runtime that
     # cannot be used must fail the deploy while the running service is still running.
+    # The resolvers are checked here for the same reason.
     network = container_network(network_mode)
     runtime = container_runtime(slug, network_mode, runtime)
+    dns = container_dns(network)
 
     # Remove any existing container with the same name
     try:
@@ -468,6 +501,8 @@ def deploy_raw(
         # onto the operator's bridge is joined to it and nothing else.
         network_mode=None if network else network_mode,
         network=network,
+        # Only set on the isolated bridge; see container_dns.
+        dns=dns,
         # These images are third-party and closed-source, so they get the minimum
         # kernel surface: every capability dropped, then only the ones the service's
         # own catalog entry declares added back (docs/fleet.md lists them; most

docs/configuration.md

@@ -65,6 +65,7 @@ defensible on its own; together they are impossible to guess.
 | `CASHPILOT_ALLOWED_VOLUME_ROOTS` | unset | Host paths a deploy may bind-mount. | — |
 | `CASHPILOT_PIDS_LIMIT` | unset | `pids` limit applied to managed containers. | — |
 | `CASHPILOT_CONTAINER_NETWORK` | unset | A bridge network you created, which containers that would land on Docker's default bridge join instead, so host firewall rules can confine them. Host-networked services are unaffected. The worker never creates it; a deploy is refused while it is missing. See [Protecting your home network](home-network-security.md). | — |
+| `CASHPILOT_CONTAINER_DNS` | `1.1.1.1,9.9.9.9` | DNS servers, IP addresses separated by commas, for containers that join `CASHPILOT_CONTAINER_NETWORK`. Other containers and the host keep the Docker daemon's list. Set it empty to give the isolated containers the daemon's list too. Applies when a container is next deployed. See [name lookups](home-network-security.md#name-lookups-need-a-resolver-the-rules-allow). | — |
 | `CASHPILOT_CONTAINER_RUNTIME` | unset | A Docker runtime, such as a gVisor runtime you registered in `daemon.json`, for containers on bridge networking. Host-networked services keep Docker's default. A deploy is refused while the daemon does not provide it. See [gVisor](home-network-security.md#7-gvisor-optional). | — |
 | `CASHPILOT_CONTAINER_RUNTIME_OVERRIDES` | unset | Per-service exceptions, `slug=runtime` separated by commas, e.g. `bitping=runsc-hostnet-raw`. `slug=runc` runs that service under `runc`, the runtime Docker ships with, whatever the daemon's default is. | — |
 | `CASHPILOT_DATA_DIR` | `/data` | Where `.worker_id` and `.worker_key` live. | — |

docs/home-network-security.md

@@ -153,24 +153,38 @@ We ran this script twice on a Docker host and checked the result:
 
 Docker answers a container's lookups and forwards them to the DNS servers it was
 given, **from inside the container's network**. On most home networks that
-server is the router, which the rules above block, so every lookup fails and the
-services stop earning. Pick one fix:
-
-- **Give Docker public resolvers.** In `/etc/docker/daemon.json`, set
-  `"dns": ["1.1.1.1", "9.9.9.9"]` and restart Docker (this restarts every
-  container). If you keep a local resolver first in that list, lookups still
-  work, a little slower, because Docker falls through to the next server.
-- **Or let the containers reach the router's DNS port and nothing else on it,**
-  by adding this to the script, with your router's address:
-
-  ```sh
-  ROUTER=192.168.1.1
-  add DOCKER-USER -i "$BR" -d "$ROUTER" -p udp --dport 53 -j RETURN
-  add DOCKER-USER -i "$BR" -d "$ROUTER" -p tcp --dport 53 -j RETURN
-  ```
-
-We tested both: with the exception, lookups through the router work and the
-router's web interface stays blocked.
+server is the router, which the rules above block. With a VPN such as Tailscale
+on the host, it is often `100.100.100.100`, which the `100.64.0.0/10` rule
+blocks. Every lookup then fails, or waits about four seconds for Docker to fall
+through to the next server on its list.
+
+The worker handles this for you. A container it puts on
+`CASHPILOT_CONTAINER_NETWORK` gets its own DNS servers, `1.1.1.1` and `9.9.9.9`
+by default, which the rules allow. Containers on other networks, and the host
+itself, keep the Docker daemon's list. To pick other servers, give the worker
+their addresses:
+
+```yaml
+CASHPILOT_CONTAINER_DNS: 9.9.9.9,149.112.112.112
+```
+
+Set it empty to give the isolated containers the daemon's list instead. A value
+that is not a list of IP addresses is refused before the running container is
+touched. The setting applies when a container is next deployed: a container
+keeps the DNS servers it was created with until then.
+
+To use your router's DNS instead, set `CASHPILOT_CONTAINER_DNS` to the router's
+address and let the containers reach its DNS port and nothing else on it, by
+adding this to the script:
+
+```sh
+ROUTER=192.168.1.1
+add DOCKER-USER -i "$BR" -d "$ROUTER" -p udp --dport 53 -j RETURN
+add DOCKER-USER -i "$BR" -d "$ROUTER" -p tcp --dport 53 -j RETURN
+```
+
+We tested the exception: lookups through the router work and the router's web
+interface stays blocked.
 
 If you enabled IPv6 on the bridge and your router also answers DNS on an IPv6
 address, add the same exception to the [IPv6 script](#the-rules) below, with

tests/test_container_dns.py

@@ -0,0 +1,85 @@
+"""Containers on the isolated bridge get resolvers the bridge's rules allow.
+
+Docker's embedded resolver forwards a container's lookups to the daemon's DNS
+servers from inside the container's network. On one fleet host the daemon listed
+Tailscale's 100.100.100.100 first, and the isolation rules reject 100.64.0.0/10
+from cp-isolated, so every lookup from an isolated earner waited about 4 s for
+the fall-through to 1.1.1.1: thousands of stalls a day and a dockerd log line
+for each. The worker now gives the containers it puts on that bridge their own
+resolvers, CASHPILOT_CONTAINER_DNS (default 1.1.1.1 and 9.9.9.9). Containers
+elsewhere, and the host, keep the daemon's list.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from app import orchestrator
+
+
+def _client(old=None):
+    client = MagicMock()
+    net = MagicMock()
+    net.attrs = {"Driver": "bridge"}
+    client.networks.get.return_value = net
+    if old is None:
+        client.containers.get.side_effect = orchestrator.NotFound("nope")
+    else:
+        client.containers.get.return_value = old
+    created = MagicMock()
+    created.id = created.short_id = "new"
+    client.containers.run.return_value = created
+    return client
+
+
+def _deploy(client, network_setting, network_mode=None, dns_setting="1.1.1.1,9.9.9.9"):
+    with (
+        patch.object(orchestrator, "_CONTAINER_NETWORK", network_setting),
+        patch.object(orchestrator, "_CONTAINER_DNS", dns_setting),
+        patch.object(orchestrator, "_get_client", return_value=client),
+    ):
+        orchestrator.deploy_raw(slug="honeygain", image="honeygain/honeygain", network_mode=network_mode)
+    return client.containers.run.call_args.kwargs
+
+
+class TestWhoGetsTheResolvers:
+    def test_a_container_on_the_isolated_bridge_gets_them(self):
+        kwargs = _deploy(_client(), "cashpilot-isolated")
+        assert kwargs["network"] == "cashpilot-isolated"
+        assert kwargs["dns"] == ["1.1.1.1", "9.9.9.9"]
+
+    def test_the_operator_can_name_others(self):
+        kwargs = _deploy(_client(), "cashpilot-isolated", dns_setting=" 9.9.9.9 , 2620:fe::fe ")
+        assert kwargs["dns"] == ["9.9.9.9", "2620:fe::fe"]
+
+    def test_empty_keeps_the_daemons_list(self):
+        kwargs = _deploy(_client(), "cashpilot-isolated", dns_setting="")
+        assert kwargs["dns"] is None
+
+    def test_without_an_isolated_bridge_nothing_is_set(self):
+        kwargs = _deploy(_client(), "")
+        assert kwargs["dns"] is None
+
+    @pytest.mark.parametrize("mode", ["host", "none"])
+    def test_host_and_none_networking_keep_the_daemons_list(self, mode):
+        kwargs = _deploy(_client(), "cashpilot-isolated", network_mode=mode)
+        assert kwargs["network"] is None
+        assert kwargs["dns"] is None
+
+
+class TestABadSettingIsRefusedBeforeAnythingIsTouched:
+    def test_a_name_instead_of_an_address(self):
+        old = MagicMock()
+        client = _client(old=old)
+        with pytest.raises(orchestrator.ContainerNetworkError, match="CASHPILOT_CONTAINER_DNS"):
+            _deploy(client, "cashpilot-isolated", dns_setting="1.1.1.1,dns.quad9.net")
+        old.stop.assert_not_called()
+        old.remove.assert_not_called()
+        client.containers.run.assert_not_called()
+
+    def test_ignored_when_no_container_joins_the_bridge(self):
+        """A bad value cannot break a worker that does not isolate anything."""
+        kwargs = _deploy(_client(), "", dns_setting="not-an-address")
+        assert kwargs["dns"] is None