From b2ab0af2527a8bf3828939e25ca592fcc742fa42 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:50:36 +0200 Subject: [PATCH] fix(worker): stop 4 s DNS stalls on the isolated bridge by giving its containers their own resolvers Docker's embedded resolver forwards a container's lookups from inside its network. When the daemon lists a LAN or tailnet resolver first, the rules that isolate cp-isolated reject it and every lookup waits about 4 s for the fall-through. Containers the worker puts on CASHPILOT_CONTAINER_NETWORK now get CASHPILOT_CONTAINER_DNS (default 1.1.1.1,9.9.9.9); everything else keeps the daemon's list. A bad value is refused before the running container is touched. --- CLAUDE.md | 1 + app/orchestrator.py | 35 +++++++++++++++ docs/configuration.md | 1 + docs/home-network-security.md | 50 +++++++++++++-------- tests/test_container_dns.py | 85 +++++++++++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 18 deletions(-) create mode 100644 tests/test_container_dns.py diff --git a/CLAUDE.md b/CLAUDE.md index 239ba526..12e459a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -153,6 +153,7 @@ Two separate Docker images: | `CASHPILOT_EGRESS_IP` | No | -- | State the public IP directly (validated; a LAN/tailnet address is rejected) | | `CASHPILOT_EGRESS_IP_URL` | No | -- | Custom IP-echo endpoint returning a bare IP | | `CASHPILOT_CONTAINER_NETWORK` | No | -- | A bridge you created; bridge-networked containers join it instead of Docker's default, so host firewall rules can confine them (docs/home-network-security.md). Missing = deploy refused | +| `CASHPILOT_CONTAINER_DNS` | No | `1.1.1.1,9.9.9.9` | Resolvers for containers on that bridge only, so their lookups skip a LAN/tailnet resolver the firewall rejects. Empty = the daemon's list | ### Egress IP awareness (CashPilot-5qc) diff --git a/app/orchestrator.py b/app/orchestrator.py index dbd530ab..9a9de6ba 100644 --- a/app/orchestrator.py +++ b/app/orchestrator.py @@ -8,6 +8,7 @@ from __future__ import annotations import contextlib +import ipaddress import logging import os import time @@ -287,6 +288,36 @@ def container_network(network_mode: str | None) -> str | None: return _CONTAINER_NETWORK +#: Resolvers for containers on CASHPILOT_CONTAINER_NETWORK. Docker's embedded +#: resolver forwards their lookups from inside that network, so a daemon whose +#: list starts with a LAN or tailnet resolver gets refused by the very rules +#: that isolate the bridge, and every lookup waits about 4 s before Docker +#: falls through to the next server. Empty keeps the daemon's own list. +_CONTAINER_DNS = os.getenv("CASHPILOT_CONTAINER_DNS", "1.1.1.1,9.9.9.9") + + +def container_dns(network: str | None) -> list[str] | None: + """The resolvers a new container gets, or None for the daemon's own. + + Only a container joined to the operator's isolated bridge gets them; every + other container, and the host, keep Docker's default. A setting that is not + a list of IP addresses is refused here, before the running container is + touched, for the same reason as container_network. + """ + if not network: + return None + servers = [entry.strip() for entry in _CONTAINER_DNS.split(",") if entry.strip()] + for server in servers: + try: + ipaddress.ip_address(server) + except ValueError: + raise ContainerNetworkError( + f"CASHPILOT_CONTAINER_DNS has {server!r}; it takes IP addresses separated by commas, " + f"for example 1.1.1.1,9.9.9.9, or nothing to keep the Docker daemon's resolvers." + ) from None + return servers or None + + #: The runtime for containers on bridge networking when the deploy request #: names none, e.g. a gVisor runtime registered in daemon.json. Empty means #: Docker's default, as before. CashPilot never picks one for the operator. @@ -395,8 +426,10 @@ def deploy_raw( # Decided before the old container is touched: a network or runtime that # cannot be used must fail the deploy while the running service is still running. + # The resolvers are checked here for the same reason. network = container_network(network_mode) runtime = container_runtime(slug, network_mode, runtime) + dns = container_dns(network) # Remove any existing container with the same name try: @@ -468,6 +501,8 @@ def deploy_raw( # onto the operator's bridge is joined to it and nothing else. network_mode=None if network else network_mode, network=network, + # Only set on the isolated bridge; see container_dns. + dns=dns, # These images are third-party and closed-source, so they get the minimum # kernel surface: every capability dropped, then only the ones the service's # own catalog entry declares added back (docs/fleet.md lists them; most diff --git a/docs/configuration.md b/docs/configuration.md index 03135467..ab4854f5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -65,6 +65,7 @@ defensible on its own; together they are impossible to guess. | `CASHPILOT_ALLOWED_VOLUME_ROOTS` | unset | Host paths a deploy may bind-mount. | — | | `CASHPILOT_PIDS_LIMIT` | unset | `pids` limit applied to managed containers. | — | | `CASHPILOT_CONTAINER_NETWORK` | unset | A bridge network you created, which containers that would land on Docker's default bridge join instead, so host firewall rules can confine them. Host-networked services are unaffected. The worker never creates it; a deploy is refused while it is missing. See [Protecting your home network](home-network-security.md). | — | +| `CASHPILOT_CONTAINER_DNS` | `1.1.1.1,9.9.9.9` | DNS servers, IP addresses separated by commas, for containers that join `CASHPILOT_CONTAINER_NETWORK`. Other containers and the host keep the Docker daemon's list. Set it empty to give the isolated containers the daemon's list too. Applies when a container is next deployed. See [name lookups](home-network-security.md#name-lookups-need-a-resolver-the-rules-allow). | — | | `CASHPILOT_CONTAINER_RUNTIME` | unset | A Docker runtime, such as a gVisor runtime you registered in `daemon.json`, for containers on bridge networking. Host-networked services keep Docker's default. A deploy is refused while the daemon does not provide it. See [gVisor](home-network-security.md#7-gvisor-optional). | — | | `CASHPILOT_CONTAINER_RUNTIME_OVERRIDES` | unset | Per-service exceptions, `slug=runtime` separated by commas, e.g. `bitping=runsc-hostnet-raw`. `slug=runc` runs that service under `runc`, the runtime Docker ships with, whatever the daemon's default is. | — | | `CASHPILOT_DATA_DIR` | `/data` | Where `.worker_id` and `.worker_key` live. | — | diff --git a/docs/home-network-security.md b/docs/home-network-security.md index 788fa445..b52b2970 100644 --- a/docs/home-network-security.md +++ b/docs/home-network-security.md @@ -153,24 +153,38 @@ We ran this script twice on a Docker host and checked the result: Docker answers a container's lookups and forwards them to the DNS servers it was given, **from inside the container's network**. On most home networks that -server is the router, which the rules above block, so every lookup fails and the -services stop earning. Pick one fix: - -- **Give Docker public resolvers.** In `/etc/docker/daemon.json`, set - `"dns": ["1.1.1.1", "9.9.9.9"]` and restart Docker (this restarts every - container). If you keep a local resolver first in that list, lookups still - work, a little slower, because Docker falls through to the next server. -- **Or let the containers reach the router's DNS port and nothing else on it,** - by adding this to the script, with your router's address: - - ```sh - ROUTER=192.168.1.1 - add DOCKER-USER -i "$BR" -d "$ROUTER" -p udp --dport 53 -j RETURN - add DOCKER-USER -i "$BR" -d "$ROUTER" -p tcp --dport 53 -j RETURN - ``` - -We tested both: with the exception, lookups through the router work and the -router's web interface stays blocked. +server is the router, which the rules above block. With a VPN such as Tailscale +on the host, it is often `100.100.100.100`, which the `100.64.0.0/10` rule +blocks. Every lookup then fails, or waits about four seconds for Docker to fall +through to the next server on its list. + +The worker handles this for you. A container it puts on +`CASHPILOT_CONTAINER_NETWORK` gets its own DNS servers, `1.1.1.1` and `9.9.9.9` +by default, which the rules allow. Containers on other networks, and the host +itself, keep the Docker daemon's list. To pick other servers, give the worker +their addresses: + +```yaml +CASHPILOT_CONTAINER_DNS: 9.9.9.9,149.112.112.112 +``` + +Set it empty to give the isolated containers the daemon's list instead. A value +that is not a list of IP addresses is refused before the running container is +touched. The setting applies when a container is next deployed: a container +keeps the DNS servers it was created with until then. + +To use your router's DNS instead, set `CASHPILOT_CONTAINER_DNS` to the router's +address and let the containers reach its DNS port and nothing else on it, by +adding this to the script: + +```sh +ROUTER=192.168.1.1 +add DOCKER-USER -i "$BR" -d "$ROUTER" -p udp --dport 53 -j RETURN +add DOCKER-USER -i "$BR" -d "$ROUTER" -p tcp --dport 53 -j RETURN +``` + +We tested the exception: lookups through the router work and the router's web +interface stays blocked. If you enabled IPv6 on the bridge and your router also answers DNS on an IPv6 address, add the same exception to the [IPv6 script](#the-rules) below, with diff --git a/tests/test_container_dns.py b/tests/test_container_dns.py new file mode 100644 index 00000000..4223e8dd --- /dev/null +++ b/tests/test_container_dns.py @@ -0,0 +1,85 @@ +"""Containers on the isolated bridge get resolvers the bridge's rules allow. + +Docker's embedded resolver forwards a container's lookups to the daemon's DNS +servers from inside the container's network. On one fleet host the daemon listed +Tailscale's 100.100.100.100 first, and the isolation rules reject 100.64.0.0/10 +from cp-isolated, so every lookup from an isolated earner waited about 4 s for +the fall-through to 1.1.1.1: thousands of stalls a day and a dockerd log line +for each. The worker now gives the containers it puts on that bridge their own +resolvers, CASHPILOT_CONTAINER_DNS (default 1.1.1.1 and 9.9.9.9). Containers +elsewhere, and the host, keep the daemon's list. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from app import orchestrator + + +def _client(old=None): + client = MagicMock() + net = MagicMock() + net.attrs = {"Driver": "bridge"} + client.networks.get.return_value = net + if old is None: + client.containers.get.side_effect = orchestrator.NotFound("nope") + else: + client.containers.get.return_value = old + created = MagicMock() + created.id = created.short_id = "new" + client.containers.run.return_value = created + return client + + +def _deploy(client, network_setting, network_mode=None, dns_setting="1.1.1.1,9.9.9.9"): + with ( + patch.object(orchestrator, "_CONTAINER_NETWORK", network_setting), + patch.object(orchestrator, "_CONTAINER_DNS", dns_setting), + patch.object(orchestrator, "_get_client", return_value=client), + ): + orchestrator.deploy_raw(slug="honeygain", image="honeygain/honeygain", network_mode=network_mode) + return client.containers.run.call_args.kwargs + + +class TestWhoGetsTheResolvers: + def test_a_container_on_the_isolated_bridge_gets_them(self): + kwargs = _deploy(_client(), "cashpilot-isolated") + assert kwargs["network"] == "cashpilot-isolated" + assert kwargs["dns"] == ["1.1.1.1", "9.9.9.9"] + + def test_the_operator_can_name_others(self): + kwargs = _deploy(_client(), "cashpilot-isolated", dns_setting=" 9.9.9.9 , 2620:fe::fe ") + assert kwargs["dns"] == ["9.9.9.9", "2620:fe::fe"] + + def test_empty_keeps_the_daemons_list(self): + kwargs = _deploy(_client(), "cashpilot-isolated", dns_setting="") + assert kwargs["dns"] is None + + def test_without_an_isolated_bridge_nothing_is_set(self): + kwargs = _deploy(_client(), "") + assert kwargs["dns"] is None + + @pytest.mark.parametrize("mode", ["host", "none"]) + def test_host_and_none_networking_keep_the_daemons_list(self, mode): + kwargs = _deploy(_client(), "cashpilot-isolated", network_mode=mode) + assert kwargs["network"] is None + assert kwargs["dns"] is None + + +class TestABadSettingIsRefusedBeforeAnythingIsTouched: + def test_a_name_instead_of_an_address(self): + old = MagicMock() + client = _client(old=old) + with pytest.raises(orchestrator.ContainerNetworkError, match="CASHPILOT_CONTAINER_DNS"): + _deploy(client, "cashpilot-isolated", dns_setting="1.1.1.1,dns.quad9.net") + old.stop.assert_not_called() + old.remove.assert_not_called() + client.containers.run.assert_not_called() + + def test_ignored_when_no_container_joins_the_bridge(self): + """A bad value cannot break a worker that does not isolate anything.""" + kwargs = _deploy(_client(), "", dns_setting="not-an-address") + assert kwargs["dns"] is None