diff --git a/dist/amazon-efs-mount-watchdog.rc b/dist/amazon-efs-mount-watchdog.rc new file mode 100644 index 00000000..613d70f3 --- /dev/null +++ b/dist/amazon-efs-mount-watchdog.rc @@ -0,0 +1,32 @@ +#!/bin/sh + +# PROVIDE: amazon_efs_mount_watchdog +# REQUIRE: DAEMON +# KEYWORD: shutdown +# +# This service does NOT need to be enabled in /etc/rc.conf: mount_efs(8) +# starts it on demand via `service amazon_efs_mount_watchdog onestart` +# whenever an EFS filesystem is mounted with TLS or IAM. It is installed +# disabled by design. +# +# You only need to enable it manually if you want the watchdog running +# independently of any mount (rare): +# +# amazon_efs_mount_watchdog_enable="YES" + +. /etc/rc.subr + +name=amazon_efs_mount_watchdog +rcvar=amazon_efs_mount_watchdog_enable +desc="Amazon EFS mount watchdog" + +load_rc_config $name + +: ${amazon_efs_mount_watchdog_enable:=NO} + +pidfile="/var/run/${name}.pid" +command=/usr/sbin/daemon +command_args="-c -f -P ${pidfile} -t ${name} -H \ + /usr/local/sbin/amazon-efs-mount-watchdog" + +run_rc_command "$1" diff --git a/src/efs_utils_common/mount_options.py b/src/efs_utils_common/mount_options.py index 4d53d014..33204343 100644 --- a/src/efs_utils_common/mount_options.py +++ b/src/efs_utils_common/mount_options.py @@ -7,6 +7,8 @@ # the License. +import sys + from efs_utils_common.constants import ( AP_REGEX_PATTERN, MOUNT_TYPE_S3FILES, @@ -65,6 +67,13 @@ def get_nfs_mount_options(options, config): "which is not compatible with S3 Files." ) options["nfsvers"] = "4.2" + elif sys.platform.startswith("freebsd"): + # FreeBSD's mount_nfs does not recognize nfsvers=4.1; it would leave + # mountmode=ANY and send an NFSv3 NULL probe which EFS rejects. Use + # the native `nfsv4` option + `minorversion=1` so mount_nfs sets + # mountmode=V4 and sends a v4 NULL probe. + options["nfsv4"] = None + options["minorversion"] = "1" else: options["nfsvers"] = "4.1" if not check_if_platform_is_mac() else "4.0" @@ -87,6 +96,15 @@ def get_nfs_mount_options(options, config): if "noresvport" not in options: options["noresvport"] = None + # FreeBSD requires oneopenown for AWS EFS compatibility. + # retrycnt=1 matches the FreeBSD-recommended EFS mount options and avoids + # the default infinite retry on failure. + if sys.platform.startswith("freebsd"): + if "oneopenown" not in options: + options["oneopenown"] = None + if "retrycnt" not in options: + options["retrycnt"] = "1" + # Set mountport to 2049 for MacOS if check_if_platform_is_mac(): options["mountport"] = "2049" diff --git a/src/efs_utils_common/mount_utils.py b/src/efs_utils_common/mount_utils.py index 2356e46a..e8d699ec 100644 --- a/src/efs_utils_common/mount_utils.py +++ b/src/efs_utils_common/mount_utils.py @@ -62,7 +62,15 @@ def mount_nfs(config, dns_name, path, mountpoint, options, fallback_ip_address=N nfs_options = get_nfs_mount_options(options, config) - if not check_if_platform_is_mac(): + if sys.platform.startswith("freebsd"): + command = [ + "/sbin/mount_nfs", + "-o", + nfs_options, + mount_path, + mountpoint, + ] + elif not check_if_platform_is_mac(): command = [ "/sbin/mount.nfs4", mount_path, diff --git a/src/efs_utils_common/proxy.py b/src/efs_utils_common/proxy.py index dd38edb3..3d3332a6 100644 --- a/src/efs_utils_common/proxy.py +++ b/src/efs_utils_common/proxy.py @@ -290,7 +290,7 @@ def write_stunnel_config_file( # Only support in stunnel version 5.25+. global_config["foreground"] = "quiet" - if any( + if sys.platform.startswith("freebsd") or any( release in system_release_version for release in SKIP_NO_SO_BINDTODEVICE_RELEASES ): @@ -595,7 +595,9 @@ def poll_tunnel_process(tunnel_proc, fs_id, mount_completed): def get_init_system(comm_file="/proc/1/comm"): init_system = DEFAULT_UNKNOWN_VALUE - if not check_if_platform_is_mac(): + if sys.platform.startswith("freebsd"): + init_system = "rc" + elif not check_if_platform_is_mac(): try: with open(comm_file) as f: init_system = f.read().strip() @@ -641,6 +643,25 @@ def start_watchdog(init_system): else: logging.debug("%s is already running", WATCHDOG_SERVICE) + elif init_system == "rc": + # FreeBSD: use service(8) with onestart/onestatus so the watchdog + # runs even when the user hasn't enabled it in rc.conf. + rc = subprocess.call( + ["/usr/sbin/service", WATCHDOG_SERVICE, "onestatus"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + close_fds=True, + ) + if rc != 0: + subprocess.Popen( + ["/usr/sbin/service", WATCHDOG_SERVICE, "onestart"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + close_fds=True, + ) + else: + logging.debug("%s is already running", WATCHDOG_SERVICE) + elif init_system == "launchd": rc = subprocess.Popen( ["sudo", "launchctl", "list", WATCHDOG_SERVICE], diff --git a/src/watchdog/__init__.py b/src/watchdog/__init__.py index adfd01b1..0ee861b6 100755 --- a/src/watchdog/__init__.py +++ b/src/watchdog/__init__.py @@ -749,6 +749,58 @@ def get_file_safe_mountpoint(mount): return mountpoint + "." + opts["port"] +def _get_freebsd_established_loopback_ports(): + """ + Return the set of local ports (as strings) with an ESTABLISHED tcp4 + loopback (127.0.0.1) connection, per sockstat(1), or None if sockstat + could not be run. The kernel NFS client's connection to a live efs-proxy + shows up here; an orphaned efs-proxy (e.g. left over from a failed mount + attempt) only has its LISTEN socket, which sockstat reports with a + foreign address of "*:*". + """ + try: + process = subprocess.run( + ["sockstat", "-4"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + universal_newlines=True, + timeout=5, + ) + except Exception as e: + logging.warning("Unable to run sockstat: %s", e) + return None + + established_ports = set() + for line in process.stdout.splitlines(): + parts = line.split() + if len(parts) < 3: + continue + proto, local, foreign = parts[-3], parts[-2], parts[-1] + if not proto.startswith("tcp"): + continue + if foreign in ("*:*", "*.*"): + continue + if not local.startswith("127.0.0.1:"): + continue + established_ports.add(local.rsplit(":", 1)[1]) + return established_ports + + +def _freebsd_proxy_port_established(port): + """ + Return True if `port` has an ESTABLISHED tcp4 loopback connection (efs-proxy + alive and serving the kernel NFS client), False if it does not, or None if + sockstat could not be run. Used by the FreeBSD stunnel health check in place + of a `df` probe: a live proxy keeps a persistent ESTABLISHED loopback socket + even when the mount is idle, whereas `df` measures the whole end-to-end path + (including the EFS backend) and times out on backend slowness alone. + """ + established_ports = _get_freebsd_established_loopback_ports() + if established_ports is None: + return None + return port in established_ports + + def get_current_local_nfs_mounts(mount_file="/proc/mounts"): """ Return a dict of the current NFS mounts for servers running on localhost, keyed by the mountpoint and port as it @@ -756,7 +808,92 @@ def get_current_local_nfs_mounts(mount_file="/proc/mounts"): """ mounts = [] - if not check_if_running_on_macos(): + if sys.platform.startswith("freebsd"): + # FreeBSD: no /proc/mounts, and neither mount(8) nor nfsstat(8) exposes + # the NFS client's TCP port. Use the watchdog's own state files as the + # source of truth: each mount created by mount.efs has a state file + # fs-.. + # in STATE_FILE_DIR. Cross-check with `mount -t nfs` so we skip state + # files whose mountpoint has already been unmounted. State files (not + # live proxy sockets) are the source of truth for which mounts exist, so + # the watchdog still notices a dead efs-proxy and restarts it; sockstat + # is used only below to pick the live port when a mountpoint has more + # than one state file (see the dedup note there). + live_mps = set() + try: + process = subprocess.run( + ["mount", "-t", "nfs"], + check=True, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + for line in process.stdout.splitlines(): + parts = line.split() + if len(parts) >= 3 and parts[1] == "on": + live_mps.add(parts[2]) + except Exception as e: + logging.warning("Unable to list NFS mounts: %s", e) + + if live_mps: + try: + state_files = os.listdir(STATE_FILE_DIR) + except OSError: + state_files = [] + # A mountpoint can end up with more than one state file/port pair, + # e.g. when mount.efs is retried on cold start after efs-proxy lost + # the race against mount_nfs: the failed attempt's efs-proxy + state + # file are never cleaned up, leaving a stale port alongside the one + # the mount actually uses. The kernel NFS client's loopback + # connection to the live proxy's port shows up as an ESTABLISHED + # tcp4 socket on 127.0.0.1: in sockstat(1), whereas an + # orphaned proxy only has its LISTEN socket. Use that to identify + # the truly live port per mountpoint; any other duplicate for the + # same mountpoint is left out of the returned mounts so the existing + # stale-mount cleanup path in check_efs_mounts + # (mark_as_unmounted/clean_up_mount_state) can reap its orphaned + # efs-proxy and state file. + established_ports = _get_freebsd_established_loopback_ports() + + candidates = {} + for sf in state_files: + if not sf.startswith("fs-"): + continue + if sf.endswith("+") or "stunnel-config" in sf: + continue + # fs-.. + stem_port = sf.rsplit(".", 1) + if len(stem_port) != 2: + continue + stem, port = stem_port + try: + int(port) + except ValueError: + continue + inner = stem[len("fs-"):] + if "." not in inner: + continue + _, _, mp_enc = inner.partition(".") + mp = "/" + mp_enc.replace(".", "/") + if mp not in live_mps: + continue + is_established = established_ports is None or port in established_ports + current = candidates.get(mp) + # Prefer a port with a confirmed established loopback connection + # over one without; if sockstat is unavailable (established_ports + # is None) or ambiguous (e.g. more than one port for the + # mountpoint looks established, such as during the brief overlap + # of a watchdog-triggered restart), keep the first one found + # rather than silently dropping a possibly-live mount. + if current is None or (is_established and not current[1]): + candidates[mp] = (port, is_established) + + for mp, (port, _is_established) in candidates.items(): + mounts.append( + Mount._make( + ["127.0.0.1:/", mp, "nfs", "port=" + port, 0, 0] + ) + ) + elif not check_if_running_on_macos(): with open(mount_file) as f: for mount in f: try: @@ -1478,6 +1615,65 @@ def check_stunnel_health( rewrite_state_file(state, state_file_dir, state_file) stunnel_pid = state["pid"] + + if sys.platform.startswith("freebsd"): + # FreeBSD does not probe with `df`. `df` triggers an NFS GETATTR that + # traverses the whole path (kernel -> efs-proxy -> TLS -> EFS backend), + # so its latency is dominated by the backend, not the tunnel: a slow or + # throttled backend makes `df` block past the timeout even though the + # proxy is perfectly healthy. Reacting to that by SIGKILLing efs-proxy + # mid-RPC can leave FreeBSD's NFSv4.1 session unrecoverable (nfsbadse), + # wedging the next `df` in unkillable D-state - the check manufacturing + # the failure it then reacts to. + # + # Instead, probe the proxy directly: a live efs-proxy keeps a persistent + # ESTABLISHED tcp4 loopback connection to the kernel NFS client, even + # when the mount is idle. Only restart when the proxy is genuinely gone, + # confirmed by BOTH signals: no ESTABLISHED socket on its port AND the + # proxy process no longer running. If sockstat is unavailable we have no + # evidence of failure, so we do nothing rather than kill on a guess. + port = os.path.basename(state_file).rsplit(".", 1)[-1] + established = _freebsd_proxy_port_established(port) + state["last_stunnel_check_time"] = current_time + + if established or established is None: + logging.debug( + "efs-proxy [PID: %s] for tls mount on %s passed health check " + "(port %s established=%s).", + stunnel_pid, + mountpoint, + port, + established, + ) + rewrite_state_file(state, state_file_dir, state_file) + return + + # No established connection on the proxy's port. Only treat this as a + # dead proxy if the process is also gone; a live-but-not-serving proxy + # (e.g. mid-reconnect) is left alone rather than SIGKILLed mid-recovery. + if is_mount_stunnel_proc_running(stunnel_pid, state_file, state_file_dir): + logging.warning( + "efs-proxy [PID: %s] for %s is running but has no established " + "connection on port %s; leaving it alone (not restarting).", + stunnel_pid, + mountpoint, + port, + ) + rewrite_state_file(state, state_file_dir, state_file) + return + + logging.warning( + "efs-proxy for %s is not running and has no established connection " + "on port %s, restarting a new efs-proxy process.", + mountpoint, + port, + ) + send_signal_to_running_stunnel_process_group( + stunnel_pid, state_file, state_file_dir, SIGKILL + ) + restart_tls_tunnel(child_procs, state, state_file_dir, state_file) + return + process = subprocess.Popen( ["df", mountpoint], stdout=subprocess.DEVNULL, diff --git a/test/watchdog_test/test_check_stunnel_health.py b/test/watchdog_test/test_check_stunnel_health.py index 194b0f29..1f839768 100644 --- a/test/watchdog_test/test_check_stunnel_health.py +++ b/test/watchdog_test/test_check_stunnel_health.py @@ -37,8 +37,16 @@ def setup_mocks( - mocker, mock_subprocess_success=False, mock_subprocess_timeout_sec=None + mocker, + mock_subprocess_success=False, + mock_subprocess_timeout_sec=None, + platform="linux", ): + # These tests exercise the `df`-based health check path. Pin the platform so + # they run identically on any host, including FreeBSD (where + # check_stunnel_health takes a different, sockstat-based branch that is + # covered by the dedicated FreeBSD tests below). + mocker.patch("watchdog.sys.platform", platform) check_time_mock = mocker.patch("time.time", return_value=FIXED_TIME) popen_mock = None if mock_subprocess_success: @@ -284,3 +292,100 @@ def _test_stunnel_health_checked_passed_for_non_first_check_helper( assert FIXED_TIME == new_state["last_stunnel_check_time"] if not mountpoint: assert mountpoint == new_state["mountpoint"] + + +# --- FreeBSD branch --------------------------------------------------------- +# On FreeBSD check_stunnel_health does not run `df`; it probes the efs-proxy's +# loopback socket via _freebsd_proxy_port_established and only restarts when the +# proxy is confirmed gone (no established socket AND process not running). + +FREEBSD_STATE_FILE_NAME = "fs-deadbeef.mnt.12345" + + +def _write_freebsd_state(tmpdir): + state = { + "mount_time": DEFAULT_MOUNT_TIME, + "mountpoint": "/mnt", + "pid": 9999, + "last_stunnel_check_time": DEFAULT_LAST_STUNNEL_CHECK_TIME, + } + state_file = tmpdir.join(FREEBSD_STATE_FILE_NAME) + state_file.write(json.dumps(state), ensure=True) + return state, state_file + + +def test_freebsd_health_established_does_not_restart(mocker, tmpdir): + setup_mocks(mocker, platform="freebsd16") + config = _get_config(stunnel_health_check_enabled=True) + state, state_file = _write_freebsd_state(tmpdir) + + established_mock = mocker.patch( + "watchdog._freebsd_proxy_port_established", return_value=True + ) + kill_mock = mocker.patch("os.killpg") + restart_mock = mocker.patch("watchdog.restart_tls_tunnel") + + watchdog.check_stunnel_health( + config, state, state_file.dirname, state_file.basename, [], DEFAULT_MOUNTS + ) + + established_mock.assert_called_once_with("12345") + assert 0 == kill_mock.call_count + assert 0 == restart_mock.call_count + + +def test_freebsd_health_sockstat_unavailable_does_not_restart(mocker, tmpdir): + # sockstat unavailable -> no evidence of failure -> do nothing (fail-safe). + setup_mocks(mocker, platform="freebsd16") + config = _get_config(stunnel_health_check_enabled=True) + state, state_file = _write_freebsd_state(tmpdir) + + mocker.patch("watchdog._freebsd_proxy_port_established", return_value=None) + kill_mock = mocker.patch("os.killpg") + restart_mock = mocker.patch("watchdog.restart_tls_tunnel") + + watchdog.check_stunnel_health( + config, state, state_file.dirname, state_file.basename, [], DEFAULT_MOUNTS + ) + + assert 0 == kill_mock.call_count + assert 0 == restart_mock.call_count + + +def test_freebsd_health_not_established_but_proc_alive_does_not_restart( + mocker, tmpdir +): + # No established socket but the proxy process is still running: leave it + # alone (mid-reconnect), do not SIGKILL it mid-recovery. + setup_mocks(mocker, platform="freebsd16") + config = _get_config(stunnel_health_check_enabled=True) + state, state_file = _write_freebsd_state(tmpdir) + + mocker.patch("watchdog._freebsd_proxy_port_established", return_value=False) + mocker.patch("watchdog.is_mount_stunnel_proc_running", return_value=True) + kill_mock = mocker.patch("os.killpg") + restart_mock = mocker.patch("watchdog.restart_tls_tunnel") + + watchdog.check_stunnel_health( + config, state, state_file.dirname, state_file.basename, [], DEFAULT_MOUNTS + ) + + assert 0 == restart_mock.call_count + + +def test_freebsd_health_not_established_and_proc_dead_restarts(mocker, tmpdir): + # No established socket AND proxy process gone: both signals agree the proxy + # is dead, so restart it. + setup_mocks(mocker, platform="freebsd16") + config = _get_config(stunnel_health_check_enabled=True) + state, state_file = _write_freebsd_state(tmpdir) + + mocker.patch("watchdog._freebsd_proxy_port_established", return_value=False) + mocker.patch("watchdog.is_mount_stunnel_proc_running", return_value=False) + restart_mock = mocker.patch("watchdog.restart_tls_tunnel") + + watchdog.check_stunnel_health( + config, state, state_file.dirname, state_file.basename, [], DEFAULT_MOUNTS + ) + + utils.assert_called_once(restart_mock) diff --git a/test/watchdog_test/test_get_current_local_nfs_mounts.py b/test/watchdog_test/test_get_current_local_nfs_mounts.py index 6c10a532..ab3ded67 100644 --- a/test/watchdog_test/test_get_current_local_nfs_mounts.py +++ b/test/watchdog_test/test_get_current_local_nfs_mounts.py @@ -8,12 +8,23 @@ import logging +import pytest + import watchdog MOUNT_FMT_LINE = "{address}:/ {mountpoint} {fs_type} {options} 0 0" DEFAULT_OPTS = "rw,port=12345" +@pytest.fixture(autouse=True) +def _pin_linux_platform(monkeypatch): + # These tests feed a fake /proc/mounts file to get_current_local_nfs_mounts, + # which is the Linux code path. Pin the platform so they run identically on + # any host; the FreeBSD branch reads state files + sockstat instead and is + # covered separately. + monkeypatch.setattr(watchdog.sys, "platform", "linux") + + def _create_mount_file(tmpdir, lines): mount_file = tmpdir.join("mounts") mount_file.write("\n".join(lines))