From 725e807997cc501acc515e18de0a962ba0e874f2 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Tue, 25 Aug 2026 00:05:47 +0200 Subject: [PATCH] feat: write a release manifest, stop apps removed from the desired set Every release now gets a manifest.json at its root: schema_version, release timestamp, resolved app_refs/env_refs (the actual tag resolve.py pulled, never @latest), and the desired apps list. No secrets, no target identifier - whoever's reading it already knows which host they're on. resolve.download_ref now returns (path, resolved_ref) instead of just path, so build_release/resolve_app_envs can capture what was actually resolved, not just what was requested. deploy_to_host reads the previous release's manifest.json off current before switching it, diffs its apps against the new desired set, and docker compose downs anything no longer wanted - using current's own path so that app's last-known compose file/.env are still there. This works even though the new release's own tree also still contains that app's compose file (build_release always copies the whole catalog, not just a target's desired subset) - using the old release avoids depending on that incidentally, and keeps the stop grounded in the last environment that app actually ran with. release_name moves from being generated once per host inside deploy_to_host to once per deploy in main() - keeps a multi-host target's manifest.json identical across every host of the same logical deploy, and deploy_to_host takes it as a parameter now. Closes #109 Co-Authored-By: Claude Sonnet 5 --- AGENTS.md | 2 +- README.md | 3 +- deploy/deploy.py | 48 +++++++++++++--- deploy/resolve.py | 2 +- deploy/tests/test_deploy.py | 105 +++++++++++++++++++++++++++++++---- deploy/tests/test_resolve.py | 6 +- 6 files changed, 143 insertions(+), 23 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f2cd3b9..44e7278 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -393,7 +393,7 @@ GitHub Actions workflow (`.github/workflows/release.yml`) manages releases via [ Deployment helpers live in this repository, entirely under `deploy/`, run only on the GitHub Actions runner - the target host never runs any of this: -- `deploy/deploy.py` is the deploy entrypoint. It resolves and downloads every ref in `app_refs` (the app bundles, at least one required — flightdeck's own `apps/` catalog is just another entry, not implicit) and merges them into a release tree locally; for each app in the target's `apps` mapping, downloads its `env_refs` (still encrypted), decrypts them with the target's private SOPS age key, writes the plaintext into that app's `.env` in the release tree, and renders that app's `*.tpl` files in place with the decrypted values (see "Config Templates" above). It then opens an SSH connection per host, pushes the finished release as one tarball (real `.env`, already-rendered config, all versioned together), bootstraps networks/directories idempotently, switches a timestamped release, and runs `docker compose pull && docker compose up -d` per app directly (no wrapper script on the host at all). +- `deploy/deploy.py` is the deploy entrypoint. It resolves and downloads every ref in `app_refs` (the app bundles, at least one required — flightdeck's own `apps/` catalog is just another entry, not implicit) and merges them into a release tree locally; for each app in the target's `apps` mapping, downloads its `env_refs` (still encrypted), decrypts them with the target's private SOPS age key, writes the plaintext into that app's `.env` in the release tree, and renders that app's `*.tpl` files in place with the decrypted values (see "Config Templates" above). It writes a `manifest.json` into the release tree (`schema_version`, `release` timestamp, resolved `app_refs`/`env_refs` - the actual tag `resolve.py` pulled, never `@latest` - and the desired `apps` list; no secrets, no target identifier, since whoever's reading it is already on that specific host). It then opens an SSH connection per host, pushes the finished release as one tarball (real `.env`, already-rendered config, the manifest, all versioned together), bootstraps networks/directories idempotently, reads the *previous* release's `manifest.json` off `current` and `docker compose down`s any app present there but no longer in the desired set (using `current` before it moves, so that app's last-known compose file/`.env` are still intact - safe even though every release tree already contains every app's compose file regardless of whether the target wants it, since `build_release` copies the whole catalog every time), switches `current` to the new timestamped release, and runs `docker compose pull && docker compose up -d` per desired app directly (no wrapper script on the host at all). - `deploy/resolve.py`, `deploy/collisions.py`, `deploy/vault.py`, and `deploy/render.py` hold, respectively, the ref-resolution, ciphertext collision-detection, decryption, and template-rendering logic - each with real `unittest` coverage in `deploy/tests/`. - `.github/actions/encrypt-env/` is a local composite action for rendering `vaults/` manifests from GitHub Secrets/Variables, encrypting them for age recipients, and publishing `.sops.env` as a GitHub Release asset — vault manifests hold only env/secrets, not app selection - `.github/workflows/deploy-shared.yml` is a reusable workflow consumer repos call to run `deploy/deploy.py` from GitHub Actions over an optional Tailscale connection, without holding any deploy secrets in this repository diff --git a/README.md b/README.md index 35c9bac..8a8fb63 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,8 @@ The deploy is push-based and runs entirely on the GitHub Actions runner: 3. Check each app's env sources for key collisions from the still-encrypted ciphertext (SOPS's dotenv output only encrypts values, so key names are readable without decryption) — scoped to that app's own sources, not across apps. 4. Decrypt each app's env with the target's private SOPS age key (a GitHub Secret) and write it straight into that app's `.env` in the release tree. 5. Render that app's `*.tpl` config files in place, next to its `docker-compose.yml`, using the decrypted values — the same substitution `envsubst` does, run here instead of on the host. -6. Push the finished release (real `.env`, already-rendered config, one tarball) to each host over SSH, switch the `current` symlink, and run `docker compose pull && docker compose up -d` per app. +6. Write a `manifest.json` into the release tree — resolved `app_refs`/`env_refs` (the actual tag pulled, not `@latest`) and the desired app set, no secrets. +7. Push the finished release (real `.env`, already-rendered config, the manifest, one tarball) to each host over SSH. Before switching `current`, compare the new desired app set against the previous release's `manifest.json` and `docker compose down` anything no longer desired, then switch the `current` symlink and run `docker compose pull && docker compose up -d` per app. What gets deployed — which app bundles, which apps actually run, and which encrypted env sources feed each one — is configured declaratively per target; see "Vaults And Targets" below for the manifest format. diff --git a/deploy/deploy.py b/deploy/deploy.py index 504b1e9..45239f2 100644 --- a/deploy/deploy.py +++ b/deploy/deploy.py @@ -40,9 +40,11 @@ def build_release(config, work_dir): apps_dir = release_dir / "apps" apps_dir.mkdir(parents=True) + resolved_app_refs = [] for index, ref in enumerate(config["app_refs"], start=1): package_dir = pull_dir / f"apps-{index}" - bundle = download_ref(ref, package_dir / "pull", default_asset=APPS_BUNDLE_ASSET) + bundle, resolved_ref = download_ref(ref, package_dir / "pull", default_asset=APPS_BUNDLE_ASSET) + resolved_app_refs.append(resolved_ref) extract_dir = package_dir / "extract" with ZipFile(bundle) as archive: archive.extractall(extract_dir) @@ -60,7 +62,7 @@ def build_release(config, work_dir): else: shutil.copy2(entry, target) - return release_dir + return release_dir, resolved_app_refs def render_app_configs(release_dir, app, values): @@ -73,11 +75,14 @@ def render_app_configs(release_dir, app, values): def resolve_app_envs(config, work_dir, release_dir, age_key_file): pull_dir = work_dir / "envs" + resolved_env_refs = {} for app, app_config in config["apps"].items(): - paths = [ + downloaded = [ download_ref(ref, pull_dir / app / str(index)) for index, ref in enumerate(app_config["env_refs"], start=1) ] + paths = [path for path, _ in downloaded] + resolved_env_refs[app] = [resolved_ref for _, resolved_ref in downloaded] check_env_collisions(paths) plaintext = f"APP_NAME={app}\n" + "".join(decrypt_env(path, age_key_file) for path in paths) @@ -86,6 +91,20 @@ def resolve_app_envs(config, work_dir, release_dir, age_key_file): app_env_path.chmod(0o600) render_app_configs(release_dir, app, parse_dotenv(plaintext)) + return resolved_env_refs + + +def write_release_manifest(release_dir, release_name, resolved_app_refs, resolved_env_refs): + manifest = { + "schema_version": 1, + "release": release_name, + "app_refs": resolved_app_refs, + "apps": sorted(resolved_env_refs), + "env_refs": resolved_env_refs, + } + manifest_path = release_dir / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + manifest_path.chmod(0o644) def list_required_networks(release_dir): @@ -123,6 +142,17 @@ def push_release(connection, archive_path, release_path): connection.run(f"chmod 600 {shlex.quote(release_path)}/apps/*/.env", hide=True) +def stop_removed_apps(connection, current_path, apps): + result = connection.run(f"cat {shlex.quote(current_path)}/manifest.json 2>/dev/null || true", hide=True) + if not result.stdout.strip(): + return + previous = json.loads(result.stdout) + removed = sorted(set(previous.get("apps", [])) - set(apps)) + for app in removed: + compose_dir = f"{current_path}/apps/{app}" + connection.run(f"cd {shlex.quote(compose_dir)} 2>/dev/null && docker compose down || true", hide=True) + + def prune_releases(connection, releases_path, keep_releases): result = connection.run(f"ls -1dt {shlex.quote(releases_path)}/*/ 2>/dev/null || true", hide=True) releases = [line.strip().rstrip("/") for line in result.stdout.splitlines() if line.strip()] @@ -131,7 +161,7 @@ def prune_releases(connection, releases_path, keep_releases): connection.run("rm -rf " + " ".join(shlex.quote(release) for release in stale), hide=True) -def deploy_to_host(host, archive_path, apps, networks, config): +def deploy_to_host(host, archive_path, apps, networks, config, release_name): connection = Connection(host) connection.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -141,7 +171,6 @@ def deploy_to_host(host, archive_path, apps, networks, config): releases_path = f"{base_path}/releases" current_path = f"{base_path}/current" - release_name = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") release_path = f"{releases_path}/{release_name}" bootstrap_host(connection, base_path, networks) @@ -152,6 +181,7 @@ def deploy_to_host(host, archive_path, apps, networks, config): env_path = f"{release_path}/apps/{app}/.env" connection.run(f"echo {shlex.quote(f'DATA_DIR={data_dir}')} >> {shlex.quote(env_path)}", hide=True) + stop_removed_apps(connection, current_path, apps) connection.run(f"ln -sfn {shlex.quote(release_path)} {shlex.quote(current_path)}", hide=True) for app in apps: @@ -183,8 +213,10 @@ def main(): age_key_file.write_text(config["sops_age_key"]) age_key_file.chmod(0o600) - release_dir = build_release(config, work_dir) - resolve_app_envs(config, work_dir, release_dir, age_key_file) + release_dir, resolved_app_refs = build_release(config, work_dir) + resolved_env_refs = resolve_app_envs(config, work_dir, release_dir, age_key_file) + release_name = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + write_release_manifest(release_dir, release_name, resolved_app_refs, resolved_env_refs) archive_path = archive_release(release_dir, work_dir) networks = list_required_networks(release_dir) @@ -192,7 +224,7 @@ def main(): for host in config["hosts"]: print(f"Deploying to {host}") - deploy_to_host(host, archive_path, apps, networks, config) + deploy_to_host(host, archive_path, apps, networks, config, release_name) if __name__ == "__main__": diff --git a/deploy/resolve.py b/deploy/resolve.py index b426b37..11c85d9 100644 --- a/deploy/resolve.py +++ b/deploy/resolve.py @@ -79,4 +79,4 @@ def download_ref(ref, out_dir, default_asset=None, run=subprocess.run): path = out_dir / resolved.asset if not path.is_file(): raise RefError(f"{resolved.asset} was not found in {resolved.repo}@{tag}") - return path + return path, f"{resolved.repo}@{tag}:{resolved.asset}" diff --git a/deploy/tests/test_deploy.py b/deploy/tests/test_deploy.py index 37cc3a3..56f90f3 100644 --- a/deploy/tests/test_deploy.py +++ b/deploy/tests/test_deploy.py @@ -1,4 +1,5 @@ import importlib.util +import json import sys import tarfile import tempfile @@ -92,12 +93,13 @@ def test_merges_app_dirs_and_shared_top_level_files(self): ) def fake_download_ref(ref, out_dir, default_asset=None, run=None): - return apps_zip + return apps_zip, "owner/repo@v1.0.0:apps.zip" config = {"app_refs": ["owner/repo@latest:apps.zip"]} with patch.object(deploy, "download_ref", side_effect=fake_download_ref): - release_dir = deploy.build_release(config, work_dir / "work") + release_dir, resolved_app_refs = deploy.build_release(config, work_dir / "work") + self.assertEqual(resolved_app_refs, ["owner/repo@v1.0.0:apps.zip"]) self.assertTrue((release_dir / "apps" / "common.yml").is_file()) self.assertTrue((release_dir / "apps" / "networks.yml").is_file()) self.assertTrue((release_dir / "apps" / "traefik" / "docker-compose.yml").is_file()) @@ -111,7 +113,7 @@ def test_raises_on_app_dir_conflict_across_bundles(self): zips = {"a.zip": first_zip, "b.zip": second_zip} def fake_download_ref(ref, out_dir, default_asset=None, run=None): - return zips["a.zip"] if "a.zip" in ref else zips["b.zip"] + return (zips["a.zip"], "owner/repo@v1.0.0:a.zip") if "a.zip" in ref else (zips["b.zip"], "owner/repo@v1.0.0:b.zip") config = {"app_refs": ["owner/repo@latest:a.zip", "owner/repo@latest:b.zip"]} with patch.object(deploy, "download_ref", side_effect=fake_download_ref), self.assertRaises(deploy.DeployError): @@ -125,7 +127,7 @@ def test_raises_on_shared_file_conflict_across_bundles(self): zips = {"a.zip": first_zip, "b.zip": second_zip} def fake_download_ref(ref, out_dir, default_asset=None, run=None): - return zips["a.zip"] if "a.zip" in ref else zips["b.zip"] + return (zips["a.zip"], "owner/repo@v1.0.0:a.zip") if "a.zip" in ref else (zips["b.zip"], "owner/repo@v1.0.0:b.zip") config = {"app_refs": ["owner/repo@latest:a.zip", "owner/repo@latest:b.zip"]} with patch.object(deploy, "download_ref", side_effect=fake_download_ref), self.assertRaises(deploy.DeployError): @@ -137,7 +139,7 @@ def test_raises_when_bundle_has_no_apps_dir(self): empty_zip = make_zip(work_dir / "src" / "empty.zip", {"README.md": "n/a\n"}) def fake_download_ref(ref, out_dir, default_asset=None, run=None): - return empty_zip + return empty_zip, "owner/repo@v1.0.0:empty.zip" config = {"app_refs": ["owner/repo@latest:empty.zip"]} with patch.object(deploy, "download_ref", side_effect=fake_download_ref), self.assertRaises(deploy.DeployError): @@ -186,10 +188,12 @@ def test_writes_decrypted_env_and_renders_configs(self): config = {"apps": {"codecov": {"env_refs": ["owner/repo@latest:a.sops.env"]}}} with ( - patch.object(deploy, "download_ref", return_value=ciphertext), + patch.object(deploy, "download_ref", return_value=(ciphertext, "owner/repo@v1.0.0:a.sops.env")), patch.object(deploy, "decrypt_env", return_value="ADMIN_MAIL=a@example.com\n"), ): - deploy.resolve_app_envs(config, work_dir / "work", release_dir, work_dir / "key.txt") + resolved_env_refs = deploy.resolve_app_envs(config, work_dir / "work", release_dir, work_dir / "key.txt") + + self.assertEqual(resolved_env_refs, {"codecov": ["owner/repo@v1.0.0:a.sops.env"]}) env_path = release_dir / "apps" / "codecov" / ".env" self.assertEqual(env_path.read_text(), "APP_NAME=codecov\nADMIN_MAIL=a@example.com\n") @@ -208,7 +212,8 @@ def test_raises_on_collision_within_one_apps_own_env_refs(self): second_env.write_text("DOMAIN=ENC[AES256_GCM,data:Cd==,iv:xx==,tag:yy==,type:str]\n") def fake_download_ref(ref, out_dir, default_asset=None, run=None): - return first_env if ref.endswith(":a.sops.env") else second_env + path = first_env if ref.endswith(":a.sops.env") else second_env + return path, ref config = { "apps": { @@ -234,7 +239,8 @@ def test_allows_same_key_across_different_apps(self): rybbit_env.write_text("DOMAIN=ENC[AES256_GCM,data:Cd==,iv:xx==,tag:yy==,type:str]\n") def fake_download_ref(ref, out_dir, default_asset=None, run=None): - return traefik_env if "traefik" in ref else rybbit_env + path = traefik_env if "traefik" in ref else rybbit_env + return path, ref def fake_decrypt_env(path, age_key_file, run=None): return "DOMAIN=example.com\n" @@ -283,16 +289,47 @@ def test_archives_release_contents_without_wrapper_dir(self): self.assertIn("apps/traefik/docker-compose.yml", names) +class WriteReleaseManifestTest(unittest.TestCase): + def test_writes_manifest_with_sorted_apps(self): + with tempfile.TemporaryDirectory() as directory: + release_dir = Path(directory) + + deploy.write_release_manifest( + release_dir, + "20260824T211714Z", + ["owner/repo@v0.8.0"], + { + "traefik": ["owner/repo@v0.8.0:traefik.sops.env"], + "cloudflared": ["owner/repo@v0.8.0:cloudflared.sops.env"], + }, + ) + + manifest = json.loads((release_dir / "manifest.json").read_text()) + self.assertEqual(manifest["schema_version"], 1) + self.assertEqual(manifest["release"], "20260824T211714Z") + self.assertEqual(manifest["app_refs"], ["owner/repo@v0.8.0"]) + self.assertEqual(manifest["apps"], ["cloudflared", "traefik"]) + self.assertEqual( + manifest["env_refs"], + { + "traefik": ["owner/repo@v0.8.0:traefik.sops.env"], + "cloudflared": ["owner/repo@v0.8.0:cloudflared.sops.env"], + }, + ) + self.assertNotIn("target", manifest) + + class FakeConnection: """Stand-in for fabric.Connection - records commands/uploads instead of opening a real SSH session, so deploy_to_host's command sequence can be verified without a local sshd.""" - def __init__(self, host): + def __init__(self, host, previous_manifest=""): self.host = host self.client = SimpleNamespace(set_missing_host_key_policy=lambda policy: None) self.commands = [] self.uploads = [] + self.previous_manifest = previous_manifest def run(self, command, hide=False): self.commands.append(command) @@ -301,6 +338,8 @@ def run(self, command, hide=False): if command.startswith("ls -1dt"): releases = "\n".join(f"/home/deploy/flightdeck/releases/rel{i}/" for i in range(7)) return SimpleNamespace(stdout=releases + "\n") + if "manifest.json" in command and command.startswith("cat "): + return SimpleNamespace(stdout=self.previous_manifest) return SimpleNamespace(stdout="") def put(self, local, remote): @@ -323,6 +362,7 @@ def test_full_sequence(self): apps=["traefik", "rybbit"], networks=["traefik", "databases", "mcp"], config=config, + release_name="20260824T211714Z", ) self.assertEqual(fake.uploads[0], (str(archive_path), fake.uploads[0][1])) @@ -334,6 +374,7 @@ def test_full_sequence(self): self.assertIn("docker network create databases", joined) self.assertIn("docker network create mcp", joined) self.assertIn("tar -xzf", joined) + self.assertIn("/home/deploy/flightdeck/releases/20260824T211714Z", joined) self.assertIn("mkdir -p /home/deploy/flightdeck/apps-data/traefik", joined) self.assertIn("mkdir -p /home/deploy/flightdeck/apps-data/rybbit", joined) self.assertIn("DATA_DIR=/home/deploy/flightdeck/apps-data/traefik", joined) @@ -350,6 +391,50 @@ def test_full_sequence(self): for kept in ("rel0", "rel1", "rel2", "rel3", "rel4"): self.assertNotIn(kept, prune_command) + def test_stops_apps_removed_from_the_desired_set(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + archive_path = work_dir / "release.tar.gz" + archive_path.write_text("fake archive\n") + config = {"hosts": ["deploy@host"], "keep_releases": 5} + + previous_manifest = json.dumps({"apps": ["traefik", "gatus"]}) + fake = FakeConnection("deploy@host", previous_manifest=previous_manifest) + with patch.object(deploy, "Connection", return_value=fake): + deploy.deploy_to_host( + "deploy@host", + archive_path, + apps=["traefik"], + networks=["traefik"], + config=config, + release_name="20260824T211714Z", + ) + + joined = "\n".join(fake.commands) + self.assertIn("cd /home/deploy/flightdeck/current/apps/gatus 2>/dev/null && docker compose down || true", joined) + self.assertNotIn("apps/traefik 2>/dev/null && docker compose down", joined) + + def test_no_previous_manifest_stops_nothing(self): + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + archive_path = work_dir / "release.tar.gz" + archive_path.write_text("fake archive\n") + config = {"hosts": ["deploy@host"], "keep_releases": 5} + + fake = FakeConnection("deploy@host") + with patch.object(deploy, "Connection", return_value=fake): + deploy.deploy_to_host( + "deploy@host", + archive_path, + apps=["traefik"], + networks=["traefik"], + config=config, + release_name="20260824T211714Z", + ) + + joined = "\n".join(fake.commands) + self.assertNotIn("docker compose down", joined) + if __name__ == "__main__": unittest.main() diff --git a/deploy/tests/test_resolve.py b/deploy/tests/test_resolve.py index 1b8c488..cbc4f60 100644 --- a/deploy/tests/test_resolve.py +++ b/deploy/tests/test_resolve.py @@ -87,9 +87,10 @@ def run(cmd, **kwargs): asset.write_text("data") return ok() - path = resolve.download_ref("owner/repo@v1.2.3:thing.zip", out, run=run) + path, resolved_ref = resolve.download_ref("owner/repo@v1.2.3:thing.zip", out, run=run) self.assertEqual(path, asset) self.assertTrue(path.is_file()) + self.assertEqual(resolved_ref, "owner/repo@v1.2.3:thing.zip") def test_resolves_latest_before_download(self): with tempfile.TemporaryDirectory() as directory: @@ -104,9 +105,10 @@ def run(cmd, **kwargs): asset.write_text("data") return ok() - resolve.download_ref("owner/repo@latest", out, default_asset="flightdeck.zip", run=run) + _, resolved_ref = resolve.download_ref("owner/repo@latest", out, default_asset="flightdeck.zip", run=run) download_call = [c for c in calls if "download" in c][0] self.assertIn("v9.9.9", download_call) + self.assertEqual(resolved_ref, "owner/repo@v9.9.9:flightdeck.zip") def test_raises_when_asset_missing_after_download(self): with tempfile.TemporaryDirectory() as directory: