Keep :latest patched between releases, not just at them

.github/workflows/refresh.yml

@@ -0,0 +1,183 @@
+name: Refresh published image
+
+on:
+  schedule:
+    # Mondays at 04:00 UTC, two hours ahead of the Trivy scan — so the scan
+    # grades a freshly-rebuilt `:latest` instead of last release's packages.
+    - cron: '0 4 * * 1'
+  workflow_dispatch:
+
+# A release only happens when there's code to ship, but OS CVEs land on their
+# own schedule. This rebuilds the newest release's *source* against today's
+# base image and republishes `:latest`, so the published image stays patched
+# between releases without inventing a version nobody wrote code for.
+jobs:
+  refresh:
+    runs-on: ubuntu-latest
+    # Same actor gate as release.yml, since this publishes and deploys — but
+    # only on the manual path. On a schedule event `github.actor` is whoever
+    # last edited the workflow, so gating cron too would let an edit silently
+    # stop the weekly patch.
+    if: github.event_name == 'schedule' || github.actor == github.repository_owner
+    permissions:
+      contents: read
+      packages: write
+      id-token: write # for cosign keyless signing via OIDC
+    steps:
+      - name: Resolve the newest release
+        id: release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          TAG=$(gh api "repos/${{ github.repository }}/releases/latest" --jq .tag_name)
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "version=${TAG#v}" >> "$GITHUB_OUTPUT"
+          echo "Refreshing $TAG"
+
+      # The tag, not main: `:latest` must keep pointing at released code. Only
+      # the OS packages underneath it are allowed to move.
+      - name: Checkout the release tag
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ steps.release.outputs.tag }}
+
+      - name: Record the tag's commit
+        id: src
+        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Login to GHCR
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Name this refresh
+        id: name
+        run: |
+          echo "tag=refresh-${{ steps.release.outputs.version }}-$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT"
+
+      # Pushed under the dated tag alone. `:latest` moves only after this image
+      # has been scanned clean and smoke-tested, further down.
+      - name: Build + push the refreshed image
+        id: build
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          sbom: true
+          provenance: true
+          tags: ghcr.io/${{ github.repository_owner }}/codecity:${{ steps.name.outputs.tag }}
+          build-args: |
+            GIT_SHA=${{ steps.src.outputs.sha }}
+            VERSION=${{ steps.release.outputs.version }}
+          # The whole point of the run is a fresh base and a fresh apt upgrade;
+          # a cached runtime layer would republish the same packages.
+          pull: true
+          no-cache-filters: runtime
+          # Import only. A weekly multi-arch `mode=max` export would evict the
+          # entries PR builds actually read out of the 10GB cache.
+          cache-from: type=gha
+
+      # If a rebuild doesn't clear the CVEs, they need a real dependency change
+      # — so fail here, leave `:latest` where it is, and let Monday's scan file
+      # the issue with an accurate "a rebuild won't fix this" premise.
+      - name: Trivy scan the rebuild
+        uses: aquasecurity/trivy-action@v0.36.0
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          image-ref: ghcr.io/${{ github.repository_owner }}/codecity:${{ steps.name.outputs.tag }}
+          format: table
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          exit-code: 1
+
+      - name: Smoke test the refreshed image
+        run: |
+          IMAGE=ghcr.io/${{ github.repository_owner }}/codecity:${{ steps.name.outputs.tag }}
+          docker run --rm -d --name smoke --init -p 18080:8080 "$IMAGE"
+          trap 'docker rm -f smoke >/dev/null 2>&1 || true' EXIT
+          waited=0
+          while [ "$waited" -lt 30 ]; do
+            status=$(docker inspect --format '{{ .State.Health.Status }}' smoke 2>/dev/null)
+            if [ "$status" = "healthy" ]; then break; fi
+            waited=$((waited + 1))
+            sleep 1
+          done
+          curl -sf http://localhost:18080/api/health
+
+      - name: Install cosign
+        uses: sigstore/cosign-installer@v3
+
+      - name: Sign image with cosign (keyless via OIDC)
+        run: |
+          cosign sign --yes \
+            ghcr.io/${{ github.repository_owner }}/codecity@${{ steps.build.outputs.digest }}
+
+      # Retags the manifest list already in the registry rather than building
+      # again, so `:latest` resolves to the digest that was just scanned,
+      # smoke-tested and signed.
+      - name: Promote to :latest
+        run: |
+          docker buildx imagetools create \
+            --tag ghcr.io/${{ github.repository_owner }}/codecity:latest \
+            ghcr.io/${{ github.repository_owner }}/codecity@${{ steps.build.outputs.digest }}
+
+      - name: Summary
+        run: |
+          {
+            echo "### Refreshed \`:latest\`"
+            echo
+            echo "| | |"
+            echo "|---|---|"
+            echo "| Source | ${{ steps.release.outputs.tag }} (\`${{ steps.src.outputs.sha }}\`) |"
+            echo "| Refresh tag | \`${{ steps.name.outputs.tag }}\` |"
+            echo "| Digest | \`${{ steps.build.outputs.digest }}\` |"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  deploy:
+    name: Deploy to production
+    needs: [refresh]
+    runs-on: ubuntu-latest
+    # Curls Forgejo with its own token and never touches the GitHub API, so it
+    # needs no GITHUB_TOKEN scopes at all.
+    permissions: {}
+    # A patched image nobody pulls patches nothing, so the refresh moves
+    # production too. Waits on `refresh` because `:latest` isn't promoted until
+    # the rebuild has passed its scan, smoke test and signature.
+    #
+    # Host and repo are secrets, not variables: this repo is public, so its
+    # Actions logs are too, and only secrets are masked in them. Nothing here
+    # echoes either one.
+    steps:
+      - name: Dispatch the Forgejo deploy workflow
+        env:
+          FORGEJO_HOST: ${{ secrets.FORGEJO_HOST }}
+          FORGEJO_REPO: ${{ secrets.FORGEJO_REPO }}
+          FORGEJO_DEPLOY_APP: ${{ vars.FORGEJO_DEPLOY_APP }}
+          FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
+        run: |
+          if [ -z "$FORGEJO_HOST" ] || [ -z "$FORGEJO_REPO" ] || [ -z "$FORGEJO_TOKEN" ]; then
+            echo "::notice::Forgejo deploy not configured (FORGEJO_HOST/FORGEJO_REPO/FORGEJO_TOKEN secrets) — skipping"
+            exit 0
+          fi
+          APP="${FORGEJO_DEPLOY_APP:-app-codecity}"
+          echo "Dispatching $APP"
+          CODE=$(curl -sS -o /tmp/resp -w '%{http_code}' -X POST \
+            "$FORGEJO_HOST/api/v1/repos/$FORGEJO_REPO/actions/workflows/deploy.yml/dispatches" \
+            -H "Authorization: token $FORGEJO_TOKEN" \
+            -H "Content-Type: application/json" \
+            -d "{\"ref\":\"main\",\"inputs\":{\"app\":\"$APP\"}}")
+          if [ "$CODE" != "204" ] && [ "$CODE" != "201" ] && [ "$CODE" != "200" ]; then
+            echo "::error::Forgejo returned $CODE"; cat /tmp/resp; exit 1
+          fi
+          echo "Deploy queued on the Forgejo instance"

.github/workflows/release.yml

@@ -69,6 +69,10 @@ jobs:
             VERSION=${{ steps.tag.outputs.version }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+          # A cached runtime layer pins its apt-get upgrade to the day it was
+          # first built, which would publish CVEs apt could already fix.
+          no-cache-filters: runtime
+          pull: true
 
       - name: Install cosign
         uses: sigstore/cosign-installer@v3
@@ -111,6 +115,9 @@ jobs:
     name: Deploy to production
     needs: [release]
     runs-on: ubuntu-latest
+    # Curls Forgejo with its own token and never touches the GitHub API, so it
+    # needs no GITHUB_TOKEN scopes at all.
+    permissions: {}
     # The image has to exist before the deploy pulls it, so this waits on the
     # release job rather than running alongside it. Skipped when the Forgejo
     # target isn't configured, so a fork's release still succeeds.

.github/workflows/scan.yml

@@ -53,10 +53,14 @@ jobs:
             const summary = vulns
               .map(v => `- **${v.Severity}** ${v.VulnerabilityID} in ${v.PkgName}@${v.InstalledVersion}: ${v.Title}`)
               .join('\n');
+            // One CVE usually lands in several packages, so count the distinct
+            // ids: `3 CVEs` for one openssl advisory sends you looking for three.
+            const ids = new Set(vulns.map(v => v.VulnerabilityID));
+            const n = `${ids.size} HIGH/CRITICAL ${ids.size === 1 ? 'CVE' : 'CVEs'}`;
             await github.rest.issues.create({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              title: `Trivy scan: ${vulns.length} HIGH/CRITICAL CVEs in codecity:latest (${new Date().toISOString().split('T')[0]})`,
-              body: `Weekly scan found ${vulns.length} HIGH/CRITICAL CVEs in the published image. Time to rebuild against a fresh base.\n\n${summary}`,
+              title: `Trivy scan: ${n} in codecity:latest (${new Date().toISOString().split('T')[0]})`,
+              body: `Weekly scan found ${n} across ${vulns.length} package(s) in the published image.\n\nrefresh.yml rebuilds \`:latest\` against a fresh base two hours before this scan, so these survived a rebuild: either that run failed, or they need a real dependency change rather than fresher OS packages.\n\n${summary}`,
               labels: ['security', 'dependencies'],
             });

AGENTS.md

@@ -13,9 +13,11 @@ gh issue list --label P1               # highest priority
 gh issue view <n>                      # full context for one item
 ```
 
-Labels are namespaced. **Category:** `cat:bug`, `cat:enhancement`, `cat:documentation`,
-`cat:security`. **Priority:** `P1`–`P4` (P1 highest). **Status:** `status:in-progress`,
-`status:duplicate`, `status:wontfix`, `status:abandoned`. Plus `question`.
+Most labels are namespaced. **Category:** `cat:bug`, `cat:enhancement`,
+`cat:documentation`. **Priority:** `P1`–`P4` (P1 highest). **Status:**
+`status:in-progress`, `status:duplicate`, `status:wontfix`, `status:abandoned`.
+Plus `question` and `idea`, and the un-namespaced `security` + `dependencies`
+that the weekly Trivy scan puts on the issues it files.
 
 If you discover new work, file an issue rather than leaving an inline `TODO`.
 

Dockerfile

@@ -32,8 +32,9 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
 # System deps. The base image's apt snapshot can lag published security fixes,
 # so apply available upgrades before installing — Trivy fails CI on FIXED
 # HIGH/CRITICAL OS CVEs (e.g. libcurl, pulled in by git). Only useful if this
-# layer actually re-runs: ci.yml excludes this stage from the build cache,
-# because a cached copy pins the upgrade to the day it was first built.
+# layer actually re-runs, so every workflow that builds the image passes
+# `no-cache-filters: runtime`: a cached copy pins the upgrade to the day it
+# was first built.
 # Note: PID 1 init duties are handled by Docker's --init flag (compose: init: true),
 # so we don't install tini here.
 RUN apt-get update \

README.md

@@ -310,6 +310,14 @@ aren't set.
 just deploy
 ```
 
+### Staying patched between releases
+
+OS fixes don't wait for a release, so `refresh.yml` runs every Monday at 04:00 UTC:
+
+- rebuilds the newest release tag against a fresh base, no cached `apt-get upgrade`
+- pushes it as `refresh-<version>-<date>`, then scans, smoke-tests and signs it
+- retags it `:latest` and deploys production, but only if all of that passed
+
 ### Verify signatures
 
 ```sh