From 726d629e3099b60e9c226fd1a7c153e75618e9bd Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:50:42 +0100 Subject: [PATCH 1/5] fix(news): gate scheduled generation on persisted-state freshness GitHub dropped most hourly News and watchdog cron runs, and the odd-local-hour gate skipped the few that fired, leaving the site 17 hours stale. Tick every 20 minutes in the active window and generate when the news-state asset is older than 100 minutes, with a failure backoff. This replaces the news-watchdog workflow. Closes #363 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/news-watchdog.yml | 134 ---------------------------- .github/workflows/news.yml | 68 +++++++++++--- README.md | 2 +- docs/architecture.md | 2 +- docs/github-actions.md | 8 +- tests/test_workflows.py | 45 +++------- 6 files changed, 76 insertions(+), 183 deletions(-) delete mode 100644 .github/workflows/news-watchdog.yml diff --git a/.github/workflows/news-watchdog.yml b/.github/workflows/news-watchdog.yml deleted file mode 100644 index d1129af4..00000000 --- a/.github/workflows/news-watchdog.yml +++ /dev/null @@ -1,134 +0,0 @@ ---- -name: News watchdog - -on: - schedule: - - cron: "37 * * * *" - workflow_dispatch: - -permissions: - actions: write - contents: read - -concurrency: - group: news-watchdog-${{ github.ref }} - cancel-in-progress: false - -jobs: - catch-up: - name: Dispatch delayed scheduled run - runs-on: ubuntu-24.04 - timeout-minutes: 5 - env: - WAZZUP_TIMEZONE: Europe/Amsterdam - TARGET_WORKFLOW: news.yml - # Briefings are generated on a two-hour daytime cadence. Allow a little - # slack below 120 minutes so jittered scheduled runs still trigger a - # catch-up instead of drifting an extra hour. - CADENCE_INTERVAL_MINUTES: "115" - # Name of the News step that actually generates and persists state. A - # cadence skip leaves this step "skipped", so its success is the signal - # that a run did real work. - GENERATE_STEP_NAME: Generate and persist retained news state - steps: - - name: Check active window - id: cadence - env: - EVENT_NAME: ${{ github.event_name }} - run: |- - set -euo pipefail - - if [[ "${EVENT_NAME}" != "schedule" ]]; then - echo "should_run=true" >>"${GITHUB_OUTPUT}" - echo "reason=manual dispatch" >>"${GITHUB_OUTPUT}" - exit 0 - fi - - local_hour="$(TZ="${WAZZUP_TIMEZONE}" date +%H)" - hour=$((10#${local_hour})) - - # Run the watchdog every hour inside the active window instead of only - # on odd local hours. GitHub schedule jitter routinely shifts runs - # across hour boundaries, so an hour-parity gate disables the watchdog - # exactly when it is needed. The two-hour cadence is enforced below by - # the elapsed time since the last effective generation, not by the - # wall-clock hour of this run. - if (( hour >= 7 && hour < 22 )); then - echo "should_run=true" >>"${GITHUB_OUTPUT}" - echo "reason=within active local window" >>"${GITHUB_OUTPUT}" - else - echo "should_run=false" >>"${GITHUB_OUTPUT}" - echo "reason=outside active local window" >>"${GITHUB_OUTPUT}" - fi - - - name: Dispatch catch-up if needed - if: ${{ steps.cadence.outputs.should_run == 'true' }} - env: - GH_TOKEN: ${{ github.token }} - CADENCE_REASON: ${{ steps.cadence.outputs.reason }} - run: |- - set -euo pipefail - - now_epoch="$(date -u +%s)" - threshold_epoch=$(( now_epoch - 10#${CADENCE_INTERVAL_MINUTES} * 60 )) - - runs_json="$(gh api -X GET \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/${TARGET_WORKFLOW}/runs?branch=${GITHUB_REF_NAME}&per_page=30")" - - # A News run that is still queued or in progress already covers this - # cadence slot, so avoid stacking duplicate catch-up dispatches. - active_runs="$(echo "${runs_json}" \ - | jq '[.workflow_runs[] | select(.status != "completed")] | length')" - - # Find the newest News run that actually generated state. A cadence - # skip completes in seconds with the generate step skipped, so the mere - # existence of a run this hour is not enough -- inspect the generate - # step result of recent completed runs. - last_effective_epoch=0 - while read -r run_id created_at; do - [[ -z "${run_id}" ]] && continue - generated="$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs" \ - | jq --arg step "${GENERATE_STEP_NAME}" \ - '[.jobs[].steps[]? | select(.name == $step and .conclusion == "success")] | length')" - if (( generated > 0 )); then - last_effective_epoch="$(date -u -d "${created_at}" +%s)" - break - fi - done < <(echo "${runs_json}" \ - | jq -r '[.workflow_runs[] | select(.status == "completed")][:15][] | "\(.id) \(.created_at)"') - - { - echo "## Wazzup news watchdog" - echo "" - echo "- Cadence: ${CADENCE_REASON}" - echo "- Cadence interval: ${CADENCE_INTERVAL_MINUTES} minutes" - echo "- In-progress News runs: ${active_runs}" - echo "- Last effective generation (epoch): ${last_effective_epoch}" - } >>"${GITHUB_STEP_SUMMARY}" - - if (( active_runs > 0 )); then - echo "A News run is already queued or in progress; skipping catch-up dispatch." - echo "- Action: skipped (News run already active)" >>"${GITHUB_STEP_SUMMARY}" - exit 0 - fi - - if (( last_effective_epoch >= threshold_epoch )); then - echo "A News run generated fresh state within the cadence interval; no catch-up needed." - echo "- Action: skipped (recent generation within cadence interval)" >>"${GITHUB_STEP_SUMMARY}" - exit 0 - fi - - gh workflow run "${TARGET_WORKFLOW}" --repo "${GITHUB_REPOSITORY}" --ref "${GITHUB_REF_NAME}" -f forceBriefing=auto -f aiProvider=copilot-cli - echo "- Action: dispatched News catch-up (stale generation)" >>"${GITHUB_STEP_SUMMARY}" - - - name: Write skipped summary - if: ${{ steps.cadence.outputs.should_run != 'true' }} - env: - CADENCE_REASON: ${{ steps.cadence.outputs.reason }} - run: |- - { - echo "## Wazzup news watchdog" - echo "" - echo "- Skipped: ${CADENCE_REASON}" - } >>"${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/news.yml b/.github/workflows/news.yml index 3aa6ee82..b8461700 100644 --- a/.github/workflows/news.yml +++ b/.github/workflows/news.yml @@ -3,7 +3,11 @@ name: News on: schedule: - - cron: "7 * * * *" + # GitHub drops or delays many scheduled runs under load, so tick every 20 + # minutes and let the freshness gate below enforce the two-hour cadence. + # 05:00-20:59 UTC covers the 07:00-21:59 Europe/Amsterdam active window in + # both CET and CEST; the gate trims the edge hours to local time. + - cron: "7,27,47 5-20 * * *" workflow_dispatch: inputs: forceBriefing: @@ -50,26 +54,66 @@ jobs: id: cadence env: EVENT_NAME: ${{ github.event_name }} + GH_TOKEN: ${{ github.token }} + STATE_RELEASE: news-state + STATE_ASSET: wazzup-state.zip + # Regenerate once the persisted state is older than this. Ticks are + # 20 minutes apart, so generations land 100-120 minutes apart. + CADENCE_INTERVAL_MINUTES: "100" + # After a failed generation, wait this long before retrying so a + # persistent failure (for example a token rate limit) does not burn + # AI requests on every tick. + FAILURE_BACKOFF_MINUTES: "55" run: |- set -euo pipefail - if [[ "${EVENT_NAME}" != "schedule" ]]; then - echo "should_run=true" >>"${GITHUB_OUTPUT}" - echo "reason=manual dispatch" >>"${GITHUB_OUTPUT}" + decide() { + echo "should_run=$1" >>"${GITHUB_OUTPUT}" + echo "reason=$2" >>"${GITHUB_OUTPUT}" + echo "Cadence decision: should_run=$1 ($2)" exit 0 + } + + if [[ "${EVENT_NAME}" != "schedule" ]]; then + decide true "manual dispatch" fi local_hour="$(TZ="${WAZZUP_TIMEZONE}" date +%H)" hour=$((10#${local_hour})) + if (( hour < 7 || hour >= 22 )); then + decide false "outside active local window" + fi - if (( hour >= 6 && hour < 22 && hour % 2 == 1 )); then - echo "should_run=true" >>"${GITHUB_OUTPUT}" - echo "reason=active two-hour local cadence" >>"${GITHUB_OUTPUT}" - else - echo "should_run=false" >>"${GITHUB_OUTPUT}" - echo "reason=outside active two-hour local cadence" >>"${GITHUB_OUTPUT}" + # Freshness is the age of the persisted state asset, so any trigger + # that generated state (scheduled, manual, or a delayed run) counts + # and dropped cron runs are made up on the next tick. + now_epoch="$(date -u +%s)" + if ! state_updated_at="$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${STATE_RELEASE}" \ + --jq ".assets[] | select(.name == \"${STATE_ASSET}\") | .updated_at" 2>&1)"; then + echo "::warning::Could not read ${STATE_RELEASE} freshness (${state_updated_at}); generating anyway." + decide true "state freshness unknown" + fi + if [[ -z "${state_updated_at}" ]]; then + decide true "no persisted state" fi + state_age_minutes=$(( (now_epoch - $(date -u -d "${state_updated_at}" +%s)) / 60 )) + echo "Persisted state updated at ${state_updated_at} (${state_age_minutes} minutes ago)." + if (( state_age_minutes < 10#${CADENCE_INTERVAL_MINUTES} )); then + decide false "state is ${state_age_minutes} minutes old, within the two-hour cadence" + fi + + last_failure_at="$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/news.yml/runs?status=failure&per_page=1" \ + --jq '.workflow_runs[0].created_at // empty' || true)" + if [[ -n "${last_failure_at}" ]]; then + failure_age_minutes=$(( (now_epoch - $(date -u -d "${last_failure_at}" +%s)) / 60 )) + if (( failure_age_minutes < 10#${FAILURE_BACKOFF_MINUTES} )); then + decide false "state is stale but a generation failed ${failure_age_minutes} minutes ago; backing off" + fi + fi + + decide true "state is ${state_age_minutes} minutes old" + - name: Checkout code if: ${{ steps.cadence.outputs.should_run == 'true' }} uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 @@ -149,7 +193,7 @@ jobs: set -euo pipefail # Dispatch the Pages deploy explicitly. The cross-workflow # workflow_run trigger does not fire for News runs started by - # workflow_dispatch (manual or the news-watchdog catch-up), which + # workflow_dispatch (manual runs), which # left freshly persisted state undeployed. Dispatching here covers # every trigger type. gh workflow run pages.yml --repo "${GITHUB_REPOSITORY}" --ref "${GITHUB_REF_NAME}" @@ -185,6 +229,6 @@ jobs: echo "## Wazzup news state" echo "" echo "- Skipped scheduled run: ${CADENCE_REASON}" - echo "- Active cadence: every two hours from 07:00 to 21:59 local time" + echo "- Active cadence: state older than two hours, checked every 20 minutes from 07:00 to 21:59 local time" echo "- Overnight cadence: skipped" } >>"${GITHUB_STEP_SUMMARY}" diff --git a/README.md b/README.md index ffe3c63c..ce0ebce3 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Important current limitations and deviations from the original target architectu - The backend is Python under [src/wazzup](src/wazzup), not TypeScript. - The frontend is vanilla JavaScript in [public/app.js](public/app.js), not TypeScript/Web Components yet. - JSON Feed and podcast adapters are modeled but not implemented yet. -- Automatic morning/evening due-time selection is implemented in the pipeline and exposed through the scheduled workflow's `auto` mode; the workflow runs on an hourly UTC cron with a local two-hour active-window cadence gate. +- Automatic morning/evening due-time selection is implemented in the pipeline and exposed through the scheduled workflow's `auto` mode; the workflow ticks every 20 minutes in the active window and regenerates when the persisted state is older than about two hours. - JSON is the canonical generated state consumed by the PWA. - Copilot CLI is optional at runtime. If no `COPILOT_REQUESTS_PAT` or `COPILOT_GITHUB_TOKEN` secret exists, the News workflow falls back to the deterministic fake provider so the pipeline and Pages deployment continue to work. - The service worker cache is versioned from generated build metadata; the footer shows the short commit and links back to the repository. diff --git a/docs/architecture.md b/docs/architecture.md index 45f40e97..f3a922ac 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -251,7 +251,7 @@ Example fields: ## Scheduling model -GitHub Actions cron runs in UTC and can be delayed. The workflow should use an hourly UTC cron with a local cadence gate, run every two hours during the configured active window, and let the pipeline decide whether a morning or evening briefing is due in the configured IANA time zone. +GitHub Actions cron runs in UTC and can be delayed or dropped. The workflow uses a frequent UTC cron (every 20 minutes) with a local active-window gate plus a freshness gate on the persisted state, so it generates roughly every two hours during the configured active window even when most ticks are lost, and lets the pipeline decide whether a morning or evening briefing is due in the configured IANA time zone. Recommended defaults: diff --git a/docs/github-actions.md b/docs/github-actions.md index 926bc0ee..361c6874 100644 --- a/docs/github-actions.md +++ b/docs/github-actions.md @@ -7,7 +7,7 @@ | CI | Pull request and manual dispatch | Formatting, syntax linting, tests, compile checks, fixture generation, and generated-data validation. | | Lint | Pull request and manual dispatch | Reusable organization lint workflow from `DevSecNinja/.github`. | | Auto-fix formatting | Manual dispatch | Reusable organization formatting workflow that commits dprint/yamlfmt fixes back to the branch. | -| News | Hourly schedule with a local two-hour active-window cadence gate and manual dispatch | Fetch feeds, generate a rolling briefing, validate data, persist release-backed state, and upload a short-lived `public` artifact for debugging. | +| News | Every 20 minutes in the active window, gated on state freshness, and manual dispatch | Fetch feeds, generate a rolling briefing, validate data, persist release-backed state, and upload a short-lived `public` artifact for debugging. | | Pages | Explicit dispatch from `News` after state persists, push to `main`, and manual dispatch | Deploy PWA and static JSON data to GitHub Pages through the reusable `DevSecNinja/.github` Pages workflow. | | Config Sync | Weekly and manual dispatch | Open PRs when shared repo config from `DevSecNinja/.github` drifts. | | Label Sync | Daily, manual dispatch, and label config changes | Sync repository labels from the org base labels plus repo-specific labels. | @@ -155,7 +155,7 @@ jobs: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_REQUESTS_PAT || secrets.COPILOT_GITHUB_TOKEN }} ``` -The workflow triggers hourly because GitHub cron is UTC-only and does not understand `Europe/Amsterdam` daylight-saving transitions. A first cadence step computes the local hour and continues only on odd local hours from 07:00 to 21:59. This aligns the first run with the configured morning briefing, keeps AI calls to a daytime two-hour cadence, and skips overnight runs entirely. Manual dispatch always runs. +GitHub cron is UTC-only and does not understand `Europe/Amsterdam` daylight-saving transitions, and GitHub drops or delays scheduled runs under load. The workflow therefore ticks every 20 minutes from 05:00 to 20:59 UTC (`7,27,47 5-20 * * *`), which covers the 07:00–21:59 local active window in both CET and CEST. A first cadence step skips ticks outside the local window, then reads the `updated_at` of the `wazzup-state.zip` asset on the `news-state` release (one API call) and continues only when the persisted state is at least 100 minutes old. Generations therefore land 100–120 minutes apart, the first stale tick after 07:00 produces the morning briefing, and a dropped tick is made up 20 minutes later. If the state is stale but a `News` run failed within the last 55 minutes, the tick backs off so a persistent failure (for example a token rate limit) does not spend AI requests every 20 minutes. If the release cannot be read, the tick generates anyway. Manual dispatch always runs. Skipped ticks finish in seconds and never install tooling. Operational learning: the first live News run failed because Copilot CLI was requested but the token secret was empty. The workflow now selects an effective provider before installing Node/Copilot. If `copilot-cli` is requested without `COPILOT_REQUESTS_PAT` or `COPILOT_GITHUB_TOKEN`, it logs a warning and uses `AI_PROVIDER=fake` so the release state and Pages deployment path can still be validated end to end. @@ -167,6 +167,8 @@ Operational learning: the site went stale for most of a day because GitHub sched Operational learning: a fully generated briefing was lost because `gh release upload news-state` failed with `release not found` although the release existed. `gh` races a published-release REST lookup against a draft-release GraphQL lookup and reports _any_ failure of the REST lookup (a 5xx, abuse limit, or network error) as "release not found" once the draft lookup returns nothing, so a transient API error was indistinguishable from a missing release. `task state:persist` now looks the release up through `gh api` so only a real HTTP 404 creates it, and retries both the lookup and each asset upload with linear backoff (`STATE_PERSIST_ATTEMPTS`, default 4; `STATE_PERSIST_RETRY_DELAY`, default 5 seconds), logging the underlying API response on each failed attempt. +Operational learning: the watchdog alone was not enough. On 2026-11-17 GitHub fired the hourly `News` cron only twice before 14:00 UTC, both on even local hours, and the hourly watchdog only once (overnight), so nothing generated for 17 hours. A recovery path that depends on another hourly cron fails the same way the primary does. The odd-hour gate and the separate `news-watchdog` workflow were replaced by the freshness gate described above: `News` itself ticks three times an hour in the active window and generates whenever the persisted state is stale, so losing most ticks still yields a fresh briefing within 20–40 minutes of the next surviving one. + ### Shared model policy Both local Python providers and Actions fetch **data only** from [DevSecNinja/.github/copilot-models.json](https://raw.githubusercontent.com/DevSecNinja/.github/refs/heads/main/copilot-models.json). There are no production model defaults in Wazzup's Python, workflow environment, or agent frontmatter. @@ -333,7 +335,7 @@ Rules: ## Scheduling details -Use a single hourly cron and calculate the active cadence plus briefing windows in application code or workflow shell using the configured IANA time zone. This handles daylight-saving transitions better than maintaining separate UTC cron expressions. The implemented cadence is every two local hours from 07:00 through 21:59, which is a better fit for the rolling daily briefing and avoids unnecessary AI calls overnight or when feeds are quiet. +Use a single frequent cron and calculate the active window plus briefing windows in application code or workflow shell using the configured IANA time zone. This handles daylight-saving transitions better than maintaining separate UTC cron expressions. The implemented cadence regenerates whenever the persisted state is older than about two hours, checked every 20 minutes from 07:00 through 21:59 local time, which is a better fit for the rolling daily briefing, survives dropped scheduled runs, and avoids unnecessary AI calls overnight or when feeds are quiet. Recommended behavior: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 2c739dfc..81aaeff6 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7,18 +7,24 @@ class WorkflowTests(unittest.TestCase): - def test_news_workflow_uses_active_two_hour_local_cadence_gate(self) -> None: + def test_news_workflow_uses_freshness_gated_cadence(self) -> None: workflow = Path(".github/workflows/news.yml").read_text(encoding="utf-8") self.assertIn("name: News\n", workflow) - self.assertIn('cron: "7 * * * *"', workflow) + # Several ticks per hour so dropped scheduled runs are made up quickly. + self.assertIn('cron: "7,27,47 5-20 * * *"', workflow) self.assertIn("default: auto", workflow) self.assertIn("- auto", workflow) self.assertIn("Check scheduled cadence", workflow) - self.assertIn("hour >= 6 && hour < 22 && hour % 2 == 1", workflow) - self.assertIn("active two-hour local cadence", workflow) - self.assertIn("outside active two-hour local cadence", workflow) - self.assertNotIn("overnight two-hour cadence", workflow) - self.assertNotIn('cron: "d"', workflow) + self.assertIn("hour < 7 || hour >= 22", workflow) + # Wall-clock hour parity skipped real work whenever ticks were dropped + # or jittered; freshness of the persisted state decides instead. + self.assertNotIn("hour % 2 == 1", workflow) + self.assertIn("releases/tags/${STATE_RELEASE}", workflow) + self.assertIn("STATE_ASSET: wazzup-state.zip", workflow) + self.assertIn('CADENCE_INTERVAL_MINUTES: "100"', workflow) + self.assertIn("FAILURE_BACKOFF_MINUTES:", workflow) + self.assertIn("runs?status=failure", workflow) + self.assertFalse(Path(".github/workflows/news-watchdog.yml").exists()) def test_copilot_agents_do_not_override_runtime_model_policy(self) -> None: for agent in ("wazzup-curator", "wazzup-transparency-reporter", "wazzup-writer"): @@ -52,31 +58,6 @@ def test_news_workflow_dispatches_pages_deploy_for_every_trigger(self) -> None: self.assertIn("gh workflow run pages.yml", workflow) self.assertIn("- Pages deployment dispatch: ${PAGES_OUTCOME}", workflow) - def test_news_watchdog_dispatches_when_effective_generation_is_stale(self) -> None: - workflow = Path(".github/workflows/news-watchdog.yml").read_text(encoding="utf-8") - self.assertIn('cron: "37 * * * *"', workflow) - self.assertIn("actions: write", workflow) - self.assertIn("TARGET_WORKFLOW: news.yml", workflow) - # The watchdog runs every hour inside the active window. Gating on - # odd-hour parity would disable it under GitHub schedule jitter, which - # is exactly when catch-up dispatches are needed. - self.assertIn("hour >= 7 && hour < 22", workflow) - self.assertNotIn("hour % 2 == 1", workflow) - # Two-hour cadence is enforced by elapsed time since the last effective - # generation, not by the wall-clock hour of this run. - self.assertIn("CADENCE_INTERVAL_MINUTES:", workflow) - self.assertIn("GENERATE_STEP_NAME: Generate and persist retained news state", workflow) - self.assertIn("last_effective_epoch", workflow) - self.assertIn("threshold_epoch", workflow) - # A cadence skip still produces a News run, so existence alone must not - # suppress catch-up; the generate step result is inspected instead. - self.assertIn("/jobs", workflow) - self.assertNotIn("Existing News runs this scheduled UTC hour", workflow) - self.assertIn("gh workflow run \"${TARGET_WORKFLOW}\"", workflow) - self.assertIn("--repo \"${GITHUB_REPOSITORY}\"", workflow) - self.assertIn("forceBriefing=auto", workflow) - self.assertIn("aiProvider=copilot-cli", workflow) - def test_pages_workflow_runs_after_main_updates(self) -> None: workflow = Path(".github/workflows/pages.yml").read_text(encoding="utf-8") self.assertIn("push:", workflow) From c6f711e22c9e987bd0bfc0304bd3fcb81e65022f Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:56:26 +0100 Subject: [PATCH 2/5] fix(news): measure failure backoff from when the failed run finished Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/news.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/news.yml b/.github/workflows/news.yml index b8461700..7f486d85 100644 --- a/.github/workflows/news.yml +++ b/.github/workflows/news.yml @@ -104,7 +104,7 @@ jobs: fi last_failure_at="$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/news.yml/runs?status=failure&per_page=1" \ - --jq '.workflow_runs[0].created_at // empty' || true)" + --jq '.workflow_runs[0].updated_at // empty' || true)" if [[ -n "${last_failure_at}" ]]; then failure_age_minutes=$(( (now_epoch - $(date -u -d "${last_failure_at}" +%s)) / 60 )) if (( failure_age_minutes < 10#${FAILURE_BACKOFF_MINUTES} )); then From 63cabc3938447816b2254b982e70266ebad5e59b Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:34:25 +0100 Subject: [PATCH 3/5] fix(news): back off after timed-out generations too Include timed_out News runs in the failure backoff and update the documented cron excerpt. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/news.yml | 6 ++++-- docs/github-actions.md | 7 +++++-- tests/test_workflows.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/news.yml b/.github/workflows/news.yml index 7f486d85..4ef26c56 100644 --- a/.github/workflows/news.yml +++ b/.github/workflows/news.yml @@ -103,8 +103,10 @@ jobs: decide false "state is ${state_age_minutes} minutes old, within the two-hour cadence" fi - last_failure_at="$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/news.yml/runs?status=failure&per_page=1" \ - --jq '.workflow_runs[0].updated_at // empty' || true)" + # Timed-out runs (conclusion timed_out) also leave state stale, so + # back off from the most recent failed or timed-out completion. + last_failure_at="$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/news.yml/runs?status=completed&per_page=30" \ + --jq '[.workflow_runs[] | select(.conclusion == "failure" or .conclusion == "timed_out") | .updated_at] | max // empty' || true)" if [[ -n "${last_failure_at}" ]]; then failure_age_minutes=$(( (now_epoch - $(date -u -d "${last_failure_at}" +%s)) / 60 )) if (( failure_age_minutes < 10#${FAILURE_BACKOFF_MINUTES} )); then diff --git a/docs/github-actions.md b/docs/github-actions.md index 361c6874..375dd850 100644 --- a/docs/github-actions.md +++ b/docs/github-actions.md @@ -90,7 +90,10 @@ name: News on: schedule: - - cron: "7 * * * *" + # Every 20 minutes, 05:00-20:59 UTC. The "Check scheduled cadence" step + # skips ticks outside 07:00-21:59 local time and generates only when the + # persisted news-state asset is stale. + - cron: "7,27,47 5-20 * * *" workflow_dispatch: inputs: forceBriefing: @@ -155,7 +158,7 @@ jobs: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_REQUESTS_PAT || secrets.COPILOT_GITHUB_TOKEN }} ``` -GitHub cron is UTC-only and does not understand `Europe/Amsterdam` daylight-saving transitions, and GitHub drops or delays scheduled runs under load. The workflow therefore ticks every 20 minutes from 05:00 to 20:59 UTC (`7,27,47 5-20 * * *`), which covers the 07:00–21:59 local active window in both CET and CEST. A first cadence step skips ticks outside the local window, then reads the `updated_at` of the `wazzup-state.zip` asset on the `news-state` release (one API call) and continues only when the persisted state is at least 100 minutes old. Generations therefore land 100–120 minutes apart, the first stale tick after 07:00 produces the morning briefing, and a dropped tick is made up 20 minutes later. If the state is stale but a `News` run failed within the last 55 minutes, the tick backs off so a persistent failure (for example a token rate limit) does not spend AI requests every 20 minutes. If the release cannot be read, the tick generates anyway. Manual dispatch always runs. Skipped ticks finish in seconds and never install tooling. +GitHub cron is UTC-only and does not understand `Europe/Amsterdam` daylight-saving transitions, and GitHub drops or delays scheduled runs under load. The workflow therefore ticks every 20 minutes from 05:00 to 20:59 UTC (`7,27,47 5-20 * * *`), which covers the 07:00–21:59 local active window in both CET and CEST. A first cadence step skips ticks outside the local window, then reads the `updated_at` of the `wazzup-state.zip` asset on the `news-state` release (one API call) and continues only when the persisted state is at least 100 minutes old. Generations therefore land 100–120 minutes apart, the first stale tick after 07:00 produces the morning briefing, and a dropped tick is made up 20 minutes later. If the state is stale but a `News` run failed or timed out within the last 55 minutes, the tick backs off so a persistent failure (for example a token rate limit) does not spend AI requests every 20 minutes. If the release cannot be read, the tick generates anyway. Manual dispatch always runs. Skipped ticks finish in seconds and never install tooling. Operational learning: the first live News run failed because Copilot CLI was requested but the token secret was empty. The workflow now selects an effective provider before installing Node/Copilot. If `copilot-cli` is requested without `COPILOT_REQUESTS_PAT` or `COPILOT_GITHUB_TOKEN`, it logs a warning and uses `AI_PROVIDER=fake` so the release state and Pages deployment path can still be validated end to end. diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 81aaeff6..3584ba3f 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -23,7 +23,7 @@ def test_news_workflow_uses_freshness_gated_cadence(self) -> None: self.assertIn("STATE_ASSET: wazzup-state.zip", workflow) self.assertIn('CADENCE_INTERVAL_MINUTES: "100"', workflow) self.assertIn("FAILURE_BACKOFF_MINUTES:", workflow) - self.assertIn("runs?status=failure", workflow) + self.assertIn('select(.conclusion == "failure" or .conclusion == "timed_out")', workflow) self.assertFalse(Path(".github/workflows/news-watchdog.yml").exists()) def test_copilot_agents_do_not_override_runtime_model_policy(self) -> None: From c0b560a3793def5d2332a8bfa1cdb0220f3873f3 Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:49:01 +0100 Subject: [PATCH 4/5] fix(news): keep due briefings and failure backoff under the freshness gate Fresh state no longer skips a tick in the morning/evening due window when the persisted state lacks today's due briefing, and missing or unreadable state now bypasses only the freshness check, not the failure backoff. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/news.yml | 52 +++++++++++++++++++++++++++++++------- docs/github-actions.md | 2 +- tests/test_workflows.py | 10 ++++++++ 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/.github/workflows/news.yml b/.github/workflows/news.yml index 4ef26c56..d3281047 100644 --- a/.github/workflows/news.yml +++ b/.github/workflows/news.yml @@ -64,6 +64,11 @@ jobs: # persistent failure (for example a token rate limit) does not burn # AI requests on every tick. FAILURE_BACKOFF_MINUTES: "55" + # Local hours when morning/evening briefings become due; keep in sync + # with config/interests.yml. The pipeline treats each as due for two + # hours, so fresh state must not hide a missing due briefing. + MORNING_LOCAL_HOUR: "7" + EVENING_LOCAL_HOUR: "20" run: |- set -euo pipefail @@ -88,18 +93,47 @@ jobs: # that generated state (scheduled, manual, or a delayed run) counts # and dropped cron runs are made up on the next tick. now_epoch="$(date -u +%s)" + stale_reason="" + state_age_minutes="" if ! state_updated_at="$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${STATE_RELEASE}" \ --jq ".assets[] | select(.name == \"${STATE_ASSET}\") | .updated_at" 2>&1)"; then - echo "::warning::Could not read ${STATE_RELEASE} freshness (${state_updated_at}); generating anyway." - decide true "state freshness unknown" + echo "::warning::Could not read ${STATE_RELEASE} freshness (${state_updated_at}); treating state as stale." + stale_reason="state freshness unknown" + elif [[ -z "${state_updated_at}" ]]; then + stale_reason="no persisted state" + else + state_age_minutes=$(( (now_epoch - $(date -u -d "${state_updated_at}" +%s)) / 60 )) + echo "Persisted state updated at ${state_updated_at} (${state_age_minutes} minutes ago)." + if (( state_age_minutes >= 10#${CADENCE_INTERVAL_MINUTES} )); then + stale_reason="state is ${state_age_minutes} minutes old" + fi fi - if [[ -z "${state_updated_at}" ]]; then - decide true "no persisted state" + + # A run that started just before a due window can leave fresh state + # without that window's briefing, so check the state for it. + if [[ -z "${stale_reason}" ]]; then + due_kind="" + if (( hour >= 10#${MORNING_LOCAL_HOUR} && hour < 10#${MORNING_LOCAL_HOUR} + 2 )); then + due_kind=morning + elif (( hour >= 10#${EVENING_LOCAL_HOUR} && hour < 10#${EVENING_LOCAL_HOUR} + 2 )); then + due_kind=evening + fi + if [[ -n "${due_kind}" ]]; then + briefing="data/briefings/$(TZ="${WAZZUP_TIMEZONE}" date +%Y/%m/%d)/${due_kind}.json" + state_dir="$(mktemp -d)" + if gh release download "${STATE_RELEASE}" --pattern "${STATE_ASSET}" --dir "${state_dir}" --repo "${GITHUB_REPOSITORY}" \ + && listing="$(unzip -Z1 "${state_dir}/${STATE_ASSET}")"; then + if ! grep -qxF "${briefing}" <<<"${listing}"; then + stale_reason="${due_kind} briefing is due but missing from the persisted state" + fi + else + echo "::warning::Could not inspect ${STATE_ASSET} for the due ${due_kind} briefing; keeping the freshness decision." + fi + rm -rf "${state_dir}" + fi fi - state_age_minutes=$(( (now_epoch - $(date -u -d "${state_updated_at}" +%s)) / 60 )) - echo "Persisted state updated at ${state_updated_at} (${state_age_minutes} minutes ago)." - if (( state_age_minutes < 10#${CADENCE_INTERVAL_MINUTES} )); then + if [[ -z "${stale_reason}" ]]; then decide false "state is ${state_age_minutes} minutes old, within the two-hour cadence" fi @@ -110,11 +144,11 @@ jobs: if [[ -n "${last_failure_at}" ]]; then failure_age_minutes=$(( (now_epoch - $(date -u -d "${last_failure_at}" +%s)) / 60 )) if (( failure_age_minutes < 10#${FAILURE_BACKOFF_MINUTES} )); then - decide false "state is stale but a generation failed ${failure_age_minutes} minutes ago; backing off" + decide false "${stale_reason}, but a generation failed ${failure_age_minutes} minutes ago; backing off" fi fi - decide true "state is ${state_age_minutes} minutes old" + decide true "${stale_reason}" - name: Checkout code if: ${{ steps.cadence.outputs.should_run == 'true' }} diff --git a/docs/github-actions.md b/docs/github-actions.md index 375dd850..8b9b22f6 100644 --- a/docs/github-actions.md +++ b/docs/github-actions.md @@ -158,7 +158,7 @@ jobs: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_REQUESTS_PAT || secrets.COPILOT_GITHUB_TOKEN }} ``` -GitHub cron is UTC-only and does not understand `Europe/Amsterdam` daylight-saving transitions, and GitHub drops or delays scheduled runs under load. The workflow therefore ticks every 20 minutes from 05:00 to 20:59 UTC (`7,27,47 5-20 * * *`), which covers the 07:00–21:59 local active window in both CET and CEST. A first cadence step skips ticks outside the local window, then reads the `updated_at` of the `wazzup-state.zip` asset on the `news-state` release (one API call) and continues only when the persisted state is at least 100 minutes old. Generations therefore land 100–120 minutes apart, the first stale tick after 07:00 produces the morning briefing, and a dropped tick is made up 20 minutes later. If the state is stale but a `News` run failed or timed out within the last 55 minutes, the tick backs off so a persistent failure (for example a token rate limit) does not spend AI requests every 20 minutes. If the release cannot be read, the tick generates anyway. Manual dispatch always runs. Skipped ticks finish in seconds and never install tooling. +GitHub cron is UTC-only and does not understand `Europe/Amsterdam` daylight-saving transitions, and GitHub drops or delays scheduled runs under load. The workflow therefore ticks every 20 minutes from 05:00 to 20:59 UTC (`7,27,47 5-20 * * *`), which covers the 07:00–21:59 local active window in both CET and CEST. A first cadence step skips ticks outside the local window, then reads the `updated_at` of the `wazzup-state.zip` asset on the `news-state` release (one API call) and continues only when the persisted state is at least 100 minutes old. Generations therefore land 100–120 minutes apart, the first stale tick after 07:00 produces the morning briefing, and a dropped tick is made up 20 minutes later. During the morning (07:00–08:59) and evening (20:00–21:59) due windows, a tick with fresh state also downloads the state zip and generates if today's due briefing is missing, so a run that started just before a window cannot hide that window's briefing. `MORNING_LOCAL_HOUR` and `EVENING_LOCAL_HOUR` must match `config/interests.yml`, and a test enforces this. A missing or unreadable state asset counts as stale. Before any scheduled generation, if a `News` run failed or timed out within the last 55 minutes, the tick backs off so a persistent failure (for example a token rate limit) does not spend AI requests every 20 minutes. Manual dispatch always runs. Skipped ticks finish in seconds and never install tooling. Operational learning: the first live News run failed because Copilot CLI was requested but the token secret was empty. The workflow now selects an effective provider before installing Node/Copilot. If `copilot-cli` is requested without `COPILOT_REQUESTS_PAT` or `COPILOT_GITHUB_TOKEN`, it logs a warning and uses `AI_PROVIDER=fake` so the release state and Pages deployment path can still be validated end to end. diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 3584ba3f..6163ea3b 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -26,6 +26,16 @@ def test_news_workflow_uses_freshness_gated_cadence(self) -> None: self.assertIn('select(.conclusion == "failure" or .conclusion == "timed_out")', workflow) self.assertFalse(Path(".github/workflows/news-watchdog.yml").exists()) + def test_news_cadence_due_hours_match_briefing_config(self) -> None: + # The gate must not let fresh state hide a due morning/evening briefing, + # so its due hours have to follow the configured briefing times. + workflow = yaml.safe_load(Path(".github/workflows/news.yml").read_text(encoding="utf-8")) + cadence = next(step for step in workflow["jobs"]["generate"]["steps"] if step["name"] == "Check scheduled cadence") + briefings = yaml.safe_load(Path("config/interests.yml").read_text(encoding="utf-8"))["briefings"] + self.assertEqual(int(briefings["morningLocalTime"].split(":")[0]), int(cadence["env"]["MORNING_LOCAL_HOUR"])) + self.assertEqual(int(briefings["eveningLocalTime"].split(":")[0]), int(cadence["env"]["EVENING_LOCAL_HOUR"])) + self.assertIn("briefing is due but missing from the persisted state", cadence["run"]) + def test_copilot_agents_do_not_override_runtime_model_policy(self) -> None: for agent in ("wazzup-curator", "wazzup-transparency-reporter", "wazzup-writer"): with self.subTest(agent=agent): From e999f38732b18f698c4ba9805c78ac32d20449ac Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:37:08 +0100 Subject: [PATCH 5/5] fix(news): pin a detected missing due briefing kind for generation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/news.yml | 7 +++++-- docs/github-actions.md | 2 +- tests/test_workflows.py | 6 ++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/news.yml b/.github/workflows/news.yml index d3281047..72fc74f2 100644 --- a/.github/workflows/news.yml +++ b/.github/workflows/news.yml @@ -125,6 +125,9 @@ jobs: && listing="$(unzip -Z1 "${state_dir}/${STATE_ASSET}")"; then if ! grep -qxF "${briefing}" <<<"${listing}"; then stale_reason="${due_kind} briefing is due but missing from the persisted state" + # Pin the kind so a slow setup that crosses the end of the due + # window still produces this briefing instead of an hourly one. + echo "missing_briefing_kind=${due_kind}" >>"${GITHUB_OUTPUT}" fi else echo "::warning::Could not inspect ${STATE_ASSET} for the due ${due_kind} briefing; keeping the freshness decision." @@ -208,7 +211,7 @@ jobs: GH_TOKEN: ${{ github.token }} AI_PROVIDER: ${{ steps.ai.outputs.provider }} COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_REQUESTS_PAT || secrets.COPILOT_GITHUB_TOKEN }} - FORCE_BRIEFING: ${{ inputs.forceBriefing || 'auto' }} + FORCE_BRIEFING: ${{ inputs.forceBriefing || steps.cadence.outputs.missing_briefing_kind || 'auto' }} run: task news:generate - name: Upload generated public artifact @@ -238,7 +241,7 @@ jobs: - name: Write workflow summary if: ${{ always() && steps.cadence.outputs.should_run == 'true' }} env: - FORCE_BRIEFING: ${{ inputs.forceBriefing || 'auto' }} + FORCE_BRIEFING: ${{ inputs.forceBriefing || steps.cadence.outputs.missing_briefing_kind || 'auto' }} AI_PROVIDER: ${{ steps.ai.outputs.provider }} GENERATION_OUTCOME: ${{ steps.generation.outcome }} PAGES_OUTCOME: ${{ steps.pages.outcome }} diff --git a/docs/github-actions.md b/docs/github-actions.md index 8b9b22f6..ec81fa03 100644 --- a/docs/github-actions.md +++ b/docs/github-actions.md @@ -158,7 +158,7 @@ jobs: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_REQUESTS_PAT || secrets.COPILOT_GITHUB_TOKEN }} ``` -GitHub cron is UTC-only and does not understand `Europe/Amsterdam` daylight-saving transitions, and GitHub drops or delays scheduled runs under load. The workflow therefore ticks every 20 minutes from 05:00 to 20:59 UTC (`7,27,47 5-20 * * *`), which covers the 07:00–21:59 local active window in both CET and CEST. A first cadence step skips ticks outside the local window, then reads the `updated_at` of the `wazzup-state.zip` asset on the `news-state` release (one API call) and continues only when the persisted state is at least 100 minutes old. Generations therefore land 100–120 minutes apart, the first stale tick after 07:00 produces the morning briefing, and a dropped tick is made up 20 minutes later. During the morning (07:00–08:59) and evening (20:00–21:59) due windows, a tick with fresh state also downloads the state zip and generates if today's due briefing is missing, so a run that started just before a window cannot hide that window's briefing. `MORNING_LOCAL_HOUR` and `EVENING_LOCAL_HOUR` must match `config/interests.yml`, and a test enforces this. A missing or unreadable state asset counts as stale. Before any scheduled generation, if a `News` run failed or timed out within the last 55 minutes, the tick backs off so a persistent failure (for example a token rate limit) does not spend AI requests every 20 minutes. Manual dispatch always runs. Skipped ticks finish in seconds and never install tooling. +GitHub cron is UTC-only and does not understand `Europe/Amsterdam` daylight-saving transitions, and GitHub drops or delays scheduled runs under load. The workflow therefore ticks every 20 minutes from 05:00 to 20:59 UTC (`7,27,47 5-20 * * *`), which covers the 07:00–21:59 local active window in both CET and CEST. A first cadence step skips ticks outside the local window, then reads the `updated_at` of the `wazzup-state.zip` asset on the `news-state` release (one API call) and continues only when the persisted state is at least 100 minutes old. Generations therefore land 100–120 minutes apart, the first stale tick after 07:00 produces the morning briefing, and a dropped tick is made up 20 minutes later. During the morning (07:00–08:59) and evening (20:00–21:59) due windows, a tick with fresh state also downloads the state zip and generates if today's due briefing is missing, so a run that started just before a window cannot hide that window's briefing. That tick passes the detected kind to the pipeline as `FORCE_BRIEFING`, so slow setup that crosses the end of the window still produces it. `MORNING_LOCAL_HOUR` and `EVENING_LOCAL_HOUR` must match `config/interests.yml`, and a test enforces this. A missing or unreadable state asset counts as stale. Before any scheduled generation, if a `News` run failed or timed out within the last 55 minutes, the tick backs off so a persistent failure (for example a token rate limit) does not spend AI requests every 20 minutes. Manual dispatch always runs. Skipped ticks finish in seconds and never install tooling. Operational learning: the first live News run failed because Copilot CLI was requested but the token secret was empty. The workflow now selects an effective provider before installing Node/Copilot. If `copilot-cli` is requested without `COPILOT_REQUESTS_PAT` or `COPILOT_GITHUB_TOKEN`, it logs a warning and uses `AI_PROVIDER=fake` so the release state and Pages deployment path can still be validated end to end. diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 6163ea3b..9bb510c7 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -35,6 +35,12 @@ def test_news_cadence_due_hours_match_briefing_config(self) -> None: self.assertEqual(int(briefings["morningLocalTime"].split(":")[0]), int(cadence["env"]["MORNING_LOCAL_HOUR"])) self.assertEqual(int(briefings["eveningLocalTime"].split(":")[0]), int(cadence["env"]["EVENING_LOCAL_HOUR"])) self.assertIn("briefing is due but missing from the persisted state", cadence["run"]) + self.assertIn('echo "missing_briefing_kind=${due_kind}" >>"${GITHUB_OUTPUT}"', cadence["run"]) + generation = next(step for step in workflow["jobs"]["generate"]["steps"] if step["name"] == "Generate and persist retained news state") + self.assertEqual( + "${{ inputs.forceBriefing || steps.cadence.outputs.missing_briefing_kind || 'auto' }}", + generation["env"]["FORCE_BRIEFING"], + ) def test_copilot_agents_do_not_override_runtime_model_policy(self) -> None: for agent in ("wazzup-curator", "wazzup-transparency-reporter", "wazzup-writer"):