test(perf): add manual-only extended coverage and DuckDB baseline

.github/workflows/scenario-dev.yml

@@ -8,6 +8,12 @@ on:
         required: true
         default: full-suite
         type: string
+      coverage_target:
+        description: Extended target; all runs each configuration in a separate job (manual only)
+        required: false
+        default: all
+        type: choice
+        options: [all, pgwire_uncached, pgwire_cached, trino, trino_cached, athena]
       properties_s3_uri:
         description: Properties Parquet S3 prefix override; blank uses the existing Athena table location
         required: false
@@ -35,6 +41,51 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  plan:
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.targets.outputs.matrix }}
+      trino_image: ${{ steps.trino.outputs.image }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - name: Select scenario targets
+        id: targets
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+        run: |
+          python3 - <<'PYTHON'
+          import json, os
+          with open(os.environ["GITHUB_EVENT_PATH"]) as event_file:
+              inputs = json.load(event_file).get("inputs") or {}
+          scenario = "posthog_frozen_perf" if os.environ["EVENT_NAME"] == "schedule" else inputs.get("scenario", "")
+          targets = ["pgwire_uncached", "pgwire_cached", "trino", "trino_cached", "athena"]
+          if scenario == "posthog_frozen_perf_extended":
+              selected = inputs.get("coverage_target") or "all"
+              if selected not in ["all", *targets]:
+                  raise SystemExit("Unsupported extended coverage target")
+              jobs = [{"target": target, "id_suffix": str(i + 1), "artifact_suffix": "-" + target}
+                      for i, target in enumerate(targets) if selected in ("all", target)]
+          else:
+              jobs = [{"target": "pgwire_uncached", "id_suffix": "", "artifact_suffix": ""}]
+          needs_trino = scenario == "posthog_frozen_perf" or (
+              scenario == "posthog_frozen_perf_extended" and any(j["target"].startswith("trino") for j in jobs))
+          if needs_trino and inputs.get("duckgres_image"):
+              raise SystemExit("Trino comparisons require the control-plane image built from this revision; omit duckgres_image")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              print("matrix=" + json.dumps({"include": jobs}, separators=(",", ":")), file=output)
+              print("needs_trino=" + str(needs_trino).lower(), file=output)
+          PYTHON
+      - name: Resolve shared Trino image
+        id: trino
+        if: steps.targets.outputs.needs_trino == 'true'
+        env:
+          TRINO_IMAGE_OVERRIDE: ${{ inputs.trino_image }}
+        run: |
+          set -euo pipefail
+          image="${TRINO_IMAGE_OVERRIDE:-$(scripts/resolve_trino_master_image.sh)}"
+          echo "image=$image" >> "$GITHUB_OUTPUT"
+          echo "Trino image: \`$image\`" >> "$GITHUB_STEP_SUMMARY"
+
   scenario-runner-image:
     uses: ./.github/workflows/_image-build.yml
     with:
@@ -65,10 +116,16 @@ jobs:
       ecr-role: ${{ vars.AWS_ECR_PRS_PUBLISH_IAM_ROLE }}
 
   scenario:
-    needs: [scenario-runner-image, duckgres-image]
-    if: ${{ always() && needs.scenario-runner-image.result == 'success' && (needs.duckgres-image.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.duckgres_image != '')) }}
+    name: ${{ github.event_name == 'schedule' && 'posthog_frozen_perf' || inputs.scenario }}${{ matrix.artifact_suffix }}
+    needs: [plan, scenario-runner-image, duckgres-image]
+    if: ${{ always() && needs.plan.result == 'success' && needs.scenario-runner-image.result == 'success' && (needs.duckgres-image.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.duckgres_image != '')) }}
     runs-on: ubuntu-24.04
     timeout-minutes: 270
+    strategy:
+      fail-fast: false
+      # Sequential targets avoid benchmark resource contention. Each gets a fresh timeout.
+      max-parallel: 1
+      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
     env:
       SCENARIO_NAME: ${{ github.event_name == 'schedule' && 'posthog_frozen_perf' || inputs.scenario }}
       SCENARIO_RUNNER_IMAGE: ${{ needs.scenario-runner-image.outputs.image }}
@@ -82,35 +139,23 @@ jobs:
       # Frozen perf uses a namespace-local Trino cell. The dedicated role is
       # consumed only when E2E_SUITE selects that isolated deployment.
       TRINO_POD_IDENTITY_ROLE: ${{ secrets.MW_DEV_TRINO_POD_IDENTITY_ROLE }}
-      E2E_SUITE: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf') && 'trino' || 'neutral' }}
-      PR_NUMBER: ${{ github.run_id }}
-      NAMESPACE: duckgres-ci-pr-${{ github.run_id }}
+      E2E_SUITE: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || (inputs.scenario == 'posthog_frozen_perf_extended' && (matrix.target == 'trino' || matrix.target == 'trino_cached'))) && 'trino' || 'neutral' }}
+      DUCKGRES_SCENARIO_COVERAGE_TARGET: ${{ matrix.target }}
+      PR_NUMBER: ${{ github.run_id }}${{ matrix.id_suffix }}
+      NAMESPACE: duckgres-ci-pr-${{ github.run_id }}${{ matrix.id_suffix }}
+      TRINO_IMAGE: ${{ needs.plan.outputs.trino_image }}
       DUCKGRES_SCENARIO_MAX_RUNTIME: 4h
       DUCKGRES_SCENARIO_GO_TEST_TIMEOUT: 4h15m
-      # Add process headroom for repeated full-dataset pgwire aggregates.
-      DUCKGRES_K8S_WORKER_CPU_REQUEST: "3"
-      DUCKGRES_K8S_WORKER_MEMORY_REQUEST: 12Gi
+      # Match the benchmark Trino fleet: three 7-CPU / 28Gi workers.
+      # Non-benchmark scenarios retain their existing small resource budget.
+      DUCKGRES_K8S_WORKER_CPU_REQUEST: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '21' || '3' }}
+      DUCKGRES_K8S_WORKER_MEMORY_REQUEST: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '84Gi' || '12Gi' }}
+      # The perf driver passes explicit sizing GUCs: keep their admission caps aligned.
+      DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '21' || '8' }}
+      DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '84Gi' || '16Gi' }}
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
-      - name: Require a current image for cached comparisons
-        if: ${{ inputs.duckgres_image != '' && inputs.scenario == 'posthog_frozen_perf' }}
-        run: |
-          echo "::error::Cached Trino comparisons require the control-plane image built from this revision. Omit duckgres_image so an older image cannot silently ignore the cache flag."
-          exit 1
-
-      - name: Resolve Trino image
-        if: env.E2E_SUITE == 'trino'
-        env:
-          TRINO_IMAGE_OVERRIDE: ${{ inputs.trino_image }}
-        run: |
-          set -euo pipefail
-          # Frozen perf follows PostHog/trino master; resolve once so every
-          # run.sh step deploys and reports the same digest.
-          image="${TRINO_IMAGE_OVERRIDE:-$(scripts/resolve_trino_master_image.sh)}"
-          echo "TRINO_IMAGE=$image" >> "$GITHUB_ENV"
-          echo "Trino image: \`$image\`" >> "$GITHUB_STEP_SUMMARY"
-
       - name: Scenario dev target selected
         run: echo "Deploying an isolated Duckgres stack for $SCENARIO_NAME."
 
@@ -149,7 +194,7 @@ jobs:
         run: aws eks update-kubeconfig --name "$CLUSTER_NAME" --region "$AWS_REGION" --alias "$KUBE_CONTEXT"
 
       - name: Load frozen perf identity and Athena configuration
-        if: env.SCENARIO_NAME == 'posthog_frozen_perf'
+        if: env.SCENARIO_NAME == 'posthog_frozen_perf' || env.SCENARIO_NAME == 'posthog_frozen_perf_extended' || env.SCENARIO_NAME == 'posthog_frozen_perf_extended_uncached'
         run: bash scripts/scenario_athena_config.sh >> "$GITHUB_ENV"
 
       - name: Load properties fixture override
@@ -199,7 +244,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: scenario-dev-${{ env.SCENARIO_NAME }}-${{ github.run_id }}-${{ github.run_attempt }}
+          name: scenario-dev-${{ env.SCENARIO_NAME }}-${{ github.run_id }}-${{ github.run_attempt }}${{ matrix.artifact_suffix }}
           path: artifacts/scenario-dev/
           if-no-files-found: warn
           retention-days: 14
@@ -212,7 +257,7 @@ jobs:
         run: |
           set -euo pipefail
           mapfile -d '' perf_summaries < <(
-            find artifacts/scenario-dev -type f \( -path '*/perf/summary.json' -o -path '*/perf-properties/summary.json' \) -print0 2>/dev/null | sort -z
+            find artifacts/scenario-dev -type f \( -path '*/perf/summary.json' -o -path '*/perf-properties/summary.json' -o -path '*/perf-coverage/summary.json' \) -print0 2>/dev/null | sort -z
           )
           if [ "${#perf_summaries[@]}" -eq 0 ]; then
             echo "No scenario perf artifact was produced; skipping historical publish."

AGENTS.md

@@ -16,6 +16,7 @@
 
 ##  Testing
 
+- Perftest query and coverage additions are exempt from the TDD requirement: do not add tests solely to test perftests. Only make minimal compatibility updates to existing tests when needed.
 - Tests should model realistic user behavior, not work around current service limitations.
 - If a test exposes a platform bug, report the bug with exact failure details instead of fixing the test.
 

docs/runbooks/perf-coverage.md

@@ -0,0 +1,175 @@
+# Extended workload coverage benchmark
+
+The workload coverage catalog supplements the original frozen table and
+properties comparisons. It measures filtered unique users, grouped trends,
+joins, event browsing, actor retrieval, a funnel, sampled-week retention,
+behavioral cohorts, and read-side modeling queries. Modeling queries measure
+the SELECT computation, not writes or a complete materialization job.
+
+## Run
+
+The regular `posthog_frozen_perf` nightly retains the original table and
+properties workloads. The twelve additional queries are an **extended, manual-only
+run**: scheduled nightly runs never include them. Start them explicitly through
+`workflow_dispatch`. This command runs all five configurations:
+
+```sh
+gh workflow run scenario-dev.yml --ref <branch> \
+  -f scenario=posthog_frozen_perf_extended
+```
+
+`coverage_target` defaults to `all`. To select one configuration, add
+`-f coverage_target=pgwire_uncached`, `pgwire_cached`, `trino`, `trino_cached`,
+or `athena`. Each selected configuration gets its own job, isolated deployment,
+and artifact. Jobs execute sequentially to avoid benchmark resource contention;
+a failed target does not cancel the others. Images are built once, and both
+Trino configurations use the same resolved image digest.
+
+Trino targets select the declarative `posthog_frozen_perf_extended_trino.yaml`
+backing scenario and deploy the Trino lane. Other targets use the neutral lane.
+Both reuse the existing frozen setup; Athena configuration also supplies the
+frozen runner identity.
+
+### Timeouts
+
+Each target has a four-hour scenario limit and a 270-minute workflow job limit.
+The Go process limit is 4h15m, leaving time for artifact collection and cleanup.
+These limits apply independently to each job, not to the sum of all target jobs.
+A single target can still time out; peer runtimes have not yet been measured.
+These runtime estimates use the historical small-worker profile, not the larger
+profile below. The measured uncached DuckDB coverage phase took 87m06s.
+
+Running all five targets sequentially inside one job would likely time out:
+the equal-speed query-work estimate alone is 7h16m. The matrix avoids that
+combined timeout without reducing queries or repetitions. Under the same
+planning assumptions and shared initial builds, allow roughly **8h16m for the
+extended workflow alone**, excluding queue delays. Standard nightly queries
+are not included; running them as well brings the estimate to about **9h19m**.
+
+For the focused uncached DuckDB baseline:
+
+```sh
+gh workflow run scenario-dev.yml --ref <branch> \
+  -f scenario=posthog_frozen_perf_extended_uncached
+```
+
+Each target job executes all twelve queries with one warmup and four
+measured repetitions, sequentially, using the same frozen files.
+
+### Worker sizing
+
+Standard frozen perf and extended perf use the following execution budgets:
+
+| Engine | Workers | CPU per worker | RAM per worker | JVM heap | Query memory per worker |
+|---|---:|---:|---:|---:|---:|
+| Trino (each cache mode) | 3 | 7 | 28 GiB | 20 GiB | 10 GiB |
+| DuckDB (each cache mode) | 1 | 21 | 84 GiB | N/A | Derived by Duckgres from pod memory |
+
+Both have **21 CPU / 84 GiB of worker resources**, with requests equal to
+limits. This compares aggregate execution resources; Trino coordinator and
+support services are additional overhead. Athena capacity is service-managed.
+Trino's cluster-wide query memory cap is 30 GiB. Its 20 GiB heap leaves 8 GiB
+outside the JVM, and its 10 GiB per-node query cap leaves room for JVM headroom.
+Worker CPU, RAM, heap and per-node query memory match the configured dev
+worker size. The cluster-wide cap scales to the requested three-worker fleet;
+this is a benchmark profile, not a copy of the complete dev deployment.
+
+DuckDB's client sizing caps are raised alongside its worker resources so the
+explicit benchmark profile is not silently clamped. At 84 GiB, the current
+Duckgres headroom policy sets DuckDB's memory limit to 63 GiB; engine memory
+limits differ even though container resources match. Other scenario types and
+the auxiliary Trino bootstrap cluster retain their small defaults.
+
+The benchmark node pool must admit nodes large enough for a 21-CPU / 84Gi
+DuckDB pod and 7-CPU / 28Gi Trino pods, with additional allocatable capacity for
+system services. Deploy the [node-size allowlist change](https://github.com/PostHog/charts/pull/16434)
+before running this profile. If pods remain Pending, inspect scheduling events and
+node-pool constraints rather than counting setup failure as query latency.
+
+The retained September 25 baseline used the old 3 CPU / 12 GiB aggregate
+profile. Keep it as historical data; it is **not a like-for-like baseline** for
+these larger workers. Runtime estimates above also describe the old profile;
+measure a fresh run before setting a new baseline or runtime forecast.
+
+"Uncached" disables the DuckDB external file cache; it does not guarantee cold
+OS or storage caches. See
+[cache settings](../../tests/perf/README.md#duckdb-cache-comparison).
+
+Target jobs have distinct benchmark run and nightly IDs. Group an all-target
+campaign by its parent workflow URL and target-suffixed artifacts. Compare
+matching recorded commits, catalog versions, dataset versions, resource settings,
+and methodology. Across separate workflow dispatches, pin the same `trino_image`
+digest when comparing Trino configurations.
+
+Artifacts live in `perf-coverage/`, with a `-coverage` run ID suffix. The
+original `perf/` and `perf-properties/` histories are preserved. Main-branch
+runs publish coverage separately; PR runs retain downloadable artifacts.
+
+## Dataset limits and validation
+
+The frozen events are sampled days, not a continuous event history. Coverage
+queries use populated fixture windows. In particular, retention measures a
+return between sampled weeks; it is a performance workload, not a production
+retention estimate. Do not interpret absent days as zero user activity.
+Person enrichment uses tenant and person keys and handles duplicate person
+records. Fixture checks must reject empty populations rather than silently
+turning joins or funnels into cheap empty-result benchmarks.
+
+## Record a comparison baseline
+
+The initial uncached DuckDB comparison base is the
+[2026-09-25 coverage v1 baseline](../../tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/README.md).
+It retains 48 successful measurements and the complete methodology. The query
+phase took 87m06s; the workflow took 1h42m45s. The equal-speed expanded campaign
+estimate recorded with that baseline was 9h37m across separate dispatches plus
+the original suite. The current shared-build matrix lowers that planning estimate
+to about 9h19m; this is not a new measured result.
+
+
+Keep the successful focused run URL, tested commit, catalog version/hash,
+dataset version, worker resources, cache settings, and repetition counts
+alongside sanitized per-query timing samples. Do not commit query result
+values, fixture locations, credentials, tenant identifiers, or raw run logs.
+GitHub artifacts expire, so retained timing samples must be sufficient to
+recalculate each query's mean, median, and observed range.
+
+Compare future runs by matching intent ID, protocol, fixture, resources, and
+methodology. A query or window change needs a new versioned intent ID and a
+new baseline. Report per-query ratios as well as the sum; a summed ratio is
+weighted toward slow queries. Four repetitions do not establish tail latency.
+The uncached DuckDB baseline is measured evidence only for that configuration;
+it is not a measured Trino, cached DuckDB, or Athena baseline.
+
+## Estimate the expanded full suite
+
+For each added query, use its measured uncached mean as the initial planning
+estimate for the warmup and four measured executions. With twelve queries and
+five configurations, the equal-speed estimate is:
+
+```text
+additional query time = sum(uncached means) * 5 repetitions * 5 configurations
+full campaign estimate = original workflow time + additional query time
+                         + shared build time + sum(non-build overhead for each target job)
+```
+
+Report this explicitly as an equal-speed estimate, not a forecast of engine
+performance. Include a sensitivity range for the other four configurations.
+The workflow builds images once; each target job repeats deployment, fixture
+validation, collection, and teardown. Estimate those separately from query work.
+For the initial baseline, the workflow's initial build phase was approximately
+272 seconds and total time outside the query phase was 939.50 seconds. Using
+`272 + 5 * (939.50 - 272)` for shared-build overhead gives the approximate
+8h16m extended-only estimate above. This assumes other engines' setup and warmup
+costs match the focused DuckDB run; actual costs can differ.
+Use the measured coverage phase wall time to check the sum-based estimate,
+because warmups and harness overhead are not in `query_results.csv`.
+
+## Failure recovery
+
+Keep failed artifacts and query errors. An empty fixture population, SQL
+compatibility failure, timeout, or platform error invalidates that query's
+baseline; never record a failure as zero latency or change assertions to make
+a failing platform look successful. Fix the cause and dispatch a fresh isolated
+run. The workflow always tears down its stack; if cancellation interrupts
+cleanup, follow the existing scenario runner cleanup runbook. Do not patch a
+running benchmark or reuse a partially configured worker for the baseline.