diff --git a/.github/workflows/scenario-dev.yml b/.github/workflows/scenario-dev.yml index f1db8dcae..2c4088911 100644 --- a/.github/workflows/scenario-dev.yml +++ b/.github/workflows/scenario-dev.yml @@ -8,6 +8,12 @@ on: required: true default: full-suite type: string + coverage_target: + description: Extended target; all runs each configuration in a separate job (manual only) + required: false + default: all + type: choice + options: [all, pgwire_uncached, pgwire_cached, trino, trino_cached, athena] properties_s3_uri: description: Properties Parquet S3 prefix override; blank uses the existing Athena table location required: false @@ -35,6 +41,51 @@ concurrency: cancel-in-progress: false jobs: + plan: + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.targets.outputs.matrix }} + trino_image: ${{ steps.trino.outputs.image }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Select scenario targets + id: targets + env: + EVENT_NAME: ${{ github.event_name }} + run: | + python3 - <<'PYTHON' + import json, os + with open(os.environ["GITHUB_EVENT_PATH"]) as event_file: + inputs = json.load(event_file).get("inputs") or {} + scenario = "posthog_frozen_perf" if os.environ["EVENT_NAME"] == "schedule" else inputs.get("scenario", "") + targets = ["pgwire_uncached", "pgwire_cached", "trino", "trino_cached", "athena"] + if scenario == "posthog_frozen_perf_extended": + selected = inputs.get("coverage_target") or "all" + if selected not in ["all", *targets]: + raise SystemExit("Unsupported extended coverage target") + jobs = [{"target": target, "id_suffix": str(i + 1), "artifact_suffix": "-" + target} + for i, target in enumerate(targets) if selected in ("all", target)] + else: + jobs = [{"target": "pgwire_uncached", "id_suffix": "", "artifact_suffix": ""}] + needs_trino = scenario == "posthog_frozen_perf" or ( + scenario == "posthog_frozen_perf_extended" and any(j["target"].startswith("trino") for j in jobs)) + if needs_trino and inputs.get("duckgres_image"): + raise SystemExit("Trino comparisons require the control-plane image built from this revision; omit duckgres_image") + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + print("matrix=" + json.dumps({"include": jobs}, separators=(",", ":")), file=output) + print("needs_trino=" + str(needs_trino).lower(), file=output) + PYTHON + - name: Resolve shared Trino image + id: trino + if: steps.targets.outputs.needs_trino == 'true' + env: + TRINO_IMAGE_OVERRIDE: ${{ inputs.trino_image }} + run: | + set -euo pipefail + image="${TRINO_IMAGE_OVERRIDE:-$(scripts/resolve_trino_master_image.sh)}" + echo "image=$image" >> "$GITHUB_OUTPUT" + echo "Trino image: \`$image\`" >> "$GITHUB_STEP_SUMMARY" + scenario-runner-image: uses: ./.github/workflows/_image-build.yml with: @@ -65,10 +116,16 @@ jobs: ecr-role: ${{ vars.AWS_ECR_PRS_PUBLISH_IAM_ROLE }} scenario: - needs: [scenario-runner-image, duckgres-image] - if: ${{ always() && needs.scenario-runner-image.result == 'success' && (needs.duckgres-image.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.duckgres_image != '')) }} + name: ${{ github.event_name == 'schedule' && 'posthog_frozen_perf' || inputs.scenario }}${{ matrix.artifact_suffix }} + needs: [plan, scenario-runner-image, duckgres-image] + if: ${{ always() && needs.plan.result == 'success' && needs.scenario-runner-image.result == 'success' && (needs.duckgres-image.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.duckgres_image != '')) }} runs-on: ubuntu-24.04 timeout-minutes: 270 + strategy: + fail-fast: false + # Sequential targets avoid benchmark resource contention. Each gets a fresh timeout. + max-parallel: 1 + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} env: SCENARIO_NAME: ${{ github.event_name == 'schedule' && 'posthog_frozen_perf' || inputs.scenario }} SCENARIO_RUNNER_IMAGE: ${{ needs.scenario-runner-image.outputs.image }} @@ -82,35 +139,23 @@ jobs: # Frozen perf uses a namespace-local Trino cell. The dedicated role is # consumed only when E2E_SUITE selects that isolated deployment. TRINO_POD_IDENTITY_ROLE: ${{ secrets.MW_DEV_TRINO_POD_IDENTITY_ROLE }} - E2E_SUITE: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf') && 'trino' || 'neutral' }} - PR_NUMBER: ${{ github.run_id }} - NAMESPACE: duckgres-ci-pr-${{ github.run_id }} + E2E_SUITE: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || (inputs.scenario == 'posthog_frozen_perf_extended' && (matrix.target == 'trino' || matrix.target == 'trino_cached'))) && 'trino' || 'neutral' }} + DUCKGRES_SCENARIO_COVERAGE_TARGET: ${{ matrix.target }} + PR_NUMBER: ${{ github.run_id }}${{ matrix.id_suffix }} + NAMESPACE: duckgres-ci-pr-${{ github.run_id }}${{ matrix.id_suffix }} + TRINO_IMAGE: ${{ needs.plan.outputs.trino_image }} DUCKGRES_SCENARIO_MAX_RUNTIME: 4h DUCKGRES_SCENARIO_GO_TEST_TIMEOUT: 4h15m - # Add process headroom for repeated full-dataset pgwire aggregates. - DUCKGRES_K8S_WORKER_CPU_REQUEST: "3" - DUCKGRES_K8S_WORKER_MEMORY_REQUEST: 12Gi + # Match the benchmark Trino fleet: three 7-CPU / 28Gi workers. + # Non-benchmark scenarios retain their existing small resource budget. + DUCKGRES_K8S_WORKER_CPU_REQUEST: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '21' || '3' }} + DUCKGRES_K8S_WORKER_MEMORY_REQUEST: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '84Gi' || '12Gi' }} + # The perf driver passes explicit sizing GUCs: keep their admission caps aligned. + DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '21' || '8' }} + DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf' || inputs.scenario == 'posthog_frozen_perf_extended' || inputs.scenario == 'posthog_frozen_perf_extended_uncached') && '84Gi' || '16Gi' }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Require a current image for cached comparisons - if: ${{ inputs.duckgres_image != '' && inputs.scenario == 'posthog_frozen_perf' }} - run: | - echo "::error::Cached Trino comparisons require the control-plane image built from this revision. Omit duckgres_image so an older image cannot silently ignore the cache flag." - exit 1 - - - name: Resolve Trino image - if: env.E2E_SUITE == 'trino' - env: - TRINO_IMAGE_OVERRIDE: ${{ inputs.trino_image }} - run: | - set -euo pipefail - # Frozen perf follows PostHog/trino master; resolve once so every - # run.sh step deploys and reports the same digest. - image="${TRINO_IMAGE_OVERRIDE:-$(scripts/resolve_trino_master_image.sh)}" - echo "TRINO_IMAGE=$image" >> "$GITHUB_ENV" - echo "Trino image: \`$image\`" >> "$GITHUB_STEP_SUMMARY" - - name: Scenario dev target selected run: echo "Deploying an isolated Duckgres stack for $SCENARIO_NAME." @@ -149,7 +194,7 @@ jobs: run: aws eks update-kubeconfig --name "$CLUSTER_NAME" --region "$AWS_REGION" --alias "$KUBE_CONTEXT" - name: Load frozen perf identity and Athena configuration - if: env.SCENARIO_NAME == 'posthog_frozen_perf' + if: env.SCENARIO_NAME == 'posthog_frozen_perf' || env.SCENARIO_NAME == 'posthog_frozen_perf_extended' || env.SCENARIO_NAME == 'posthog_frozen_perf_extended_uncached' run: bash scripts/scenario_athena_config.sh >> "$GITHUB_ENV" - name: Load properties fixture override @@ -199,7 +244,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: scenario-dev-${{ env.SCENARIO_NAME }}-${{ github.run_id }}-${{ github.run_attempt }} + name: scenario-dev-${{ env.SCENARIO_NAME }}-${{ github.run_id }}-${{ github.run_attempt }}${{ matrix.artifact_suffix }} path: artifacts/scenario-dev/ if-no-files-found: warn retention-days: 14 @@ -212,7 +257,7 @@ jobs: run: | set -euo pipefail mapfile -d '' perf_summaries < <( - find artifacts/scenario-dev -type f \( -path '*/perf/summary.json' -o -path '*/perf-properties/summary.json' \) -print0 2>/dev/null | sort -z + find artifacts/scenario-dev -type f \( -path '*/perf/summary.json' -o -path '*/perf-properties/summary.json' -o -path '*/perf-coverage/summary.json' \) -print0 2>/dev/null | sort -z ) if [ "${#perf_summaries[@]}" -eq 0 ]; then echo "No scenario perf artifact was produced; skipping historical publish." diff --git a/AGENTS.md b/AGENTS.md index f779ee1ff..bd396aaa2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,7 @@ ## Testing +- Perftest query and coverage additions are exempt from the TDD requirement: do not add tests solely to test perftests. Only make minimal compatibility updates to existing tests when needed. - Tests should model realistic user behavior, not work around current service limitations. - If a test exposes a platform bug, report the bug with exact failure details instead of fixing the test. diff --git a/docs/runbooks/perf-coverage.md b/docs/runbooks/perf-coverage.md new file mode 100644 index 000000000..fa2a34653 --- /dev/null +++ b/docs/runbooks/perf-coverage.md @@ -0,0 +1,175 @@ +# Extended workload coverage benchmark + +The workload coverage catalog supplements the original frozen table and +properties comparisons. It measures filtered unique users, grouped trends, +joins, event browsing, actor retrieval, a funnel, sampled-week retention, +behavioral cohorts, and read-side modeling queries. Modeling queries measure +the SELECT computation, not writes or a complete materialization job. + +## Run + +The regular `posthog_frozen_perf` nightly retains the original table and +properties workloads. The twelve additional queries are an **extended, manual-only +run**: scheduled nightly runs never include them. Start them explicitly through +`workflow_dispatch`. This command runs all five configurations: + +```sh +gh workflow run scenario-dev.yml --ref \ + -f scenario=posthog_frozen_perf_extended +``` + +`coverage_target` defaults to `all`. To select one configuration, add +`-f coverage_target=pgwire_uncached`, `pgwire_cached`, `trino`, `trino_cached`, +or `athena`. Each selected configuration gets its own job, isolated deployment, +and artifact. Jobs execute sequentially to avoid benchmark resource contention; +a failed target does not cancel the others. Images are built once, and both +Trino configurations use the same resolved image digest. + +Trino targets select the declarative `posthog_frozen_perf_extended_trino.yaml` +backing scenario and deploy the Trino lane. Other targets use the neutral lane. +Both reuse the existing frozen setup; Athena configuration also supplies the +frozen runner identity. + +### Timeouts + +Each target has a four-hour scenario limit and a 270-minute workflow job limit. +The Go process limit is 4h15m, leaving time for artifact collection and cleanup. +These limits apply independently to each job, not to the sum of all target jobs. +A single target can still time out; peer runtimes have not yet been measured. +These runtime estimates use the historical small-worker profile, not the larger +profile below. The measured uncached DuckDB coverage phase took 87m06s. + +Running all five targets sequentially inside one job would likely time out: +the equal-speed query-work estimate alone is 7h16m. The matrix avoids that +combined timeout without reducing queries or repetitions. Under the same +planning assumptions and shared initial builds, allow roughly **8h16m for the +extended workflow alone**, excluding queue delays. Standard nightly queries +are not included; running them as well brings the estimate to about **9h19m**. + +For the focused uncached DuckDB baseline: + +```sh +gh workflow run scenario-dev.yml --ref \ + -f scenario=posthog_frozen_perf_extended_uncached +``` + +Each target job executes all twelve queries with one warmup and four +measured repetitions, sequentially, using the same frozen files. + +### Worker sizing + +Standard frozen perf and extended perf use the following execution budgets: + +| Engine | Workers | CPU per worker | RAM per worker | JVM heap | Query memory per worker | +|---|---:|---:|---:|---:|---:| +| Trino (each cache mode) | 3 | 7 | 28 GiB | 20 GiB | 10 GiB | +| DuckDB (each cache mode) | 1 | 21 | 84 GiB | N/A | Derived by Duckgres from pod memory | + +Both have **21 CPU / 84 GiB of worker resources**, with requests equal to +limits. This compares aggregate execution resources; Trino coordinator and +support services are additional overhead. Athena capacity is service-managed. +Trino's cluster-wide query memory cap is 30 GiB. Its 20 GiB heap leaves 8 GiB +outside the JVM, and its 10 GiB per-node query cap leaves room for JVM headroom. +Worker CPU, RAM, heap and per-node query memory match the configured dev +worker size. The cluster-wide cap scales to the requested three-worker fleet; +this is a benchmark profile, not a copy of the complete dev deployment. + +DuckDB's client sizing caps are raised alongside its worker resources so the +explicit benchmark profile is not silently clamped. At 84 GiB, the current +Duckgres headroom policy sets DuckDB's memory limit to 63 GiB; engine memory +limits differ even though container resources match. Other scenario types and +the auxiliary Trino bootstrap cluster retain their small defaults. + +The benchmark node pool must admit nodes large enough for a 21-CPU / 84Gi +DuckDB pod and 7-CPU / 28Gi Trino pods, with additional allocatable capacity for +system services. Deploy the [node-size allowlist change](https://github.com/PostHog/charts/pull/16434) +before running this profile. If pods remain Pending, inspect scheduling events and +node-pool constraints rather than counting setup failure as query latency. + +The retained September 25 baseline used the old 3 CPU / 12 GiB aggregate +profile. Keep it as historical data; it is **not a like-for-like baseline** for +these larger workers. Runtime estimates above also describe the old profile; +measure a fresh run before setting a new baseline or runtime forecast. + +"Uncached" disables the DuckDB external file cache; it does not guarantee cold +OS or storage caches. See +[cache settings](../../tests/perf/README.md#duckdb-cache-comparison). + +Target jobs have distinct benchmark run and nightly IDs. Group an all-target +campaign by its parent workflow URL and target-suffixed artifacts. Compare +matching recorded commits, catalog versions, dataset versions, resource settings, +and methodology. Across separate workflow dispatches, pin the same `trino_image` +digest when comparing Trino configurations. + +Artifacts live in `perf-coverage/`, with a `-coverage` run ID suffix. The +original `perf/` and `perf-properties/` histories are preserved. Main-branch +runs publish coverage separately; PR runs retain downloadable artifacts. + +## Dataset limits and validation + +The frozen events are sampled days, not a continuous event history. Coverage +queries use populated fixture windows. In particular, retention measures a +return between sampled weeks; it is a performance workload, not a production +retention estimate. Do not interpret absent days as zero user activity. +Person enrichment uses tenant and person keys and handles duplicate person +records. Fixture checks must reject empty populations rather than silently +turning joins or funnels into cheap empty-result benchmarks. + +## Record a comparison baseline + +The initial uncached DuckDB comparison base is the +[2026-09-25 coverage v1 baseline](../../tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/README.md). +It retains 48 successful measurements and the complete methodology. The query +phase took 87m06s; the workflow took 1h42m45s. The equal-speed expanded campaign +estimate recorded with that baseline was 9h37m across separate dispatches plus +the original suite. The current shared-build matrix lowers that planning estimate +to about 9h19m; this is not a new measured result. + + +Keep the successful focused run URL, tested commit, catalog version/hash, +dataset version, worker resources, cache settings, and repetition counts +alongside sanitized per-query timing samples. Do not commit query result +values, fixture locations, credentials, tenant identifiers, or raw run logs. +GitHub artifacts expire, so retained timing samples must be sufficient to +recalculate each query's mean, median, and observed range. + +Compare future runs by matching intent ID, protocol, fixture, resources, and +methodology. A query or window change needs a new versioned intent ID and a +new baseline. Report per-query ratios as well as the sum; a summed ratio is +weighted toward slow queries. Four repetitions do not establish tail latency. +The uncached DuckDB baseline is measured evidence only for that configuration; +it is not a measured Trino, cached DuckDB, or Athena baseline. + +## Estimate the expanded full suite + +For each added query, use its measured uncached mean as the initial planning +estimate for the warmup and four measured executions. With twelve queries and +five configurations, the equal-speed estimate is: + +```text +additional query time = sum(uncached means) * 5 repetitions * 5 configurations +full campaign estimate = original workflow time + additional query time + + shared build time + sum(non-build overhead for each target job) +``` + +Report this explicitly as an equal-speed estimate, not a forecast of engine +performance. Include a sensitivity range for the other four configurations. +The workflow builds images once; each target job repeats deployment, fixture +validation, collection, and teardown. Estimate those separately from query work. +For the initial baseline, the workflow's initial build phase was approximately +272 seconds and total time outside the query phase was 939.50 seconds. Using +`272 + 5 * (939.50 - 272)` for shared-build overhead gives the approximate +8h16m extended-only estimate above. This assumes other engines' setup and warmup +costs match the focused DuckDB run; actual costs can differ. +Use the measured coverage phase wall time to check the sum-based estimate, +because warmups and harness overhead are not in `query_results.csv`. + +## Failure recovery + +Keep failed artifacts and query errors. An empty fixture population, SQL +compatibility failure, timeout, or platform error invalidates that query's +baseline; never record a failure as zero latency or change assertions to make +a failing platform look successful. Fix the cause and dispatch a fresh isolated +run. The workflow always tears down its stack; if cancellation interrupts +cleanup, follow the existing scenario runner cleanup runbook. Do not patch a +running benchmark or reuse a partially configured worker for the baseline. diff --git a/tests/mw-dev/manifests.tmpl.yaml b/tests/mw-dev/manifests.tmpl.yaml index eef280e28..8c86a33b2 100644 --- a/tests/mw-dev/manifests.tmpl.yaml +++ b/tests/mw-dev/manifests.tmpl.yaml @@ -363,9 +363,9 @@ spec: - { name: DUCKGRES_EXPLORATORY_WORKER_MEMORY, value: "2Gi" } - { name: DUCKGRES_EXPLORATORY_WORKER_TTL, value: "10m" } - { name: DUCKGRES_K8S_WORKER_PROFILE_MIN_CPU, value: "1" } - - { name: DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU, value: "8" } + - { name: DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU, value: "${DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU}" } - { name: DUCKGRES_K8S_WORKER_PROFILE_MIN_MEMORY, value: "2Gi" } - - { name: DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY, value: "16Gi" } + - { name: DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY, value: "${DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY}" } # Prod-logic parity (sizes differ, the knobs match prod-us): # - workerQueueTimeout bounds how long a connection waits for its # on-demand cold spawn (binary default 60s is too tight for a diff --git a/tests/mw-dev/manifests.trino.tmpl.yaml b/tests/mw-dev/manifests.trino.tmpl.yaml index 0b91caf8f..b571933bc 100644 --- a/tests/mw-dev/manifests.trino.tmpl.yaml +++ b/tests/mw-dev/manifests.trino.tmpl.yaml @@ -90,9 +90,9 @@ data: http-server.https.keystore.path=/etc/trino/tls/keystore.p12 http-server.https.keystore.key=${ENV:TRINO_TLS_KEYSTORE_PASSWORD} discovery.uri=http://duckgres-trino.${NAMESPACE}.svc:8080 - query.max-memory=6GB + query.max-memory=${TRINO_QUERY_MEMORY} # The coordinator does not execute tasks and retains headroom in its 2G - # heap; execution workers carry the 2GB per-node allowance below. + # heap; execution workers use the selected profile below. query.max-memory-per-node=1GB catalog.management=dynamic # Both targets load the same managers; catalog fs.cache.enabled gates use. @@ -134,7 +134,7 @@ data: node.data-dir=/data/trino jvm.config: | -server - -Xmx3G + -Xmx${TRINO_WORKER_HEAP} -XX:+UseG1GC -XX:+ExitOnOutOfMemoryError -Dfile.encoding=UTF-8 @@ -150,12 +150,11 @@ data: coordinator=false http-server.http.port=8080 discovery.uri=http://duckgres-trino.${NAMESPACE}.svc:8080 - query.max-memory=6GB - query.max-memory-per-node=2GB - # 1-CPU workers: raised from 8/16 to overlap S3 latency (keep task.concurrency default). - # Experiment measured by the targeted frozen perf run; revert if events by-name uncached does not improve. - task.max-worker-threads=24 - task.min-drivers=48 + query.max-memory=${TRINO_QUERY_MEMORY} + query.max-memory-per-node=${TRINO_WORKER_QUERY_MEMORY} + # Worker parallelism follows the rendering profile; task.concurrency stays default. + task.max-worker-threads=${TRINO_WORKER_THREADS} + task.min-drivers=${TRINO_WORKER_MIN_DRIVERS} shutdown.grace-period=10s catalog.management=dynamic # Both targets load the same managers; catalog fs.cache.enabled gates use. @@ -319,11 +318,9 @@ spec: initialDelaySeconds: 20 periodSeconds: 5 resources: - requests: { cpu: "1", memory: 4Gi } - # Three replicas provide the same aggregate 3-CPU/12Gi execution - # budget as the frozen-perf Duckgres worker. Keep requests equal - # to limits so node contention cannot change the comparison. - limits: { cpu: "1", memory: 4Gi } + requests: { cpu: "${TRINO_WORKER_CPU}", memory: "${TRINO_WORKER_MEMORY}" } + # Requests equal limits for the selected three-worker profile. + limits: { cpu: "${TRINO_WORKER_CPU}", memory: "${TRINO_WORKER_MEMORY}" } volumeMounts: - { name: config, mountPath: /etc/trino/node.properties, subPath: node.properties } - { name: config, mountPath: /etc/trino/jvm.config, subPath: jvm.config } diff --git a/tests/mw-dev/run.sh b/tests/mw-dev/run.sh index 3051249ed..b077d4a4d 100755 --- a/tests/mw-dev/run.sh +++ b/tests/mw-dev/run.sh @@ -30,6 +30,8 @@ SCENARIO_NAME="${SCENARIO_NAME:-full-suite}" SCENARIO_ARTIFACTS_DIR="${SCENARIO_ARTIFACTS_DIR:-$HERE/../../artifacts/scenario-dev}" DUCKGRES_K8S_WORKER_CPU_REQUEST="${DUCKGRES_K8S_WORKER_CPU_REQUEST:-750m}" DUCKGRES_K8S_WORKER_MEMORY_REQUEST="${DUCKGRES_K8S_WORKER_MEMORY_REQUEST:-1536Mi}" +DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU="${DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU:-8}" +DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY="${DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY:-16Gi}" E2E_SUITE="${E2E_SUITE:-neutral}" case "$E2E_SUITE" in neutral|duckdb|trino|reshard) ;; @@ -63,7 +65,7 @@ esac # it to a digest pin when TRINO_IMAGE is unset. Regular Trino tests keep a # promoted pin because they require atomic writes. TRINO_MASTER_IMAGE_RESOLVER="${TRINO_MASTER_IMAGE_RESOLVER:-$HERE/../../scripts/resolve_trino_master_image.sh}" -if [ "$SCENARIO_NAME" = posthog_frozen_perf ]; then +if [ "$SCENARIO_NAME" = posthog_frozen_perf ] || [ "$SCENARIO_NAME" = posthog_frozen_perf_extended ]; then TRINO_IMAGE="${TRINO_IMAGE:-}" else TRINO_IMAGE="${TRINO_IMAGE:-ghcr.io/posthog/trino:86468a7955788b90fe2072f80d86d548972ff28b@sha256:64927a71d2870802a56b671828c6052e7aa37317a7c3a50bd50a93960402d67b}" @@ -130,7 +132,7 @@ require_pr_identity() { } frozen_perf_scenario() { - [ "$SCENARIO_NAME" = posthog_frozen_perf ] + [ "$SCENARIO_NAME" = posthog_frozen_perf ] || [ "$SCENARIO_NAME" = posthog_frozen_perf_extended ] || [ "$SCENARIO_NAME" = posthog_frozen_perf_extended_uncached ] } hoglake_perf_enabled() { @@ -141,6 +143,33 @@ trino_multicell_enabled() { [ "$E2E_SUITE" = trino ] && [ "$SCENARIO_NAME" = full-suite ] } +# Bootstrap and general E2E keep the small fleet. Only comparison clusters +# use dev-sized workers; both cache modes use the same profile. +render_trino_template() { + local TRINO_WORKER_CPU=1 TRINO_WORKER_MEMORY=4Gi TRINO_WORKER_HEAP=3G + local TRINO_QUERY_MEMORY=6GB TRINO_WORKER_QUERY_MEMORY=2GB + local TRINO_WORKER_THREADS=24 TRINO_WORKER_MIN_DRIVERS=48 + if [ "${1:-}" = perf ]; then + TRINO_WORKER_CPU=7 TRINO_WORKER_MEMORY=28Gi TRINO_WORKER_HEAP=20G + TRINO_QUERY_MEMORY=30GB TRINO_WORKER_QUERY_MEMORY=10GB + TRINO_WORKER_THREADS=14 TRINO_WORKER_MIN_DRIVERS=28 + fi + export TRINO_WORKER_CPU TRINO_WORKER_MEMORY TRINO_WORKER_HEAP + export TRINO_QUERY_MEMORY TRINO_WORKER_QUERY_MEMORY TRINO_WORKER_THREADS TRINO_WORKER_MIN_DRIVERS + envsubst '$NAMESPACE $PR_NUMBER $TRINO_IMAGE $TRINO_TLS_PASSWORD $TRINO_CA_CERT_B64 $TRINO_SERVER_P12_B64 $CONFIG_STORE_PASSWORD $TRINO_WORKER_CPU $TRINO_WORKER_MEMORY $TRINO_WORKER_HEAP $TRINO_QUERY_MEMORY $TRINO_WORKER_QUERY_MEMORY $TRINO_WORKER_THREADS $TRINO_WORKER_MIN_DRIVERS' < "$HERE/manifests.trino.tmpl.yaml" +} + +trino_perf_modes() { + if [ "$SCENARIO_NAME" = posthog_frozen_perf_extended ]; then + case "${DUCKGRES_SCENARIO_COVERAGE_TARGET:-}" in + trino) echo perf ;; + trino_cached) echo cached ;; + esac + else + echo 'perf cached' + fi +} + render_trino_backend() { local color="$1" local forwarded_config="" @@ -152,8 +181,7 @@ render_trino_backend() { TRINO_IMAGE="$TRINO_IMAGE" TRINO_TLS_PASSWORD="$TRINO_TLS_PASSWORD" \ CONFIG_STORE_PASSWORD="$(cat "$config_store_password_file")" \ NAMESPACE="$TRINO_CELL_NS" PR_NUMBER="$PR_NUMBER" \ - envsubst '$NAMESPACE $PR_NUMBER $TRINO_IMAGE $TRINO_TLS_PASSWORD $TRINO_CA_CERT_B64 $TRINO_SERVER_P12_B64 $CONFIG_STORE_PASSWORD' \ - < "$HERE/manifests.trino.tmpl.yaml" \ + render_trino_template \ | sed -e "s/duckgres-trino-coordinator/duckgres-trino-$color-coordinator/g" \ -e "s/duckgres-trino-worker/duckgres-trino-$color-worker/g" \ -e "s/labels: { app: duckgres-trino, component: coordinator }/labels: { app: duckgres-trino, component: coordinator, posthog.com\/trino-cell: cell-test, posthog.com\/trino-color: $color, app.kubernetes.io\/component: coordinator, app.kubernetes.io\/name: trino-coordinator }/" \ @@ -170,7 +198,7 @@ render_trino_backend() { -e "$forwarded_config" } -# Reuse the baseline template so both clusters retain identical resource budgets. +# Reuse the template with the larger comparison-worker profile for both modes. # Select only cluster-specific resources; TLS, auth, tenant secrets, OPA and the # config-store connection are intentionally shared in the throwaway namespace. render_trino_perf_cluster() { @@ -188,7 +216,7 @@ stringData: shared-secret: "$(cat "$cluster_secret_file")" EOF TRINO_IMAGE="$TRINO_IMAGE" NAMESPACE="$NS" PR_NUMBER="$PR_NUMBER" \ - envsubst '$NAMESPACE $PR_NUMBER $TRINO_IMAGE' < "$HERE/manifests.trino.tmpl.yaml" \ + render_trino_template perf \ | awk ' /^---$/ { if (selected) printf "%s", document; document="---\n"; selected=0; next } { document=document $0 "\n" } @@ -277,9 +305,11 @@ render() { USER_SECRET_KEY="$(cat "$user_secret_key_file")" \ NAMESPACE="$NS" PR_NUMBER="$PR_NUMBER" \ WORKER_IMAGE="$WORKER_IMAGE" CONTROLPLANE_IMAGE="$CONTROLPLANE_IMAGE" \ + DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU="$DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU" \ + DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY="$DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY" \ DUCKGRES_K8S_WORKER_CPU_REQUEST="$DUCKGRES_K8S_WORKER_CPU_REQUEST" \ DUCKGRES_K8S_WORKER_MEMORY_REQUEST="$DUCKGRES_K8S_WORKER_MEMORY_REQUEST" \ - envsubst '$NAMESPACE $PR_NUMBER $WORKER_IMAGE $CONTROLPLANE_IMAGE $INTERNAL_SECRET $INTERNAL_SECRET_FALLBACK $USER_SECRET_KEY $DUCKGRES_K8S_WORKER_CPU_REQUEST $DUCKGRES_K8S_WORKER_MEMORY_REQUEST $CONFIG_STORE_PASSWORD' \ + envsubst '$NAMESPACE $PR_NUMBER $WORKER_IMAGE $CONTROLPLANE_IMAGE $INTERNAL_SECRET $INTERNAL_SECRET_FALLBACK $USER_SECRET_KEY $DUCKGRES_K8S_WORKER_CPU_REQUEST $DUCKGRES_K8S_WORKER_MEMORY_REQUEST $DUCKGRES_K8S_WORKER_PROFILE_MAX_CPU $DUCKGRES_K8S_WORKER_PROFILE_MAX_MEMORY $CONFIG_STORE_PASSWORD' \ < "$HERE/manifests.tmpl.yaml" if [ "$E2E_SUITE" = "trino" ]; then @@ -289,11 +319,10 @@ render() { TRINO_IMAGE="$TRINO_IMAGE" TRINO_TLS_PASSWORD="$TRINO_TLS_PASSWORD" \ CONFIG_STORE_PASSWORD="$(cat "$config_store_password_file")" \ NAMESPACE="$NS" PR_NUMBER="$PR_NUMBER" \ - envsubst '$NAMESPACE $PR_NUMBER $TRINO_IMAGE $TRINO_TLS_PASSWORD $TRINO_CA_CERT_B64 $TRINO_SERVER_P12_B64 $CONFIG_STORE_PASSWORD' \ - < "$HERE/manifests.trino.tmpl.yaml" + render_trino_template if hoglake_perf_enabled; then - render_trino_perf_cluster perf - render_trino_perf_cluster cached + local mode + for mode in $(trino_perf_modes); do render_trino_perf_cluster "$mode"; done fi HOGLAKE_SERVICE_ACCOUNT=hoglake # Server default (15 min). Frozen perf registers footer-only fixtures and @@ -622,9 +651,8 @@ cmd_deploy() { # Patch the Deployment resources directly. The CI deployer intentionally # cannot patch the deployments/scale subresource, while it already needs # narrowly scoped Deployment patch access for the control-plane config. - # Three 1-CPU/4Gi workers match the frozen-perf Duckgres worker's - # aggregate 3-CPU/12Gi execution budget while exercising Trino's - # distributed path. + # The bootstrap cluster keeps three small workers for catalog admission. + # Comparison clusters below use their own larger worker profile. "${KUBECTL[@]}" -n "$NS" patch deployment duckgres-trino-coordinator \ --type=merge -p '{"spec":{"replicas":1}}' "${KUBECTL[@]}" -n "$NS" patch deployment duckgres-trino-worker \ @@ -632,7 +660,7 @@ cmd_deploy() { "${KUBECTL[@]}" -n "$NS" rollout status deploy/duckgres-trino-coordinator --timeout=300s "${KUBECTL[@]}" -n "$NS" rollout status deploy/duckgres-trino-worker --timeout=300s if hoglake_perf_enabled; then - for mode in perf cached; do + for mode in $(trino_perf_modes); do "${KUBECTL[@]}" -n "$NS" patch deployment "duckgres-trino-$mode-coordinator" --type=merge -p '{"spec":{"replicas":1}}' "${KUBECTL[@]}" -n "$NS" patch deployment "duckgres-trino-$mode-worker" --type=merge -p '{"spec":{"replicas":3}}' "${KUBECTL[@]}" -n "$NS" rollout status "deploy/duckgres-trino-$mode-coordinator" --timeout=300s @@ -773,7 +801,9 @@ scenario_name_for_file() { scenario_job_name() { local name="$1" run_hash run_hash="$(printf '%s' "${DUCKGRES_SCENARIO_RUN_ID:?DUCKGRES_SCENARIO_RUN_ID is required}" | cksum | awk '{print $1}')" - printf 'duckgres-scenario-%s-%s\n' "$name" "$run_hash" | tr '_' '-' + # Kubernetes copies the Job name into a label, whose limit is 63 bytes. + # Reserve 18 for the prefix, one separator, and ten for the cksum value. + printf 'duckgres-scenario-%s-%s\n' "${name:0:34}" "$run_hash" | tr '_' '-' } cmd_test_scenario() { @@ -787,8 +817,16 @@ cmd_test_scenario() { ;; esac - scenario_file="tests/mw-dev/scenario/scenarios/${SCENARIO_NAME}.yaml" - if [ ! -f "$HERE/scenario/scenarios/${SCENARIO_NAME}.yaml" ]; then + local backing_scenario="$SCENARIO_NAME" + if [ "$SCENARIO_NAME" = posthog_frozen_perf_extended ]; then + case "${DUCKGRES_SCENARIO_COVERAGE_TARGET:-}" in + pgwire_uncached|pgwire_cached|athena) ;; + trino|trino_cached) backing_scenario=posthog_frozen_perf_extended_trino ;; + *) echo "DUCKGRES_SCENARIO_COVERAGE_TARGET must select one coverage protocol" >&2; return 2 ;; + esac + fi + scenario_file="tests/mw-dev/scenario/scenarios/${backing_scenario}.yaml" + if [ ! -f "$HERE/scenario/scenarios/${backing_scenario}.yaml" ]; then echo "Scenario '$SCENARIO_NAME' does not exist at $scenario_file." >&2 return 2 fi @@ -800,6 +838,10 @@ cmd_test_scenario() { run_scenario() { local scenario_name="$1" scenario_file="$2" job pod api_base pg suffix internal_secret artifact_rc=0 container_rc=0 container_exit_code scenario_rc=0 + local pg_connect_timeout=10 + # Sized benchmark workers may cold-start during the PostgreSQL handshake. + # Allow the server's five-minute worker queue budget plus startup headroom. + if frozen_perf_scenario; then pg_connect_timeout=360; fi api_base="http://duckgres-control-plane.$NS.svc:8080" pg="$("${KUBECTL[@]}" -n "$NS" get svc duckgres-control-plane -o jsonpath='{.spec.clusterIP}')" suffix=".ci.duckgres.local" @@ -864,6 +906,8 @@ spec: - { name: DUCKGRES_SCENARIO_ORG_ID, value: "ci-pr-${PR_NUMBER}-cnpg" } - { name: DUCKGRES_SCENARIO_OUTPUT_BASE, value: "/artifacts/scenario-dev" } - { name: DUCKGRES_SCENARIO_RUN_ID, value: "$DUCKGRES_SCENARIO_RUN_ID" } + - { name: DUCKGRES_SCENARIO_COVERAGE_TARGET, value: "${DUCKGRES_SCENARIO_COVERAGE_TARGET:-}" } + - { name: DUCKGRES_SCENARIO_PG_CONNECT_TIMEOUT, value: "${DUCKGRES_SCENARIO_PG_CONNECT_TIMEOUT:-$pg_connect_timeout}" } - { name: DUCKGRES_SCENARIO_MAX_RUNTIME, value: "${DUCKGRES_SCENARIO_MAX_RUNTIME:-4h}" } - { name: DUCKGRES_SCENARIO_GO_TEST_TIMEOUT, value: "${DUCKGRES_SCENARIO_GO_TEST_TIMEOUT:-4h15m}" } - { name: GOCACHE, value: "/tmp/go-cache" } @@ -1099,7 +1143,7 @@ cmd_diagnostics() { "${KUBECTL[@]}" -n "$NS" logs deploy/duckgres-trino-coordinator -c duckgres-trino-opa --tail=300 || true "${KUBECTL[@]}" -n "$NS" logs deploy/duckgres-trino-worker -c trino-worker --tail=300 || true if hoglake_perf_enabled; then - for mode in perf cached; do + for mode in $(trino_perf_modes); do "${KUBECTL[@]}" -n "$NS" logs "deploy/duckgres-trino-$mode-coordinator" -c trino-coordinator --tail=300 || true "${KUBECTL[@]}" -n "$NS" logs "deploy/duckgres-trino-$mode-coordinator" -c duckgres-trino-opa --tail=300 || true "${KUBECTL[@]}" -n "$NS" logs "deploy/duckgres-trino-$mode-worker" -c trino-worker --tail=300 || true diff --git a/tests/mw-dev/run_sh_test.go b/tests/mw-dev/run_sh_test.go index 001e4916e..1e2c2ff46 100644 --- a/tests/mw-dev/run_sh_test.go +++ b/tests/mw-dev/run_sh_test.go @@ -168,6 +168,13 @@ func TestTrinoWorkersMatchDuckgresAggregateCompute(t *testing.T) { "${TRINO_TLS_PASSWORD}", "test-password", "${TRINO_CA_CERT_B64}", "dGVzdA==", "${TRINO_SERVER_P12_B64}", "dGVzdA==", + "${TRINO_WORKER_CPU}", "1", + "${TRINO_WORKER_MEMORY}", "4Gi", + "${TRINO_WORKER_HEAP}", "3G", + "${TRINO_QUERY_MEMORY}", "6GB", + "${TRINO_WORKER_QUERY_MEMORY}", "2GB", + "${TRINO_WORKER_THREADS}", "24", + "${TRINO_WORKER_MIN_DRIVERS}", "48", ).Replace(string(raw)) decoder := utilyaml.NewYAMLOrJSONDecoder(strings.NewReader(rendered), 4096) diff --git a/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_extended.yaml b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_extended.yaml new file mode 100644 index 000000000..a3c0f0f57 --- /dev/null +++ b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_extended.yaml @@ -0,0 +1,96 @@ +name: posthog-frozen-perf-extended +run_id_prefix: scenario-frozen-extended +required_env: + - DUCKGRES_SCENARIO_COVERAGE_TARGET + - DUCKGRES_SCENARIO_ATHENA_REGION + - DUCKGRES_SCENARIO_ATHENA_WORKGROUP + - DUCKGRES_SCENARIO_ATHENA_DATABASE + - DUCKGRES_SCENARIO_ATHENA_RESULTS_S3_URI + - DUCKGRES_SCENARIO_ORG_ID + - DUCKGRES_SCENARIO_FROZEN_S3_URI + - DUCKGRES_K8S_WORKER_CPU_REQUEST + - DUCKGRES_K8S_WORKER_MEMORY_REQUEST +steps: + - id: provision + type: provision_warehouse + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + request: + database_name: scenario-frozen-extended-${run_id_token} + team_id: 1 + metadata_store: + type: cnpg-shard + ducklake: + enabled: true + data_store: + type: s3bucket + + - id: wait_ready + type: wait_warehouse_ready + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + timeout: 15m + poll_interval: 10s + + - id: setup_frozen_views + type: sql + depends_on: [wait_ready] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/setup_frozen_views.sql + max_attempts: 12 + retry_interval: 10s + + - id: setup_posthog_tables + type: sql + depends_on: [setup_frozen_views] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/setup_posthog_tables.sql + max_attempts: 3 + retry_interval: 10s + + - id: validate_posthog_tables + type: sql + depends_on: [setup_posthog_tables] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/validate_posthog_tables.sql + max_attempts: 3 + retry_interval: 10s + + - id: validate_coverage + type: sql + depends_on: [validate_posthog_tables] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/validate_posthog_coverage.sql + max_attempts: 1 + + - id: coverage_queries + type: perf_queries + depends_on: [validate_coverage] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + catalog_file: ../../../perf/queries/ducklake_posthog_coverage.yaml + targets: ["${env:DUCKGRES_SCENARIO_COVERAGE_TARGET}"] + athena_region: ${env:DUCKGRES_SCENARIO_ATHENA_REGION} + athena_workgroup: ${env:DUCKGRES_SCENARIO_ATHENA_WORKGROUP} + athena_catalog: AwsDataCatalog + athena_database: ${env:DUCKGRES_SCENARIO_ATHENA_DATABASE} + athena_output_location: ${env:DUCKGRES_SCENARIO_ATHENA_RESULTS_S3_URI} + athena_poll_interval: 500ms + athena_query_timeout: 30m + run_id: ${run_id}-coverage + nightly_run_id: ${run_id} + output_subdir: perf-coverage + suite: coverage + dataset_version: posthog-file-views-v1 + fail_on_query_errors: true + worker_cpu: ${env:DUCKGRES_K8S_WORKER_CPU_REQUEST} + worker_memory: ${env:DUCKGRES_K8S_WORKER_MEMORY_REQUEST} diff --git a/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_extended_trino.yaml b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_extended_trino.yaml new file mode 100644 index 000000000..4eb48a6aa --- /dev/null +++ b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_extended_trino.yaml @@ -0,0 +1,131 @@ +name: posthog-frozen-perf-extended-trino +run_id_prefix: scenario-frozen-extended-trino +required_env: + - DUCKGRES_SCENARIO_COVERAGE_TARGET + - DUCKGRES_SCENARIO_ORG_ID + - DUCKGRES_SCENARIO_HOGLAKE_URI + - DUCKGRES_SCENARIO_FROZEN_S3_URI + - DUCKGRES_SCENARIO_TRINO_CA_CERT + - DUCKGRES_SCENARIO_TRINO_CATALOG_STORE_CELL_ID + - DUCKGRES_SCENARIO_TRINO_CATALOG_STORE_DSN + - DUCKGRES_SCENARIO_TRINO_PERF_URL + - DUCKGRES_SCENARIO_TRINO_PERF_CELL_ID + - DUCKGRES_SCENARIO_TRINO_CACHED_URL + - DUCKGRES_SCENARIO_TRINO_CACHED_CELL_ID + - DUCKGRES_SCENARIO_TRINO_ADMIN_PASSWORD_FILE + - DUCKGRES_SCENARIO_ATHENA_REGION + - DUCKGRES_SCENARIO_ATHENA_WORKGROUP + - DUCKGRES_SCENARIO_ATHENA_DATABASE + - DUCKGRES_SCENARIO_ATHENA_RESULTS_S3_URI + - DUCKGRES_K8S_WORKER_CPU_REQUEST + - DUCKGRES_K8S_WORKER_MEMORY_REQUEST +steps: + - id: provision + type: provision_warehouse + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + request: + database_name: scenario-frozen-perf-${run_id_token} + team_id: 1 + metadata_store: + type: cnpg-shard + ducklake: + enabled: true + data_store: + type: s3bucket + trino: + enabled: true + tier: free + + - id: wait_ready + type: wait_warehouse_ready + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + timeout: 15m + poll_interval: 10s + + - id: wait_trino_ready + type: wait_trino_ready + depends_on: [wait_ready] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + timeout: 15m + poll_interval: 10s + + - id: setup_frozen_views + type: sql + depends_on: [wait_ready] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/setup_frozen_views.sql + max_attempts: 12 + retry_interval: 10s + + - id: setup_posthog_tables + type: sql + depends_on: [setup_frozen_views] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/setup_posthog_tables.sql + max_attempts: 3 + retry_interval: 10s + + - id: validate_posthog_tables + type: sql + depends_on: [setup_posthog_tables] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/validate_posthog_tables.sql + max_attempts: 3 + retry_interval: 10s + + - id: setup_hoglake + type: setup_hoglake + depends_on: [validate_posthog_tables] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + uri: ${env:DUCKGRES_SCENARIO_HOGLAKE_URI} + source: ${env:DUCKGRES_SCENARIO_FROZEN_S3_URI} + hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}-frozen + file: ../perf/setup_hoglake.py + + - id: validate_coverage + type: sql + depends_on: [setup_hoglake, wait_trino_ready] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/validate_posthog_coverage.sql + max_attempts: 1 + + - id: coverage_queries + type: perf_queries + depends_on: [validate_coverage] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + catalog_file: ../../../perf/queries/ducklake_posthog_coverage.yaml + targets: ["${env:DUCKGRES_SCENARIO_COVERAGE_TARGET}"] + trino_hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}-frozen + trino_ca_cert_file: ${env:DUCKGRES_SCENARIO_TRINO_CA_CERT} + trino_startup_timeout: 2m + trino_startup_poll_interval: 2s + athena_region: ${env:DUCKGRES_SCENARIO_ATHENA_REGION} + athena_workgroup: ${env:DUCKGRES_SCENARIO_ATHENA_WORKGROUP} + athena_catalog: AwsDataCatalog + athena_database: ${env:DUCKGRES_SCENARIO_ATHENA_DATABASE} + athena_output_location: ${env:DUCKGRES_SCENARIO_ATHENA_RESULTS_S3_URI} + athena_poll_interval: 500ms + athena_query_timeout: 30m + run_id: ${run_id}-coverage + nightly_run_id: ${run_id} + output_subdir: perf-coverage + suite: coverage + dataset_version: posthog-file-views-v1 + fail_on_query_errors: true + worker_cpu: ${env:DUCKGRES_K8S_WORKER_CPU_REQUEST} + worker_memory: ${env:DUCKGRES_K8S_WORKER_MEMORY_REQUEST} + diff --git a/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_extended_uncached.yaml b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_extended_uncached.yaml new file mode 100644 index 000000000..aee9b90e5 --- /dev/null +++ b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf_extended_uncached.yaml @@ -0,0 +1,84 @@ +name: posthog-frozen-perf-extended-uncached +run_id_prefix: scenario-frozen-extended-uncached +required_env: + - DUCKGRES_SCENARIO_ORG_ID + - DUCKGRES_SCENARIO_FROZEN_S3_URI + - DUCKGRES_K8S_WORKER_CPU_REQUEST + - DUCKGRES_K8S_WORKER_MEMORY_REQUEST +steps: + - id: provision + type: provision_warehouse + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + request: + database_name: scenario-frozen-extended-uncached-${run_id_token} + team_id: 1 + metadata_store: + type: cnpg-shard + ducklake: + enabled: true + data_store: + type: s3bucket + + - id: wait_ready + type: wait_warehouse_ready + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + timeout: 15m + poll_interval: 10s + + - id: setup_frozen_views + type: sql + depends_on: [wait_ready] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/setup_frozen_views.sql + max_attempts: 12 + retry_interval: 10s + + - id: setup_posthog_tables + type: sql + depends_on: [setup_frozen_views] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/setup_posthog_tables.sql + max_attempts: 3 + retry_interval: 10s + + - id: validate_posthog_tables + type: sql + depends_on: [setup_posthog_tables] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/validate_posthog_tables.sql + max_attempts: 3 + retry_interval: 10s + + - id: validate_coverage + type: sql + depends_on: [validate_posthog_tables] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + file: ../sql/validate_posthog_coverage.sql + max_attempts: 1 + + - id: coverage_queries + type: perf_queries + depends_on: [validate_coverage] + with: + org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} + catalog: ducklake + catalog_file: ../../../perf/queries/ducklake_posthog_coverage.yaml + targets: [pgwire_uncached] + run_id: ${run_id}-coverage + nightly_run_id: ${run_id} + output_subdir: perf-coverage + suite: coverage + dataset_version: posthog-file-views-v1 + fail_on_query_errors: true + worker_cpu: ${env:DUCKGRES_K8S_WORKER_CPU_REQUEST} + worker_memory: ${env:DUCKGRES_K8S_WORKER_MEMORY_REQUEST} diff --git a/tests/mw-dev/scenario/script_test.go b/tests/mw-dev/scenario/script_test.go index 63e45b169..ead1dde86 100644 --- a/tests/mw-dev/scenario/script_test.go +++ b/tests/mw-dev/scenario/script_test.go @@ -76,8 +76,8 @@ func TestDevScenarioWorkflowPreservesIsolationCleanupAndPublishing(t *testing.T) workflow := string(raw) for _, required := range []string{ - "PR_NUMBER: ${{ github.run_id }}", - "NAMESPACE: duckgres-ci-pr-${{ github.run_id }}", + "PR_NUMBER: ${{ github.run_id }}${{ matrix.id_suffix }}", + "NAMESPACE: duckgres-ci-pr-${{ github.run_id }}${{ matrix.id_suffix }}", "- name: Teardown\n if: always()\n run: tests/mw-dev/run.sh teardown", "github.ref == 'refs/heads/main'", "--connection-secret-stdin", diff --git a/tests/mw-dev/scenario/sql/validate_posthog_coverage.sql b/tests/mw-dev/scenario/sql/validate_posthog_coverage.sql new file mode 100644 index 000000000..43506ec58 --- /dev/null +++ b/tests/mw-dev/scenario/sql/validate_posthog_coverage.sql @@ -0,0 +1,56 @@ +-- Fail rather than record misleading fast results for an empty workload. +-- Checks expose only fixed assertion labels, never fixture identifiers. +SELECT CASE WHEN EXISTS ( + SELECT 1 FROM posthog.events + WHERE timestamp >= TIMESTAMPTZ '2026-03-17 00:00:00+00' + AND timestamp < TIMESTAMPTZ '2026-03-24 00:00:00+00' + AND event = '$pageview' AND person_id IS NOT NULL +) THEN 1 ELSE error('coverage fixture has no first-week pageview actors') END; + +SELECT CASE WHEN EXISTS ( + SELECT 1 FROM posthog.events + WHERE timestamp >= TIMESTAMPTZ '2026-03-17 00:00:00+00' + AND timestamp < TIMESTAMPTZ '2026-03-24 00:00:00+00' + AND event = '$autocapture' AND person_id IS NOT NULL +) THEN 1 ELSE error('coverage fixture has no first-week autocapture actors') END; + +SELECT CASE WHEN EXISTS ( + SELECT 1 FROM posthog.events + WHERE timestamp >= TIMESTAMPTZ '2026-03-24 00:00:00+00' + AND timestamp < TIMESTAMPTZ '2026-03-31 00:00:00+00' + AND person_id IS NOT NULL +) THEN 1 ELSE error('coverage fixture has no return-week actors') END; + +SELECT CASE WHEN EXISTS ( + SELECT 1 FROM posthog.events e + INNER JOIN posthog.persons p ON e.team_id = p.team_id AND e.person_id = p.id + WHERE e.timestamp >= TIMESTAMPTZ '2026-03-17 00:00:00+00' + AND e.timestamp < TIMESTAMPTZ '2026-03-24 00:00:00+00' +) THEN 1 ELSE error('coverage fixture has no first-week event/person overlap') END; + +WITH starts AS ( + SELECT team_id, person_id, MIN(timestamp) AS started_at + FROM posthog.events + WHERE timestamp >= TIMESTAMPTZ '2026-03-17 00:00:00+00' + AND timestamp < TIMESTAMPTZ '2026-03-24 00:00:00+00' + AND event = '$pageview' AND person_id IS NOT NULL + GROUP BY team_id, person_id +) +SELECT CASE WHEN EXISTS ( + SELECT 1 FROM starts s + INNER JOIN posthog.events e ON s.team_id = e.team_id AND s.person_id = e.person_id + WHERE e.event = '$autocapture' AND e.timestamp > s.started_at + AND e.timestamp <= s.started_at + INTERVAL '7' DAY + AND e.timestamp >= TIMESTAMPTZ '2026-03-17 00:00:00+00' + AND e.timestamp < TIMESTAMPTZ '2026-03-31 00:00:00+00' +) THEN 1 ELSE error('coverage fixture has no ordered funnel completions') END; + +SELECT CASE WHEN EXISTS ( + SELECT 1 FROM posthog.events first_week + INNER JOIN posthog.events second_week + ON first_week.team_id = second_week.team_id AND first_week.person_id = second_week.person_id + WHERE first_week.timestamp >= TIMESTAMPTZ '2026-03-17 00:00:00+00' + AND first_week.timestamp < TIMESTAMPTZ '2026-03-24 00:00:00+00' + AND second_week.timestamp >= TIMESTAMPTZ '2026-03-24 00:00:00+00' + AND second_week.timestamp < TIMESTAMPTZ '2026-03-31 00:00:00+00' +) THEN 1 ELSE error('coverage fixture has no retained actors across weeks') END; diff --git a/tests/perf/README.md b/tests/perf/README.md index 90028f147..cf4376f94 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -2,6 +2,18 @@ This package contains the golden-query performance harness. +Manual-only extended coverage scenarios exercise twelve additional workload shapes in +`queries/ducklake_posthog_coverage.yaml`. They retain separate `perf-coverage/` +artifacts and versioned intent IDs. These queries are excluded from standard +nightly runs and execute only when explicitly dispatched. For the focused uncached DuckDB run, +baseline recording, runtime estimates, and failure recovery, see the +[coverage runbook](../../docs/runbooks/perf-coverage.md). Frozen standard and extended benchmarks +use three Trino workers at 7 CPU / 28 GiB each and one DuckDB worker at +21 CPU / 84 GiB, matching aggregate worker resources. Trino uses a 20 GiB JVM +heap, 10 GiB per-worker query cap, and 30 GiB cluster-wide query cap. +Historical 3 CPU / 12 GiB results require a new baseline before comparison to +this larger profile; see the runbook for scheduling prerequisites. + ## Protocol Drivers Catalogs may target `pgwire`, `pgwire_uncached`, `pgwire_cached`, `trino`, `trino_cached`, @@ -325,8 +337,10 @@ Both clusters load the same Alluxio and memory cache managers. Each node has a 16GB disk-cache budget in a 20Gi ephemeral volume, with 64kB pages and a seven-day TTL. For connectors that use this cache, entries persist between queries/iterations until eviction or teardown; one warm-up does not guarantee -every replica holds the complete working set. Worker CPU/memory limits remain -three workers at 1 CPU/4Gi per cluster; cache volumes reserve additional storage. +every replica holds the complete working set. Each comparison cluster uses +three workers at 7 CPU / 28 GiB each, matching the DuckDB worker's aggregate +21 CPU / 84 GiB budget. The auxiliary bootstrap cluster retains three workers +at 1 CPU / 4 GiB each; cache volumes reserve additional storage. The frozen suite runs the newest PostHog/trino master build, whose Hoglake connector honors `fs.cache.enabled` through Trino's shared filesystem module @@ -367,7 +381,7 @@ Both suites of one nightly publish under the same dataset version (`posthog-file-views-v1`, shared through a YAML anchor in the scenario), with three columns on `runs` and `query_results`: -- `suite` (`tables` | `properties`, a closed set in `core`, validated when the +- `suite` (`tables` | `properties` | `coverage`, a closed set in `core`, validated when the step starts so a typo fails before hours of measurement); - `nightly_run_id`, the table-suite run's ID, which pairs a nightly's suites explicitly (a standalone run is its own nightly); @@ -393,3 +407,19 @@ Scenario Trino connections require `DUCKGRES_SCENARIO_TRINO_CATALOG_STORE_CELL_I (no default), matching the baseline coordinator's `catalog-store.cell-id`. The isolated workflow supplies it; local runs must set it. The public readiness API cell ID is used only for API identity validation, not catalog-store lookups. + +### Benchmark worker connection startup + +Frozen core and extended scenario runs default PostgreSQL connection startup to +360 seconds, allowing the server's five-minute worker-acquisition budget plus +headroom for cold worker startup. Other scenarios retain a 10-second default. +Override with `DUCKGRES_SCENARIO_PG_CONNECT_TIMEOUT` (seconds) when invoking +`tests/mw-dev/run.sh`. This timeout is passed into the scenario Job; query +execution timeouts and measured query durations are unchanged. + +If a run reports `pin pgwire cache-variant connection` timeouts, inspect +control-plane worker-acquisition logs and pod scheduling/startup events. The +cache-variant driver retains its initial setup error, so one failed connection +can mark every query for that target as failed. Resolve worker readiness or +connection-timeout configuration before rerunning; these are not query latency +measurements. diff --git a/tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/README.md b/tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/README.md new file mode 100644 index 000000000..7b16aedfb --- /dev/null +++ b/tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/README.md @@ -0,0 +1,77 @@ +# Coverage v1: uncached DuckDB baseline + +This is the historical small-resource comparison base for the twelve coverage v1 intents on +`pgwire_uncached`. [Successful source run](https://github.com/PostHog/duckgres/actions/runs/36159233201) +measured commit `9c9f5f305f87b7f0d41c0b54d64e5648aeb8ef83` on 2026-09-25. +The catalog and query settings match the measured revision. The focused entry +point was subsequently renamed to `posthog_frozen_perf_extended_uncached`; the +manifest preserves the original scenario name used by the source run. + +[Samples](samples.csv) retain all 48 successful measurements; +[manifest](manifest.json) records the catalog hash, dataset version, cache +settings, resource requests, methodology, and timing calculations. No query +result values or internal fixture locations are retained. + +The current benchmark uses larger workers. These samples and runtime estimates +apply only to the original 3 CPU / 12 GiB aggregate profile; they must not be +used as a like-for-like baseline for the new 21 CPU / 84 GiB profile. + +## Measured times + +- Coverage phase: **87m06s**, including 12 warmups and 48 measurements; zero errors. +- Complete workflow: **1h42m45s** (creation through completed-run update). +- Time outside the coverage phase: **15m39s**, including build, deployment, + fixture validation, artifact collection, and teardown. Coverage validation + itself took **3m50s**. +- Sum of per-query means: **17m26s**. The two person joins account for **57.1%**. + +| Query family | Mean (s) | Median (s) | Observed min–max (s) | +|---|---:|---:|---:| +| Daily unique users | 61.35 | 61.68 | 57.99–64.03 | +| Event trends | 38.86 | 39.16 | 36.72–40.39 | +| Person enrichment join | 361.08 | 362.75 | 348.63–370.20 | +| Person activity join | 235.86 | 235.82 | 229.27–242.52 | +| Recent events | 72.50 | 72.99 | 68.57–75.46 | +| Active actors | 41.93 | 42.27 | 40.39–42.77 | +| Ordered funnel | 67.24 | 67.42 | 64.07–70.07 | +| Sampled-week retention | 42.17 | 43.28 | 37.92–44.19 | +| Repeat-activity cohort | 21.09 | 21.76 | 18.82–22.01 | +| Exclusion cohort | 27.72 | 28.50 | 25.09–28.80 | +| Daily-person model | 22.81 | 23.04 | 20.99–24.17 | +| Session model | 53.31 | 52.58 | 51.32–56.74 | + +## Original separate-dispatch estimate + +The original planning estimate assumed a separate workflow dispatch per +configuration. With one warmup plus four measurements, equal query speeds +would add **7h16m of query work** across five configurations. Using the measured +15m39s overhead for each dispatch adds another **1h18m**. Adding the +[original 63m20s workflow](https://github.com/PostHog/duckgres/actions/runs/36113843696) +gives **about 9h37m total**, or **8h34m additional**. + +This is a serial planning estimate, not a measured result for the other engines. +Their setup, warmup, and query costs may differ. It excludes queue delays. +Holding DuckDB uncached fixed and varying the other four configurations: + +| Other configurations' query durations relative to uncached DuckDB | Estimated campaign | +|---|---:| +| 0.5× | 6h43m | +| 1× | 9h37m | +| 2× | 15h26m | + +These are scenarios, not statistical confidence bounds. Even the equal-speed +query work exceeds the four-hour scenario limit, which is why coverage uses +separate target jobs. The current workflow builds images once and runs targets +as sequential matrix jobs, reducing the equal-speed planning estimate to +approximately **8h16m extended-only**, or **9h19m including the original suite**. +The manifest retains the original estimate for provenance; see the runbook for +the shared-build calculation. This baseline does not establish Trino or Athena +performance. Resource requests are 3 CPU / 12Gi for DuckDB workers only. + +## Future comparisons + +Compare matching intent IDs and protocol using the same catalog hash, fixture, +resource requests, and cache settings. Report per-query mean/median ratios and +absolute times; a total ratio is dominated by slow queries. A changed query or +window requires a new intent version and baseline. Four repetitions do not +establish tail latency. See the [runbook](../../../../docs/runbooks/perf-coverage.md). diff --git a/tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/manifest.json b/tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/manifest.json new file mode 100644 index 000000000..1a740e045 --- /dev/null +++ b/tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/manifest.json @@ -0,0 +1,56 @@ +{ + "baseline_id": "coverage-v1-duckdb-uncached-2026-09-25", + "source_run_url": "https://github.com/PostHog/duckgres/actions/runs/36159233201", + "tested_commit": "9c9f5f305f87b7f0d41c0b54d64e5648aeb8ef83", + "catalog_path": "tests/perf/queries/ducklake_posthog_coverage.yaml", + "catalog_sha256": "e2971f25433a6067f1c0ac42229bf7dc7c82c0c8ab51d6e476b4848df5c59226", + "dataset_version": "posthog-file-views-v1", + "suite": "coverage", + "protocol": "pgwire_uncached", + "scenario": "posthog_frozen_perf_coverage_uncached", + "worker_resource_requests": { + "cpu": "3", + "memory": "12Gi" + }, + "cache_settings": { + "enable_external_file_cache": false, + "parquet_metadata_cache": false, + "enable_http_metadata_cache": false + }, + "warmup_iterations": 1, + "measure_iterations": 4, + "query_intents": 12, + "measured_queries": 48, + "warmup_queries": 12, + "query_errors": 0, + "timing_seconds": { + "coverage_phase": 5225.503589484, + "coverage_fixture_validation": 230.235161795, + "workflow_created_to_completed_updated_at": 6165, + "workflow_outside_coverage_phase": 939.4964105159997, + "measured_query_total": 4183.687134048, + "sum_query_means": 1045.921783512 + }, + "workflow_created_at": "2026-09-25T16:12:09Z", + "completed_workflow_updated_at": "2026-09-25T17:54:54Z", + "coverage_started_at": "2026-09-25T16:26:19.084523672Z", + "coverage_finished_at": "2026-09-25T17:53:24.58464969Z", + "estimate": { + "original_workflow_url": "https://github.com/PostHog/duckgres/actions/runs/36113843696", + "original_workflow_seconds": 3800, + "repetitions_per_configuration": 5, + "configurations": 5, + "query_seconds_at_equal_speed": 26148.044587800003, + "assumed_overhead_per_dispatch_seconds": 939.4964105159997, + "campaign_seconds_at_equal_speed": 34645.526640380005 + }, + "limitations": [ + "Only uncached DuckDB was measured; peer speeds and setup overhead are assumptions.", + "Resource values are DuckDB worker requests, not equal engine hardware budgets.", + "Uncached disables named DuckDB caches, not OS or storage caches.", + "Frozen fixture contains sampled days; retention is a performance shape, not a production rate.", + "Four repetitions characterize observed variation, not tail latency.", + "Independent coverage starts with different cache history from coverage appended after other suites.", + "Workflow wall time uses createdAt through the completed run updatedAt; no queue time from separate future dispatches is estimated." + ] +} diff --git a/tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/samples.csv b/tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/samples.csv new file mode 100644 index 000000000..e4a154e5e --- /dev/null +++ b/tests/perf/baselines/coverage-v1-duckdb-uncached-2026-09-25/samples.csv @@ -0,0 +1,49 @@ +query_id,intent_id,protocol,measure_iteration,duration_ms +q_coverage_daily_uniques_v1__ducklake_table,intent_coverage_daily_uniques_v1,pgwire_uncached,1,57990.076097 +q_coverage_event_trends_v1__ducklake_table,intent_coverage_event_trends_v1,pgwire_uncached,1,36721.211893 +q_coverage_person_enrichment_v1__ducklake_table,intent_coverage_person_enrichment_v1,pgwire_uncached,1,348630.727166 +q_coverage_person_activity_join_v1__ducklake_table,intent_coverage_person_activity_join_v1,pgwire_uncached,1,229274.438235 +q_coverage_recent_events_v1__ducklake_table,intent_coverage_recent_events_v1,pgwire_uncached,1,68569.479379 +q_coverage_active_actors_v1__ducklake_table,intent_coverage_active_actors_v1,pgwire_uncached,1,40394.218659 +q_coverage_ordered_funnel_v1__ducklake_table,intent_coverage_ordered_funnel_v1,pgwire_uncached,1,64066.620569 +q_coverage_weekly_retention_v1__ducklake_table,intent_coverage_weekly_retention_v1,pgwire_uncached,1,37916.014597 +q_coverage_repeat_activity_cohort_v1__ducklake_table,intent_coverage_repeat_activity_cohort_v1,pgwire_uncached,1,18816.490929 +q_coverage_exclusion_cohort_v1__ducklake_table,intent_coverage_exclusion_cohort_v1,pgwire_uncached,1,25091.107455 +q_coverage_daily_person_model_v1__ducklake_table,intent_coverage_daily_person_model_v1,pgwire_uncached,1,20991.281524 +q_coverage_session_model_v1__ducklake_table,intent_coverage_session_model_v1,pgwire_uncached,1,51323.299029 +q_coverage_daily_uniques_v1__ducklake_table,intent_coverage_daily_uniques_v1,pgwire_uncached,2,60034.007037 +q_coverage_event_trends_v1__ducklake_table,intent_coverage_event_trends_v1,pgwire_uncached,2,39430.339134 +q_coverage_person_enrichment_v1__ducklake_table,intent_coverage_person_enrichment_v1,pgwire_uncached,2,370202.228797 +q_coverage_person_activity_join_v1__ducklake_table,intent_coverage_person_activity_join_v1,pgwire_uncached,2,242519.554305 +q_coverage_recent_events_v1__ducklake_table,intent_coverage_recent_events_v1,pgwire_uncached,2,73454.258391 +q_coverage_active_actors_v1__ducklake_table,intent_coverage_active_actors_v1,pgwire_uncached,2,42441.753842 +q_coverage_ordered_funnel_v1__ducklake_table,intent_coverage_ordered_funnel_v1,pgwire_uncached,2,70066.542500 +q_coverage_weekly_retention_v1__ducklake_table,intent_coverage_weekly_retention_v1,pgwire_uncached,2,43317.646138 +q_coverage_repeat_activity_cohort_v1__ducklake_table,intent_coverage_repeat_activity_cohort_v1,pgwire_uncached,2,21521.700775 +q_coverage_exclusion_cohort_v1__ducklake_table,intent_coverage_exclusion_cohort_v1,pgwire_uncached,2,28393.201396 +q_coverage_daily_person_model_v1__ducklake_table,intent_coverage_daily_person_model_v1,pgwire_uncached,2,23109.121973 +q_coverage_session_model_v1__ducklake_table,intent_coverage_session_model_v1,pgwire_uncached,2,52682.057170 +q_coverage_daily_uniques_v1__ducklake_table,intent_coverage_daily_uniques_v1,pgwire_uncached,3,64029.693248 +q_coverage_event_trends_v1__ducklake_table,intent_coverage_event_trends_v1,pgwire_uncached,3,40387.651561 +q_coverage_person_enrichment_v1__ducklake_table,intent_coverage_person_enrichment_v1,pgwire_uncached,3,362181.664088 +q_coverage_person_activity_join_v1__ducklake_table,intent_coverage_person_activity_join_v1,pgwire_uncached,3,238072.547003 +q_coverage_recent_events_v1__ducklake_table,intent_coverage_recent_events_v1,pgwire_uncached,3,75460.142206 +q_coverage_active_actors_v1__ducklake_table,intent_coverage_active_actors_v1,pgwire_uncached,3,42773.287401 +q_coverage_ordered_funnel_v1__ducklake_table,intent_coverage_ordered_funnel_v1,pgwire_uncached,3,69409.701471 +q_coverage_weekly_retention_v1__ducklake_table,intent_coverage_weekly_retention_v1,pgwire_uncached,3,44191.445753 +q_coverage_repeat_activity_cohort_v1__ducklake_table,intent_coverage_repeat_activity_cohort_v1,pgwire_uncached,3,22001.911651 +q_coverage_exclusion_cohort_v1__ducklake_table,intent_coverage_exclusion_cohort_v1,pgwire_uncached,3,28798.335575 +q_coverage_daily_person_model_v1__ducklake_table,intent_coverage_daily_person_model_v1,pgwire_uncached,3,24172.015999 +q_coverage_session_model_v1__ducklake_table,intent_coverage_session_model_v1,pgwire_uncached,3,52483.776282 +q_coverage_daily_uniques_v1__ducklake_table,intent_coverage_daily_uniques_v1,pgwire_uncached,4,63326.694027 +q_coverage_event_trends_v1__ducklake_table,intent_coverage_event_trends_v1,pgwire_uncached,4,38884.138434 +q_coverage_person_enrichment_v1__ducklake_table,intent_coverage_person_enrichment_v1,pgwire_uncached,4,363320.561007 +q_coverage_person_activity_join_v1__ducklake_table,intent_coverage_person_activity_join_v1,pgwire_uncached,4,233574.135155 +q_coverage_recent_events_v1__ducklake_table,intent_coverage_recent_events_v1,pgwire_uncached,4,72533.884957 +q_coverage_active_actors_v1__ducklake_table,intent_coverage_active_actors_v1,pgwire_uncached,4,42101.277939 +q_coverage_ordered_funnel_v1__ducklake_table,intent_coverage_ordered_funnel_v1,pgwire_uncached,4,65435.443518 +q_coverage_weekly_retention_v1__ducklake_table,intent_coverage_weekly_retention_v1,pgwire_uncached,4,43249.354433 +q_coverage_repeat_activity_cohort_v1__ducklake_table,intent_coverage_repeat_activity_cohort_v1,pgwire_uncached,4,22014.864298 +q_coverage_exclusion_cohort_v1__ducklake_table,intent_coverage_exclusion_cohort_v1,pgwire_uncached,4,28613.186554 +q_coverage_daily_person_model_v1__ducklake_table,intent_coverage_daily_person_model_v1,pgwire_uncached,4,22972.236876 +q_coverage_session_model_v1__ducklake_table,intent_coverage_session_model_v1,pgwire_uncached,4,56741.807622 diff --git a/tests/perf/core/catalog_test.go b/tests/perf/core/catalog_test.go index 6b6875ec0..a1f190df6 100644 --- a/tests/perf/core/catalog_test.go +++ b/tests/perf/core/catalog_test.go @@ -25,7 +25,7 @@ func TestCheckedInCatalogsLoad(t *testing.T) { t.Fatalf("LoadCatalog(%s): %v", path, err) } wantTargets := []Protocol{ProtocolPGWire} - if filepath.Base(path) == "ducklake_posthog_tables.yaml" { + if filepath.Base(path) == "ducklake_posthog_tables.yaml" || filepath.Base(path) == "ducklake_posthog_coverage.yaml" { wantTargets = []Protocol{ProtocolPGWireUncached, ProtocolPGWireCached, ProtocolTrino, ProtocolTrinoCached, ProtocolAthena} } if !reflect.DeepEqual(catalog.Targets, wantTargets) { diff --git a/tests/perf/core/types.go b/tests/perf/core/types.go index b32024beb..10d2d2c47 100644 --- a/tests/perf/core/types.go +++ b/tests/perf/core/types.go @@ -157,13 +157,14 @@ type QueryResult struct { const ( SuiteTables = "tables" SuiteProperties = "properties" + SuiteCoverage = "coverage" ) // ValidateSuite rejects a suite outside the closed set, so a typo fails when a // run starts rather than after hours of measurement at publish time. func ValidateSuite(suite string) error { - if suite != SuiteTables && suite != SuiteProperties { - return fmt.Errorf("unknown suite %q (want %q or %q)", suite, SuiteTables, SuiteProperties) + if suite != SuiteTables && suite != SuiteProperties && suite != SuiteCoverage { + return fmt.Errorf("unknown suite %q (want %q, %q, or %q)", suite, SuiteTables, SuiteProperties, SuiteCoverage) } return nil } @@ -171,8 +172,8 @@ func ValidateSuite(suite string) error { type RunSummary struct { RunID string `json:"run_id"` DatasetVersion string `json:"dataset_version"` - // Suite names the comparison family inside a dataset ("tables" or - // "properties"); one nightly publishes several suites under one dataset. + // Suite names the comparison family inside a dataset (tables, properties, + // or coverage); one nightly publishes several suites under one dataset. Suite string `json:"suite,omitempty"` // FixtureVersion identifies the exact fixture a suite measured when it is // finer-grained than the dataset (e.g. the properties object inventory). diff --git a/tests/perf/queries/ducklake_posthog_coverage.yaml b/tests/perf/queries/ducklake_posthog_coverage.yaml new file mode 100644 index 000000000..028c6bc70 --- /dev/null +++ b/tests/perf/queries/ducklake_posthog_coverage.yaml @@ -0,0 +1,175 @@ +name: posthog-frozen-coverage-v1 +description: Bounded analytical workload coverage on the frozen events/persons snapshot; background shapes measure SELECT computation, not materialization writes. +seed: 42 +dataset_scale: 1 +targets: [pgwire_uncached, pgwire_cached, trino, trino_cached, athena] +warmup_iterations: 1 +measure_iterations: 4 +relation_variants: + ducklake_table: {events: posthog.events, persons: posthog.persons} + hoglake_table: {events: posthog.events, persons: posthog.persons} + athena_external: {events: events, persons: persons} + +# Fixed UTC windows make comparisons reproducible. Persons are deduplicated by +# tenant and ID before joins because the frozen export contains snapshot rows. +# Identification means observed identified in the snapshot, not as-of event time. +# Funnel: first pageview followed by autocapture within seven days. +# Retention: activity in week one and again in week two (not new-user retention). +# Session model: new session after an inactivity gap strictly greater than 30m. +# Verify date/event/join population before treating a run as representative. +paired_queries: + - query_id_base: q_coverage_daily_uniques_v1 + intent_id: intent_coverage_daily_uniques_v1 + tags: [nightly, frozen, posthog, coverage, paired, foreground, distinct, trends] + params: {} + sql_template: | + SELECT CAST(day AS VARCHAR) AS day, COUNT(*) AS persons FROM ( + SELECT CAST("timestamp" AT TIME ZONE 'UTC' AS DATE) AS day, team_id, person_id + FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-27 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND person_id IS NOT NULL AND event = '$pageview' + GROUP BY 1, 2, 3 + ) active GROUP BY day ORDER BY day + + - query_id_base: q_coverage_event_trends_v1 + intent_id: intent_coverage_event_trends_v1 + tags: [nightly, frozen, posthog, coverage, paired, foreground, aggregate, trends] + params: {} + sql_template: | + SELECT CAST(CAST("timestamp" AT TIME ZONE 'UTC' AS DATE) AS VARCHAR) AS day, event, COUNT(*) AS events FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-27 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND event IN ('$pageview', '$autocapture') GROUP BY 1, 2 ORDER BY 1, 2 + + - query_id_base: q_coverage_person_enrichment_v1 + intent_id: intent_coverage_person_enrichment_v1 + tags: [nightly, frozen, posthog, coverage, paired, foreground, join] + params: {} + sql_template: | + SELECT * FROM ( + WITH deduplicated_persons AS ( + SELECT team_id, id, MAX(CASE WHEN is_identified THEN 1 ELSE 0 END) AS identified + FROM {{ relation "persons" }} GROUP BY team_id, id + ) + SELECT p.identified = 1 AS is_identified, COUNT(*) AS events + FROM (SELECT * FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE)) e JOIN deduplicated_persons p ON e.team_id = p.team_id AND e.person_id = p.id + GROUP BY p.identified ORDER BY p.identified + ) coverage_query ORDER BY 1 + + - query_id_base: q_coverage_person_activity_join_v1 + intent_id: intent_coverage_person_activity_join_v1 + tags: [nightly, frozen, posthog, coverage, paired, foreground, join] + params: {} + sql_template: | + SELECT * FROM ( + WITH activity AS (SELECT team_id, person_id, COUNT(*) AS events FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND person_id IS NOT NULL GROUP BY team_id, person_id), deduplicated_persons AS (SELECT DISTINCT team_id, id FROM {{ relation "persons" }}) + SELECT COUNT(*) AS matched_persons, COALESCE(SUM(a.events), 0) AS events + FROM activity a JOIN deduplicated_persons p ON a.team_id = p.team_id AND a.person_id = p.id + ) coverage_query + + - query_id_base: q_coverage_recent_events_v1 + intent_id: intent_coverage_recent_events_v1 + tags: [nightly, frozen, posthog, coverage, paired, foreground, retrieval] + params: {} + sql_template: | + SELECT uuid, event, person_id FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + ORDER BY "timestamp" DESC, uuid DESC LIMIT 100 + + - query_id_base: q_coverage_active_actors_v1 + intent_id: intent_coverage_active_actors_v1 + tags: [nightly, frozen, posthog, coverage, paired, foreground, retrieval] + params: {} + sql_template: | + SELECT team_id, person_id, COUNT(*) AS events FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND person_id IS NOT NULL GROUP BY team_id, person_id ORDER BY events DESC, team_id, person_id LIMIT 100 + + - query_id_base: q_coverage_ordered_funnel_v1 + intent_id: intent_coverage_ordered_funnel_v1 + tags: [nightly, frozen, posthog, coverage, paired, foreground, funnel] + params: {} + sql_template: | + SELECT * FROM ( + WITH starts AS ( + SELECT team_id, person_id, MIN("timestamp") AS started_at FROM {{ relation "events" }} + WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND event = '$pageview' AND person_id IS NOT NULL GROUP BY team_id, person_id + ), completions AS ( + SELECT s.team_id, s.person_id, MIN(e."timestamp") AS completed_at + FROM starts s JOIN {{ relation "events" }} e ON s.team_id = e.team_id AND s.person_id = e.person_id + WHERE e.event = '$autocapture' AND e."timestamp" > s.started_at + AND e."timestamp" <= s.started_at + INTERVAL '7' DAY + AND e."timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND e."timestamp" < CAST('2026-03-31 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + GROUP BY s.team_id, s.person_id + ) + SELECT COUNT(*) AS started_persons, COUNT(c.person_id) AS completed_persons + FROM starts s LEFT JOIN completions c ON s.team_id = c.team_id AND s.person_id = c.person_id + ) coverage_query + + - query_id_base: q_coverage_weekly_retention_v1 + intent_id: intent_coverage_weekly_retention_v1 + tags: [nightly, frozen, posthog, coverage, paired, foreground, retention] + params: {} + sql_template: | + SELECT * FROM ( + WITH activity AS ( + SELECT team_id, person_id, + MAX(CASE WHEN "timestamp" < CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) THEN 1 ELSE 0 END) AS first_week, + MAX(CASE WHEN "timestamp" >= CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) THEN 1 ELSE 0 END) AS second_week + FROM {{ relation "events" }} + WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-31 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND person_id IS NOT NULL GROUP BY team_id, person_id + ) + SELECT COALESCE(SUM(first_week), 0) AS cohort_persons, + COALESCE(SUM(CASE WHEN first_week = 1 AND second_week = 1 THEN 1 ELSE 0 END), 0) AS retained_persons FROM activity + ) coverage_query + + - query_id_base: q_coverage_repeat_activity_cohort_v1 + intent_id: intent_coverage_repeat_activity_cohort_v1 + tags: [nightly, frozen, posthog, coverage, paired, background, cohort] + params: {} + sql_template: | + SELECT COUNT(*) AS cohort_persons FROM (SELECT team_id, person_id, COUNT(*) AS events FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND person_id IS NOT NULL GROUP BY team_id, person_id HAVING COUNT(*) >= 3) members + + - query_id_base: q_coverage_exclusion_cohort_v1 + intent_id: intent_coverage_exclusion_cohort_v1 + tags: [nightly, frozen, posthog, coverage, paired, background, cohort] + params: {} + sql_template: | + SELECT COUNT(*) AS cohort_persons FROM ( + SELECT team_id, person_id FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND person_id IS NOT NULL + GROUP BY team_id, person_id + HAVING SUM(CASE WHEN event = '$pageview' THEN 1 ELSE 0 END) > 0 + AND SUM(CASE WHEN event = '$autocapture' THEN 1 ELSE 0 END) = 0 + ) members + + - query_id_base: q_coverage_daily_person_model_v1 + intent_id: intent_coverage_daily_person_model_v1 + tags: [nightly, frozen, posthog, coverage, paired, background, modeling] + params: {} + sql_template: | + SELECT COUNT(*) AS person_days, COALESCE(SUM(events), 0) AS events FROM ( + SELECT team_id, person_id, CAST("timestamp" AT TIME ZONE 'UTC' AS DATE) AS day, COUNT(*) AS events + FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND person_id IS NOT NULL GROUP BY 1, 2, 3 + ) daily_activity + + - query_id_base: q_coverage_session_model_v1 + intent_id: intent_coverage_session_model_v1 + tags: [nightly, frozen, posthog, coverage, paired, background, modeling, window] + params: {} + sql_template: | + SELECT * FROM ( + WITH ordered_events AS ( + SELECT team_id, person_id, "timestamp", + LAG("timestamp") OVER (PARTITION BY team_id, person_id ORDER BY "timestamp", uuid) AS previous_at + FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-17 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) + AND "timestamp" < CAST('2026-03-24 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND person_id IS NOT NULL + ) + SELECT COALESCE(SUM(CASE WHEN previous_at IS NULL OR "timestamp" > previous_at + INTERVAL '30' MINUTE THEN 1 ELSE 0 END), 0) AS sessions, + COUNT(*) AS events FROM ordered_events + ) coverage_query +