From bf8154e524d1c22d81d99b306aebb51aad57bc8c Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 24 Sep 2026 08:01:36 -0400 Subject: [PATCH 1/2] test(perf): targeted Hoglake footer-cache run (do not merge) Narrows posthog_frozen_perf to the Trino targets and the queries bounded by per-split Parquet footer handling, to measure PostHog/trino#72 without the full five-engine run: - targets: trino, trino_cached (no PGWire, no Athena) - queries: q_events_total_v5, q_events_count_one_day_v5, and q_events_by_name_march_2026_v5 as a scan-bound regression check - 1 warmup + 2 measured iterations; no properties comparison The scenario keeps its name because scenario-dev.yml and run.sh select the Trino suite and image by it. The scenario tests that pin the nightly shape are adjusted to the targeted one. Branch runs do not publish to the perf dashboard (publishing requires refs/heads/main). Co-Authored-By: Claude Opus 5.5 --- tests/mw-dev/scenario/runner_test.go | 6 ++-- .../scenarios/posthog_frozen_perf.yaml | 32 +---------------- .../perf/queries/ducklake_posthog_tables.yaml | 34 ++----------------- 3 files changed, 8 insertions(+), 64 deletions(-) diff --git a/tests/mw-dev/scenario/runner_test.go b/tests/mw-dev/scenario/runner_test.go index 74a34a768..4b980a052 100644 --- a/tests/mw-dev/scenario/runner_test.go +++ b/tests/mw-dev/scenario/runner_test.go @@ -525,6 +525,7 @@ func TestFrozenPerfScenarioUsesSupportedStepsAndOriginalCatalog(t *testing.T) { } func TestFrozenPerfScenarioRunsOptionalPropertiesAfterOriginalPerf(t *testing.T) { + t.Skip("targeted Hoglake footer-cache run drops the properties comparison") t.Setenv("DUCKGRES_SCENARIO_PROPERTIES_S3_URI", "s3://example/properties/") t.Setenv("DUCKGRES_SCENARIO_HOGLAKE_URI", "http://hoglake:8080") t.Setenv("DUCKGRES_SCENARIO_FROZEN_S3_URI", "s3://example-frozen/frozen_v1/") @@ -715,8 +716,9 @@ func assertPerfTargetsOnlyPGWire(t *testing.T, step core.Step) { func assertPerfTargetsPGWireTrinoAndAthena(t *testing.T, step core.Step) { t.Helper() targets, ok := step.With["targets"].([]any) - if !ok || len(targets) != 5 || targets[0] != "pgwire_uncached" || targets[1] != "pgwire_cached" || targets[2] != "trino" || targets[3] != "trino_cached" || targets[4] != "athena" { - t.Fatalf("perf step %s targets = %#v, want [pgwire_uncached pgwire_cached trino trino_cached athena]", step.ID, step.With["targets"]) + // Targeted Hoglake footer-cache run: Trino targets only. + if !ok || len(targets) != 2 || targets[0] != "trino" || targets[1] != "trino_cached" { + t.Fatalf("perf step %s targets = %#v, want [trino trino_cached]", step.ID, step.With["targets"]) } } diff --git a/tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml index 3dcbce2f3..82194543f 100644 --- a/tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml +++ b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml @@ -98,7 +98,7 @@ steps: org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} catalog: ducklake catalog_file: ../../../perf/queries/ducklake_posthog_tables.yaml - targets: [pgwire_uncached, pgwire_cached, trino, trino_cached, athena] + targets: [trino, trino_cached] trino_hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}-frozen trino_ca_cert_file: ${env:DUCKGRES_SCENARIO_TRINO_CA_CERT} trino_startup_timeout: 2m @@ -116,36 +116,6 @@ steps: worker_cpu: ${env:DUCKGRES_K8S_WORKER_CPU_REQUEST} worker_memory: ${env:DUCKGRES_K8S_WORKER_MEMORY_REQUEST} - - id: properties_comparison - type: properties_comparison - depends_on: [perf_queries] - with: - org_id: ${env:DUCKGRES_SCENARIO_ORG_ID} - catalog: ducklake - hoglake_uri: ${env:DUCKGRES_SCENARIO_HOGLAKE_URI} - hoglake_file: ../perf/setup_hoglake.py - targets: [pgwire_uncached, pgwire_cached, trino, trino_cached, athena] - trino_hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}-frozen - hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}-frozen - trino_schema: properties_perf - trino_ca_cert_file: ${env:DUCKGRES_SCENARIO_TRINO_CA_CERT} - trino_startup_timeout: 2m - trino_startup_poll_interval: 2s - athena_region: ${env:DUCKGRES_SCENARIO_ATHENA_REGION} - athena_workgroup: ${env:DUCKGRES_SCENARIO_ATHENA_WORKGROUP} - athena_catalog: AwsDataCatalog - athena_database: ${env:DUCKGRES_SCENARIO_ATHENA_DATABASE} - athena_output_location: ${env:DUCKGRES_SCENARIO_ATHENA_RESULTS_S3_URI} - athena_poll_interval: 500ms - athena_query_timeout: 30m - run_id: ${run_id} - # Same dataset as perf_queries: both suites of one nightly publish together. - dataset_version: *nightly_dataset - output_subdir: perf-properties - fail_on_query_errors: true - worker_cpu: ${env:DUCKGRES_K8S_WORKER_CPU_REQUEST} - worker_memory: ${env:DUCKGRES_K8S_WORKER_MEMORY_REQUEST} - # Managed Hoglake retains tenant ownership and rejects public deprovision. # The workflow's always-run isolated-stack teardown owns cleanup of the # Duckling, namespace, metadata store and dedicated managed storage prefix. diff --git a/tests/perf/queries/ducklake_posthog_tables.yaml b/tests/perf/queries/ducklake_posthog_tables.yaml index 46a3641b2..ed32c87de 100644 --- a/tests/perf/queries/ducklake_posthog_tables.yaml +++ b/tests/perf/queries/ducklake_posthog_tables.yaml @@ -2,14 +2,13 @@ name: posthog-frozen-ducklake-golden-v5 description: Identical query shapes with protocol-isolated execution over frozen Parquet files registered as DuckLake tables on PGWire, Hoglake tables on Trino, and Athena external tables. seed: 42 dataset_scale: 1 +# Targeted Hoglake footer-cache run (PostHog/trino#72): Trino only, the +# queries bounded by per-split footer decoding plus one scan-bound check. targets: - - pgwire_uncached - - pgwire_cached - trino - trino_cached - - athena warmup_iterations: 1 -measure_iterations: 4 +measure_iterations: 2 relation_variants: ducklake_table: @@ -18,9 +17,6 @@ relation_variants: hoglake_table: events: posthog.events persons: posthog.persons - athena_external: - events: events - persons: persons paired_queries: - query_id_base: q_events_total_v5 @@ -44,27 +40,3 @@ paired_queries: tags: [nightly, frozen, posthog, events, aggregate, analytics, paired] params: {} sql_template: SELECT event, COUNT(*) AS events FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-01 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND "timestamp" < CAST('2026-03-18 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) GROUP BY event ORDER BY events DESC, event LIMIT 20 - - - query_id_base: q_events_distinct_persons_v5 - intent_id: intent_events_distinct_persons_v5 - tags: [nightly, frozen, posthog, events, aggregate, distinct, paired] - params: {} - sql_template: SELECT COUNT(DISTINCT person_id) AS distinct_persons FROM {{ relation "events" }} WHERE person_id IS NOT NULL - - - query_id_base: q_persons_total_v5 - intent_id: intent_persons_total_v5 - tags: [nightly, frozen, posthog, persons, aggregate, paired] - params: {} - sql_template: SELECT COUNT(*) AS persons FROM {{ relation "persons" }} - - - query_id_base: q_persons_daily_april_2026_v5 - intent_id: intent_persons_daily_april_2026_v5 - tags: [nightly, frozen, posthog, persons, aggregate, time-series, paired] - params: {} - sql_template: SELECT date_trunc('day', _timestamp) AS day, COUNT(*) AS persons FROM {{ relation "persons" }} WHERE _timestamp >= CAST('2026-04-01 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND _timestamp < CAST('2026-05-01 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) GROUP BY 1 ORDER BY 1 - - - query_id_base: q_events_daily_march_2026_v5 - intent_id: intent_events_daily_march_2026_v5 - tags: [nightly, frozen, posthog, events, aggregate, time-series, paired] - params: {} - sql_template: SELECT date_trunc('day', "timestamp") AS day, COUNT(*) AS events FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-01 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND "timestamp" < CAST('2026-03-18 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) GROUP BY 1 ORDER BY 1 From 866ad8243756b9a77587f753ff7eebad0dc44279 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 24 Sep 2026 15:02:32 -0400 Subject: [PATCH 2/2] test(perf): run every query on Trino against a hoglake build with column bounds (do not merge) Restore the full nightly query set for the trino and trino_cached targets, still with one warmup and two measured iterations. Pin the isolated hoglake server to a build of main 27a705ed, which serves /scan column_stats (PostHog/hoglake#189), so the connector's planning-time pruning (PostHog/trino#77) has bounds to prune on. The merge of main brings the stats-hydration wait (#1229). Co-Authored-By: Claude Opus 5.5 --- tests/mw-dev/run.sh | 4 ++- .../perf/queries/ducklake_posthog_tables.yaml | 28 +++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/tests/mw-dev/run.sh b/tests/mw-dev/run.sh index 93aea687c..872bed80b 100755 --- a/tests/mw-dev/run.sh +++ b/tests/mw-dev/run.sh @@ -65,7 +65,9 @@ else fi TRINO_TLS_PASSWORD="${TRINO_TLS_PASSWORD:-duckgres-e2e-keystore}" # Managed admission requires atomic-table-creation-v1, including frozen perf. -HOGLAKE_IMAGE="${HOGLAKE_IMAGE:-ghcr.io/posthog/hoglake-server@sha256:fcd2bdc2b17cbe7bdf4b52b19ebe1c901c853c5ec925cf52a94609e464e96a02}" +# Targeted run (do not merge): a hoglake build with /scan column_stats +# (PostHog/hoglake#189, main 27a705ed), so the connector can prune files. +HOGLAKE_IMAGE="${HOGLAKE_IMAGE:-ghcr.io/posthog/hoglake-server@sha256:12d713e688902ea3f46aeee7266139abdc7eeb30bb52f22e67d03c1ad9f65b76}" HOGLAKE_DATA_PATH="${HOGLAKE_DATA_PATH:-}" if [ "${GITHUB_ACTIONS:-}" = true ] && [ -n "$HOGLAKE_DATA_PATH" ]; then hoglake_bucket="${HOGLAKE_DATA_PATH#s3://}" diff --git a/tests/perf/queries/ducklake_posthog_tables.yaml b/tests/perf/queries/ducklake_posthog_tables.yaml index ed32c87de..120a88894 100644 --- a/tests/perf/queries/ducklake_posthog_tables.yaml +++ b/tests/perf/queries/ducklake_posthog_tables.yaml @@ -2,8 +2,8 @@ name: posthog-frozen-ducklake-golden-v5 description: Identical query shapes with protocol-isolated execution over frozen Parquet files registered as DuckLake tables on PGWire, Hoglake tables on Trino, and Athena external tables. seed: 42 dataset_scale: 1 -# Targeted Hoglake footer-cache run (PostHog/trino#72): Trino only, the -# queries bounded by per-split footer decoding plus one scan-bound check. +# Targeted Hoglake connector run (do not merge): every nightly query, on the +# two Trino targets only, with fewer measured iterations than the nightly. targets: - trino - trino_cached @@ -40,3 +40,27 @@ paired_queries: tags: [nightly, frozen, posthog, events, aggregate, analytics, paired] params: {} sql_template: SELECT event, COUNT(*) AS events FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-01 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND "timestamp" < CAST('2026-03-18 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) GROUP BY event ORDER BY events DESC, event LIMIT 20 + + - query_id_base: q_events_distinct_persons_v5 + intent_id: intent_events_distinct_persons_v5 + tags: [nightly, frozen, posthog, events, aggregate, distinct, paired] + params: {} + sql_template: SELECT COUNT(DISTINCT person_id) AS distinct_persons FROM {{ relation "events" }} WHERE person_id IS NOT NULL + + - query_id_base: q_persons_total_v5 + intent_id: intent_persons_total_v5 + tags: [nightly, frozen, posthog, persons, aggregate, paired] + params: {} + sql_template: SELECT COUNT(*) AS persons FROM {{ relation "persons" }} + + - query_id_base: q_persons_daily_april_2026_v5 + intent_id: intent_persons_daily_april_2026_v5 + tags: [nightly, frozen, posthog, persons, aggregate, time-series, paired] + params: {} + sql_template: SELECT date_trunc('day', _timestamp) AS day, COUNT(*) AS persons FROM {{ relation "persons" }} WHERE _timestamp >= CAST('2026-04-01 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND _timestamp < CAST('2026-05-01 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) GROUP BY 1 ORDER BY 1 + + - query_id_base: q_events_daily_march_2026_v5 + intent_id: intent_events_daily_march_2026_v5 + tags: [nightly, frozen, posthog, events, aggregate, time-series, paired] + params: {} + sql_template: SELECT date_trunc('day', "timestamp") AS day, COUNT(*) AS events FROM {{ relation "events" }} WHERE "timestamp" >= CAST('2026-03-01 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) AND "timestamp" < CAST('2026-03-18 00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE) GROUP BY 1 ORDER BY 1