Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 97 additions & 0 deletions .github/workflows/api-perf.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
name: API performance

on:
schedule:
- cron: "17 2 * * *"
workflow_dispatch:
pull_request:
paths:
- '.github/workflows/api-perf.yml'
- 'backend/tests/perf/**'

permissions:
contents: read

jobs:
api-perf:
name: Locust on 10,000 runs
runs-on: ubuntu-latest
timeout-minutes: 30
services:
postgres:
image: postgres:17
env:
POSTGRES_USER: roboprobe
POSTGRES_PASSWORD: roboprobe-ci
POSTGRES_DB: roboprobe_perf
ports: ["5432:5432"]
options: >-
--health-cmd "pg_isready -U roboprobe -d roboprobe_perf"
--health-interval 5s --health-timeout 5s --health-retries 10
env:
PYTHON_VERSION: '3.12'
LOCUST_VERSION: '2.47.0'
OPENBLAS_NUM_THREADS: '1'
PGHOST: 127.0.0.1
PGUSER: roboprobe
PGPASSWORD: roboprobe-ci
PGDATABASE: roboprobe_perf
ROBOPROBE_ENV: ci
ROBOPROBE_DATABASE__URL: postgresql+psycopg://roboprobe:roboprobe-ci@127.0.0.1:5432/roboprobe_perf
ROBOPROBE_API__HOST: 127.0.0.1
ROBOPROBE_API__PORT: '8000'
defaults:
run:
working-directory: backend
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: ./.github/actions/setup-backend
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Seed and verify 10,000 completed runs
run: |
set -euo pipefail
mkdir -p test-results/perf
export ROBOPROBE_STORAGE__WORKDIR_ROOT="$RUNNER_TEMP/roboprobe-perf-work"
export ROBOPROBE_STORAGE__ARTIFACT_ROOT="$RUNNER_TEMP/roboprobe-perf-artifacts"
printf '%s\n' "ROBOPROBE_STORAGE__WORKDIR_ROOT=$ROBOPROBE_STORAGE__WORKDIR_ROOT" \
"ROBOPROBE_STORAGE__ARTIFACT_ROOT=$ROBOPROBE_STORAGE__ARTIFACT_ROOT" >> "$GITHUB_ENV"
uv run --locked python ../scripts/seed_demo.py --size large \
--database-url "$ROBOPROBE_DATABASE__URL" \
--workdir-root "$ROBOPROBE_STORAGE__WORKDIR_ROOT" \
--artifact-root "$ROBOPROBE_STORAGE__ARTIFACT_ROOT" \
> test-results/perf/seed.log 2>&1 || { tail -50 test-results/perf/seed.log; exit 1; }
tail -8 test-results/perf/seed.log
campaign="$(psql -XAt -v ON_ERROR_STOP=1 -c "
SELECT c.id FROM campaigns AS c JOIN runs AS r ON r.campaign_id = c.id
WHERE c.backend = 'benchmark' AND c.status = 'COMPLETED'
GROUP BY c.id
HAVING count(*) = 10000 AND count(*) FILTER (WHERE r.status = 'DONE') = 10000;")"
[[ "$campaign" =~ ^[0-9a-f-]{36}$ ]]
echo "ROBOPROBE_PERF_CAMPAIGN_ID=$campaign" >> "$GITHUB_ENV"
- name: Explain the runs-list keyset query
run: |
set -euo pipefail
psql -X -v ON_ERROR_STOP=1 -v campaign_id="$ROBOPROBE_PERF_CAMPAIGN_ID" \
-f tests/perf/explain_runs.sql | tee test-results/perf/runs-explain.txt
- name: Run Locust and enforce per-endpoint p95 budgets
run: |
set -euo pipefail
uv run --locked roboprobe api > test-results/perf/api.log 2>&1 &
server=$!
trap 'kill "$server" 2>/dev/null || true' EXIT
host="http://$ROBOPROBE_API__HOST:$ROBOPROBE_API__PORT"
curl -fsS --retry 30 --retry-delay 1 --retry-connrefused "$host/health/ready"
locust_exit=0
uvx --python "$PYTHON_VERSION" "locust==$LOCUST_VERSION" \
--headless -u 20 -r 5 -t 60s --only-summary --host "$host" \
-f tests/perf/locustfile.py --csv test-results/perf/perf || locust_exit=$?
mv test-results/perf/perf_complete_stats.csv test-results/perf/perf_stats.csv
uv run --locked python tests/perf/check_stats.py test-results/perf/perf_stats.csv
exit "$locust_exit"
- name: Upload performance evidence
if: ${{ !cancelled() }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: api-performance
path: backend/test-results/perf/
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,9 @@ jobs:
- name: tsc -b && vite build
working-directory: web
run: pnpm build
- name: Initial JS bundle budget (T-114)
working-directory: web
run: pnpm size-limit

config-docs-check:
name: config-docs-check
Expand Down
148 changes: 71 additions & 77 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
@@ -1,97 +1,91 @@
name: API performance
# Playwright E2E against compose core+app (T-117, T-118, T-114).
name: e2e

on:
schedule:
- cron: "17 2 * * *"
workflow_dispatch:
pull_request:
paths:
- 'web/**'
- 'backend/src/roboprobe/api/**'
- 'docker/**'
- 'scripts/e2e_stack.sh'
- '.github/workflows/e2e.yml'
- 'backend/tests/perf/**'
schedule:
- cron: '30 20 * * *' # 02:00 IST
workflow_dispatch:

permissions:
contents: read

concurrency:
group: e2e-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

env:
PYTHON_VERSION: '3.12'
NODE_VERSION: '24'
E2E_BASE_URL: http://127.0.0.1:8080

jobs:
api-perf:
name: Locust on 10,000 runs
playwright:
name: playwright
runs-on: ubuntu-latest
timeout-minutes: 30
services:
postgres:
image: postgres:17
env:
POSTGRES_USER: roboprobe
POSTGRES_PASSWORD: roboprobe-ci
POSTGRES_DB: roboprobe_perf
ports: ["5432:5432"]
options: >-
--health-cmd "pg_isready -U roboprobe -d roboprobe_perf"
--health-interval 5s --health-timeout 5s --health-retries 10
env:
PYTHON_VERSION: '3.12'
LOCUST_VERSION: '2.47.0'
OPENBLAS_NUM_THREADS: '1'
PGHOST: 127.0.0.1
PGUSER: roboprobe
PGPASSWORD: roboprobe-ci
PGDATABASE: roboprobe_perf
ROBOPROBE_ENV: ci
ROBOPROBE_DATABASE__URL: postgresql+psycopg://roboprobe:roboprobe-ci@127.0.0.1:5432/roboprobe_perf
ROBOPROBE_API__HOST: 127.0.0.1
ROBOPROBE_API__PORT: '8000'
defaults:
run:
working-directory: backend
timeout-minutes: 25
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: ./.github/actions/setup-backend
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Seed and verify 10,000 completed runs
- uses: ./.github/actions/setup-web
with:
node-version: ${{ env.NODE_VERSION }}
- name: Install Playwright browsers
working-directory: web
run: pnpm exec playwright install --with-deps chromium firefox
- name: Stack up, seed DB, run Playwright
run: |
set -euo pipefail
mkdir -p test-results/perf
export ROBOPROBE_STORAGE__WORKDIR_ROOT="$RUNNER_TEMP/roboprobe-perf-work"
export ROBOPROBE_STORAGE__ARTIFACT_ROOT="$RUNNER_TEMP/roboprobe-perf-artifacts"
printf '%s\n' "ROBOPROBE_STORAGE__WORKDIR_ROOT=$ROBOPROBE_STORAGE__WORKDIR_ROOT" \
"ROBOPROBE_STORAGE__ARTIFACT_ROOT=$ROBOPROBE_STORAGE__ARTIFACT_ROOT" >> "$GITHUB_ENV"
uv run --locked python ../scripts/seed_demo.py --size large \
--database-url "$ROBOPROBE_DATABASE__URL" \
--workdir-root "$ROBOPROBE_STORAGE__WORKDIR_ROOT" \
--artifact-root "$ROBOPROBE_STORAGE__ARTIFACT_ROOT" \
> test-results/perf/seed.log 2>&1 || { tail -50 test-results/perf/seed.log; exit 1; }
tail -8 test-results/perf/seed.log
campaign="$(psql -XAt -v ON_ERROR_STOP=1 -c "
SELECT c.id FROM campaigns AS c JOIN runs AS r ON r.campaign_id = c.id
WHERE c.backend = 'benchmark' AND c.status = 'COMPLETED'
GROUP BY c.id
HAVING count(*) = 10000 AND count(*) FILTER (WHERE r.status = 'DONE') = 10000;")"
[[ "$campaign" =~ ^[0-9a-f-]{36}$ ]]
echo "ROBOPROBE_PERF_CAMPAIGN_ID=$campaign" >> "$GITHUB_ENV"
- name: Explain the runs-list keyset query
make setup
cd web && pnpm exec playwright test --project=chromium
- name: Playwright summary
if: always()
working-directory: web
run: |
set -euo pipefail
psql -X -v ON_ERROR_STOP=1 -v campaign_id="$ROBOPROBE_PERF_CAMPAIGN_ID" \
-f tests/perf/explain_runs.sql | tee test-results/perf/runs-explain.txt
- name: Run Locust and enforce per-endpoint p95 budgets
if [[ -d playwright-report ]]; then
echo "### Playwright report" >>"$GITHUB_STEP_SUMMARY"
echo "Artefacts uploaded as playwright-report" >>"$GITHUB_STEP_SUMMARY"
fi
- name: Upload Playwright report
if: failure()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: playwright-report
path: |
web/playwright-report/
web/e2e-results/
web/e2e-artifacts/
if-no-files-found: ignore

lighthouse:
name: lighthouse
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
python-version: ${{ env.PYTHON_VERSION }}
- uses: ./.github/actions/setup-web
with:
node-version: ${{ env.NODE_VERSION }}
- name: Lighthouse on seeded stack
run: |
set -euo pipefail
uv run --locked roboprobe api > test-results/perf/api.log 2>&1 &
server=$!
trap 'kill "$server" 2>/dev/null || true' EXIT
host="http://$ROBOPROBE_API__HOST:$ROBOPROBE_API__PORT"
curl -fsS --retry 30 --retry-delay 1 --retry-connrefused "$host/health/ready"
locust_exit=0
uvx --python "$PYTHON_VERSION" "locust==$LOCUST_VERSION" \
--headless -u 20 -r 5 -t 60s --only-summary --host "$host" \
-f tests/perf/locustfile.py --csv test-results/perf/perf || locust_exit=$?
mv test-results/perf/perf_complete_stats.csv test-results/perf/perf_stats.csv
uv run --locked python tests/perf/check_stats.py test-results/perf/perf_stats.csv
exit "$locust_exit"
- name: Upload performance evidence
if: ${{ !cancelled() }}
make setup
bash scripts/e2e_stack.sh up
cd web && pnpm exec lhci autorun --config=lighthouserc.live.json
- name: Collect compose logs
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: api-performance
path: backend/test-results/perf/
name: e2e-compose-logs
path: web/e2e-artifacts/compose.log
if-no-files-found: ignore
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ pnpm-debug.log*
yarn-error.log*
playwright-report/
test-results/
web/e2e-results/
web/e2e-artifacts/

# --- ROS 2 / colcon ---
ros_ws/build/
Expand Down
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ versioning: [Semantic Versioning 2.0.0](https://semver.org/) (policy in `docs/33

### Added

- Playwright E2E harness and journeys (T-117, T-118, T-114): compose-backed global setup with
seeded warehouse campaign, axe helper and page objects, journey specs, `e2e.yml`, Lighthouse
nightly gate, and `size-limit` bundle budget in `build-web`.
- Reproducible headless Gazebo and Nav2 warehouse measurements with a one-worker laptop recommendation (T-090).
- Nightly API load test (T-106) checks endpoint p95 with a 10,000-run PostgreSQL seed and uploads the Locust CSV and runs-list query plan.
- Dashboard polish (T-113a/b, T-115): Spaces list with YAML view and validation,
Expand Down
6 changes: 6 additions & 0 deletions docker/e2e.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# E2E stack overrides (T-117). Composed with repo `.env` when present; tunables live in docs/27.
ROBOPROBE_K2D_REPLICAS=2
ROBOPROBE_RUNNER__MAX_CONCURRENT_CAMPAIGNS=2
ROBOPROBE_API__HOST=0.0.0.0
ROBOPROBE_WEB_PORT=8080
ROBOPROBE_DB_PORT=5432
2 changes: 1 addition & 1 deletion docs/18-testing-strategy-and-qa.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ API responses against the committed OpenAPI schema and fixture JSON.
| SEC-001/002 | bind-address test; `pip-audit`, `pnpm audit`, gitleaks, Trivy image scan |
| ACC-001 | axe-core in Playwright |

The [API performance workflow](../.github/workflows/e2e.yml) runs nightly and on changes to the
The [API performance workflow](../.github/workflows/api-perf.yml) runs nightly and on changes to the
load harness. It starts with a fresh PostgreSQL database, verifies all 10,000 benchmark runs
completed, then runs Locust headless for 60 seconds with 20 users and a spawn rate of 5 per second.
The workflow checks p95 per endpoint and uploads the CSV, API log and a five-page keyset query plan.
Expand Down
2 changes: 1 addition & 1 deletion docs/evidence/T-106-perf.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ Execution Time: 0.040 ms

## Reproduction

Run the [API performance workflow](../../.github/workflows/e2e.yml) with a fresh PostgreSQL 17
Run the [API performance workflow](../../.github/workflows/api-perf.yml) with a fresh PostgreSQL 17
service, or follow the same steps against a disposable local database. The seed command rejects
databases with existing application data.

Expand Down
29 changes: 29 additions & 0 deletions docs/evidence/T-114-a11y.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# T-114 — Accessibility & performance evidence

| Tool | Version (pinned in `web/package.json`) |
|------|----------------------------------------|
| Playwright | 1.65.x |
| axe-core (`@axe-core/playwright`) | 4.14.x |
| Lighthouse CI | 0.14.x |
| size-limit | 11.2.x |

## Automated gates

- **axe:** `web/e2e/specs/a11y-sweep.spec.ts` visits every MVP route in Paper and Night Shift themes;
failures on serious/critical impacts (colour-contrast disabled for token v1 parity with Vitest).
- **Keyboard:** each route receives a 12-step Tab walk with visible focus.
- **Lighthouse:** nightly `e2e.yml` job `lighthouse` scores accessibility ≥ 0.95 on `/`, the seeded
campaign console, and a medoid run entry (`lighthouserc.live.json` thresholds).
- **Bundle:** `build-web` runs `pnpm size-limit` with gzip budget 350 KB on the initial JS chunk
(`.size-limit.json`, doc 13 §7).

## Manual spot-check (2026-11-22)

- **Keyboard:** `g b`, `g c`, `/`, `j/k`, `r`, `?` chords open Bench, campaign list, search, stream
navigation, replay drawer, and the shortcuts dialog without traps on Chrome and Firefox.
- **Screen reader:** VoiceOver (macOS) announces Bench headings, campaign progress `progressbar`,
and Casebook mode cards; NVDA not re-run on this change set (prior M8 passes only).

Assumption: Plotly-backed Atlas/Insights routes are covered by axe with the shared colour-contrast
rule disabled until token v2 contrast work lands; Lighthouse still enforces the 0.95 accessibility
category score on shell routes.
Loading
Loading