Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@
!docker/web/
**/node_modules/
web/dist/
web/coverage/
web/playwright-report/
web/test-results/
web/e2e-results/
web/e2e-artifacts/
web/.lighthouseci/
**/__pycache__/
**/*.py[cod]
**/.DS_Store
49 changes: 24 additions & 25 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ on:
paths:
- 'web/**'
- 'backend/src/roboprobe/api/**'
- 'backend/alembic/**'
- 'backend/tests/fixtures/db/**'
- 'docker/**'
- 'scripts/e2e_stack.sh'
- '.github/workflows/e2e.yml'
Expand All @@ -21,48 +23,50 @@ concurrency:
cancel-in-progress: true

env:
PYTHON_VERSION: '3.12'
NODE_VERSION: '24'
E2E_BASE_URL: http://127.0.0.1:8080
E2E_COMPOSE_PROJECT: roboprobe-e2e-${{ github.run_id }}-${{ github.run_attempt }}

jobs:
playwright:
name: playwright
name: Playwright (${{ matrix.browser }})
runs-on: ubuntu-latest
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
include:
- browser: chromium
spec: ''
- browser: firefox
spec: smoke.spec.ts
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
python-version: ${{ env.PYTHON_VERSION }}
- uses: ./.github/actions/setup-web
with:
node-version: ${{ env.NODE_VERSION }}
- name: Install Playwright browsers
working-directory: web
run: pnpm exec playwright install --with-deps chromium firefox
run: pnpm exec playwright install --with-deps ${{ matrix.browser }}
- name: Stack up, seed DB, run Playwright
run: |
make setup
cd web && pnpm exec playwright test --project=chromium
- name: Playwright summary
if: always()
working-directory: web
run: |
if [[ -d playwright-report ]]; then
echo "### Playwright report" >>"$GITHUB_STEP_SUMMARY"
echo "Artefacts uploaded as playwright-report" >>"$GITHUB_STEP_SUMMARY"
fi
run: pnpm exec playwright test --project=${{ matrix.browser }} ${{ matrix.spec }}
- name: Upload Playwright report
if: failure()
id: reports
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: playwright-report
name: playwright-${{ matrix.browser }}-report
path: |
web/playwright-report/
web/e2e-results/
web/e2e-artifacts/
if-no-files-found: ignore
- name: Link Playwright report
if: ${{ failure() && steps.reports.outputs.artifact-url != '' }}
env:
ARTIFACT_URL: ${{ steps.reports.outputs.artifact-url }}
run: printf '[Download the browser report and Compose logs](%s)\n' "$ARTIFACT_URL" >> "$GITHUB_STEP_SUMMARY"

lighthouse:
name: lighthouse
Expand All @@ -71,21 +75,16 @@ jobs:
timeout-minutes: 20
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
python-version: ${{ env.PYTHON_VERSION }}
- uses: ./.github/actions/setup-web
with:
node-version: ${{ env.NODE_VERSION }}
- name: Lighthouse on seeded stack
run: |
make setup
bash scripts/e2e_stack.sh up
cd web && pnpm exec lhci autorun --config=lighthouserc.live.json
working-directory: web
run: bash ../scripts/e2e_stack.sh pnpm exec lhci autorun --config=lighthouserc.live.json
- name: Collect compose logs
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: e2e-compose-logs
path: web/e2e-artifacts/compose.log
if-no-files-found: ignore
if-no-files-found: ignore
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,10 @@ versioning: [Semantic Versioning 2.0.0](https://semver.org/) (policy in `docs/33

### Fixed

- Campaign reports render legacy risk slices, and authenticated dashboard sessions can preview
and download completed reports.
- Isolate seeded Playwright and Lighthouse runs, and report stale scenario versions as validation
errors during campaign creation (T-117).
- Embed the API comparison payload in offline experiment reports and surface restart-interrupted
report jobs as errors (T-084a).
- Casebook interaction checks, replays, and Atlas manual probes run after campaign completion.
Expand Down
5 changes: 5 additions & 0 deletions backend/src/roboprobe/api/campaign_create.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,11 @@ def resolve_campaign_space(ref: SpaceRef, *, uow: Any, root: Path) -> ScenarioSp
space = ScenarioSpace.from_text(
yaml.safe_dump(existing.definition), source="registered space", base_dir=root
)
if (space.id, space.version) != (space_id, ref.version):
msg = "Task files changed for this space version. Register the current definition."
raise ValidationError(
msg, context={"errors": [{"path": "/space/version", "message": msg}]}
)
else:
space_yaml = _config_path(root, "scenario_spaces", space_id)
if not space_yaml.is_file():
Expand Down
51 changes: 46 additions & 5 deletions backend/src/roboprobe/reporting/campaign.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,16 @@
from roboprobe.metrics.discovery_metrics import discovery_curve
from roboprobe.reporting import figures
from roboprobe.reporting._env import template_env
from roboprobe.reporting.figures import _ImportanceLike
from roboprobe.reporting.figures import _ImportanceLike, _RiskSlicePlot
from roboprobe.reporting.ports import ReportArtifactStore, ReportUnitOfWork
from roboprobe.reporting.theme import load_theme, tokens_path
from roboprobe.risk.mapper import RiskSlice, load_slices
from roboprobe.risk.mapper import (
REGION_FAILURE,
REGION_SAFE,
REGION_UNCERTAIN,
RiskSlice,
load_slices,
)
from roboprobe.risk.volume import RegionVolumes, robustness_score

__all__ = ["CampaignReportContext", "render_campaign_report"]
Expand Down Expand Up @@ -135,9 +141,38 @@ def _volumes_from_summary(summary: Mapping[str, Any]) -> RegionVolumes | None:
return None


def _legacy_slice_plots(payload: object) -> list[_RiskSlicePlot]:
"""Read only the chart coordinates present in JSON fixture and legacy records."""
if not isinstance(payload, list):
return []
plots: list[_RiskSlicePlot] = []
for row in payload:
if not isinstance(row, dict):
continue
x_param, y_param = row.get("x_param"), row.get("y_param")
if not isinstance(x_param, str) or not isinstance(y_param, str):
continue
x = np.asarray(row.get("x_values"), dtype=np.float64)
y = np.asarray(row.get("y_values"), dtype=np.float64)
region = np.asarray(row.get("region"), dtype=np.float64)
if (
x.ndim != 1
or y.ndim != 1
or not x.size
or not y.size
or not np.isfinite(x).all()
or not np.isfinite(y).all()
or region.shape != (x.size, y.size)
or not np.isin(region, (REGION_SAFE, REGION_UNCERTAIN, REGION_FAILURE)).all()
):
continue
plots.append(_RiskSlicePlot(x_param, y_param, x, y, region.astype(np.int8)))
return plots


def _load_slices_from_store(
store: ReportArtifactStore, uow: ReportUnitOfWork, campaign_id: UUID
) -> list[RiskSlice]:
) -> Sequence[RiskSlice | _RiskSlicePlot]:
latest = uow.risk_maps.latest(campaign_id)
if latest is None or latest.slices_artifact is None:
return []
Expand All @@ -149,9 +184,15 @@ def _load_slices_from_store(
blob = store.path(ref)
except Exception:
return []
if not Path(blob).is_file():
try:
with Path(blob).open("rb") as stream:
prefix = stream.read(2)
stream.seek(0)
if prefix == b"PK":
return load_slices(stream)
return _legacy_slice_plots(json.load(stream))
except (OSError, KeyError, TypeError, ValueError):
return []
return load_slices(Path(blob))


def build_campaign_context( # noqa: PLR0915
Expand Down
21 changes: 16 additions & 5 deletions backend/src/roboprobe/reporting/figures.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from __future__ import annotations

from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from io import StringIO
from typing import TYPE_CHECKING, Any, Protocol

Expand All @@ -30,6 +31,17 @@ class _ImportanceLike(Protocol):
mean_abs_shap: Sequence[float]


@dataclass(frozen=True, slots=True)
class _RiskSlicePlot:
"""Axis and region data needed to render a persisted slice."""

x_param: str
y_param: str
x_values: npt.NDArray[np.float64]
y_values: npt.NDArray[np.float64]
region: npt.NDArray[np.int8]


__all__ = [
"discovery_curve_svg",
"discovery_curve_with_ci_svg",
Expand Down Expand Up @@ -309,7 +321,7 @@ def interaction_heatmap_svg(


def risk_slice_svg(
slice_: RiskSlice | None,
slice_: RiskSlice | _RiskSlicePlot | None,
*,
settings: ReportingSettings | None = None,
) -> str:
Expand All @@ -335,11 +347,10 @@ def risk_slice_svg(
)
uncertain = slice_.region == REGION_UNCERTAIN
if np.any(uncertain):
yy, xx = np.meshgrid(slice_.y_values, slice_.x_values, indexing="ij")
ax.contourf(
xx,
yy,
uncertain.astype(np.float64),
slice_.x_values,
slice_.y_values,
uncertain.T.astype(np.float64),
levels=[0.5, 1.5],
colors=["none"],
hatches=["//"],
Expand Down
24 changes: 24 additions & 0 deletions backend/tests/api/test_campaigns_crud.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

from dataclasses import replace
from pathlib import Path
from typing import Any
from uuid import UUID
Expand Down Expand Up @@ -60,6 +61,29 @@ def test_create_returns_location_and_snapshot(client: TestClient) -> None:
assert "allow_repeats" in response.json()["strategy"]["params"]


def test_campaign_create_rejects_drifted_registered_space(client: TestClient) -> None:
spaces = "/api/v1/spaces"
config = discover_repo_root() / "configs/scenario_spaces/warehouse_v1.yaml"
registered = client.post(
spaces, content=config.read_bytes(), headers={"Content-Type": "application/yaml"}
)
assert registered.status_code == 201, registered.text
space = registered.json()
previous = "0123456789ab"
assert space["version"] != previous
with client.app.state.services.uow_factory() as uow:
record = uow.spaces.get(space["id"], space["version"])
assert record is not None
uow.spaces.add(replace(record, version=previous))
uow.commit()

assert client.get(f"{spaces}/{space['id']}/versions/{previous}").json()["version"] == previous
response = client.post(API, json=body(space={"id": space["id"], "version": previous}))
assert response.status_code == 422, response.text
assert response.json()["errors"][0]["pointer"] == "/space/version"
assert not client.get(API).json()["items"]


def test_start_true_commits_before_lifecycle(client: TestClient) -> None:
response = client.post(API, json=body(start=True))
assert response.status_code == 201, response.text
Expand Down
26 changes: 26 additions & 0 deletions backend/tests/unit/reporting/test_campaign_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

from __future__ import annotations

import json
import re
from dataclasses import dataclass, field
from html.parser import HTMLParser
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from uuid import UUID, uuid4

Expand Down Expand Up @@ -118,6 +120,30 @@ def test_render_campaign_report_offline_html(tmp_path: Path) -> None:
assert parser.external == []


def test_report_renders_legacy_json_risk_regions(tmp_path: Path) -> None:
"""Older records still supply a plotted risk slice."""
risk = tmp_path / "risk.json"
risk.write_text(
json.dumps(
[
{
"x_param": "robot.max_speed",
"y_param": "env.illuminance",
"x_values": [0.1, 0.2],
"y_values": [1.0, 2.0, 3.0],
"region": [[0, 1, 2], [0, 0, 1]],
}
]
),
encoding="utf-8",
)
uow = _FakeUow()
uow.risk_maps = SimpleNamespace(latest=lambda _id: SimpleNamespace(slices_artifact="risk"))
uow.artifacts = SimpleNamespace(get=lambda _id: SimpleNamespace(sha256="0" * 64, rel_path=risk))
out = render_campaign_report(CID, uow, _FakeStore(), out=tmp_path / "report.html")
assert "robot.max_speed x env.illuminance" in out.read_text(encoding="utf-8")


def test_xss_escaped_mode_label(tmp_path: Path) -> None:
uow = _FakeUow()
uow.modes = type(
Expand Down
27 changes: 27 additions & 0 deletions docker/e2e.compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Playwright overrides. Docker Compose resolves these paths relative to compose.yaml.
x-e2e-backend: &e2e-backend
env_file: !override [./e2e.env]
volumes:
- e2e-artifacts:/data/artifacts

services:
migrate:
<<: *e2e-backend
orchestrator:
<<: *e2e-backend
worker-k2d:
<<: *e2e-backend
api:
<<: *e2e-backend
secrets: [api_token]
web:
env_file: !override [./e2e.env]

secrets:
db_password:
file: ${E2E_DB_PASSWORD_FILE:-/dev/null}
api_token:
file: ${E2E_API_TOKEN_FILE:-/dev/null}

volumes:
e2e-artifacts: {}
13 changes: 9 additions & 4 deletions docker/e2e.env
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
# E2E stack overrides (T-117). Composed with repo `.env` when present; tunables live in docs/27.
# Local bindings and runtime settings for the isolated Playwright Compose project.
ROBOPROBE_WEB_PORT=8080
ROBOPROBE_K2D_REPLICAS=2
ROBOPROBE_DB_NAME=roboprobe_e2e
ROBOPROBE_STORAGE__ARTIFACT_ROOT=/data/artifacts/e2e
ROBOPROBE_STORAGE__WORKDIR_ROOT=/tmp/roboprobe-work
ROBOPROBE_RUNNER__MAX_BATCH_SIZE=4
ROBOPROBE_RUNNER__MAX_CONCURRENT_CAMPAIGNS=2
ROBOPROBE_API__HOST=0.0.0.0
ROBOPROBE_WEB_PORT=8080
ROBOPROBE_DB_PORT=5432
ROBOPROBE_RUNNER__MAX_MANUAL_REPEATS=5
OMP_NUM_THREADS=1
MKL_NUM_THREADS=1
Loading
Loading