From a4fe3965d46c983ab7012df1d5283bd8d1081f2b Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 22 Sep 2026 17:29:48 -0700 Subject: [PATCH 1/2] chore(testlib): Make host test names mirror in-worker ones Register each in-worker suite on the host under its file name and mirror in-worker test classes as nested classes. This is intended so that passing a `-k` filter picks up the same tests both inside and outside of the worker. --- AGENTS.md | 6 ++ packages/runtime-sdk/tests/conftest.py | 7 +- packages/testlib/AGENTS.md | 37 +++++++++ packages/testlib/testlib/host.py | 103 ++++++++++++++++--------- 4 files changed, 109 insertions(+), 44 deletions(-) create mode 100644 packages/testlib/AGENTS.md diff --git a/AGENTS.md b/AGENTS.md index 72a78265..e6b32e81 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,8 @@ This repository (`workers-py`) contains three Python packages that are used for - `packages/runtime-sdk` contains the runtime SDK for Python Workers, which provides a base class for Python Workers and utilities for working with Cloudflare's runtime. - `packages/django-cf` is the Django integration package, providing database backends for D1 and Durable Objects, an R2 storage backend, and Cloudflare Access middleware. +There is also an unpublished `packages/testlib`, shared by the `runtime-sdk` and `django-cf` test suites for running pytest inside workerd and reporting the results as host-side tests. + ### `packages/cli` For cli conventions, see `packages/cli/AGENTS.md`. @@ -28,6 +30,10 @@ For runtime-sdk conventions, see `packages/runtime-sdk/AGENTS.md`. For django-cf conventions, see `packages/django-cf/AGENTS.md`. +### `packages/testlib` + +For how in-worker test suites are exposed as host tests, see `packages/testlib/AGENTS.md`. + ## Build System & Commands This project uses the following tools to manage the build process: diff --git a/packages/runtime-sdk/tests/conftest.py b/packages/runtime-sdk/tests/conftest.py index 7e2c0c01..dbaadc3c 100644 --- a/packages/runtime-sdk/tests/conftest.py +++ b/packages/runtime-sdk/tests/conftest.py @@ -101,9 +101,4 @@ def register_in_worker_suites( src_dir: Path, marks: dict[str, pytest.MarkDecorator] | None = None, ) -> None: - register_testlib_suites( - namespace, - src_dir, - marks=marks, - class_name=str.upper, - ) + register_testlib_suites(namespace, src_dir, marks=marks) diff --git a/packages/testlib/AGENTS.md b/packages/testlib/AGENTS.md new file mode 100644 index 00000000..ac5e4362 --- /dev/null +++ b/packages/testlib/AGENTS.md @@ -0,0 +1,37 @@ +# packages/testlib + +## Overview + +`testlib` holds shared helpers for tests that run pytest *inside* workerd. It is +not published; `runtime-sdk` and `django-cf` vendor it into their test workers +via `../packages/testlib` in `[tool.uv.sources]`. + +## Key modules + +| Module | Runs on | Purpose | +|---|---|---| +| `testlib/host.py` | host | `dev_server`, `pywrangler_sync`, `register_in_worker_suites` | +| `testlib/entrypoint.py` | worker | `TestRunner`, `TestRunnerEntrypoint` (`/run-tests/`, `/health`), `ResultCollector` | +| `testlib/tracebacks.py` | both | Pickle worker exceptions and remap their frames onto host source roots | + +## How in-worker suites are exposed on the host + +- A worker project has `src/test_.py` modules. The worker serves + `GET /run-tests/`, runs that module with `pytest.main` and returns + per-test JSON results keyed by `ResultCollector._key` (`Class__name`). +- `register_in_worker_suites(globals(), src_dir)` in a host test module parses + each `src/test_.py` with `ast` and generates one host test per + in-worker test. Host node IDs mirror the in-worker ones: a class registered + under the key `test_kv.py` (collected thanks to `__test__ = True`), with + in-worker classes mirrored as nested classes, e.g. + `tests/test_bindings.py::test_kv.py::TestFoo::test_bar[3.12]`. +- The suite is run once per `dev_server` (`functools.cache` on + `get_suite_results`); each host test just looks up its result. + +## Conventions and pitfalls + +- Keep `host._result_key` and `entrypoint.ResultCollector._key` in sync. +- Python 3.12 (Pyodide 0.26.0a2) reports false passes for async in-worker + tests; verify failure behaviour on 3.13+. +- `pywrangler sync` in tests may need `UV_NATIVE_TLS=1` on hosts with custom + CA certificates. diff --git a/packages/testlib/testlib/host.py b/packages/testlib/testlib/host.py index 8aff536e..1edacea1 100644 --- a/packages/testlib/testlib/host.py +++ b/packages/testlib/testlib/host.py @@ -233,17 +233,19 @@ def get_suite_results(server: str, suite: str) -> SuiteResults | str: def _make_test( - suite: str, test_name: str, source_roots: list[Path] | None = None + suite: str, result_key: str, name: str, source_roots: Sequence[Path] = () ) -> Callable: + """Build a host test method that reports the in-worker result *result_key*.""" + def test_fn(self: Any, dev_server: str) -> None: # Hide this frame: the interesting traceback is the one from the worker. __tracebackhide__ = True results = get_suite_results(dev_server, suite) if isinstance(results, str): pytest.fail(results) - result = results.get(test_name) + result = results.get(result_key) assert result is not None, ( - f"Test {suite}::{test_name} not found in results; " + f"Test {suite}::{result_key} not found in results; " f"available keys: {sorted(results)}" ) if result["status"] == "skipped": @@ -253,40 +255,73 @@ def test_fn(self: Any, dev_server: str) -> None: exception = result.get("exception") if exception is None: pytest.fail(f"{result['error']}\n{result.get('traceback', '')}".rstrip()) - if source_roots is None: - source_roots_ = [] - else: - source_roots_ = source_roots - source_roots_.append(WORKERS_RUNTIME_SDK) - exc = load_exception(exception, source_roots) + exc = load_exception(exception, [*source_roots, WORKERS_RUNTIME_SDK]) when = exception.get("when", "call") if when != "call": exc.add_note(f"raised in the worker during test {when}") raise exc - test_fn.__name__ = f"test_{test_name}" + test_fn.__name__ = name return test_fn -def _normalize_test_name(*parts: str) -> str: +def _result_key(*parts: str) -> str: + """Key under which ``ResultCollector`` (worker side) records a test. + + Must stay in sync with ``testlib.entrypoint.ResultCollector._key``. + """ return "__".join(part.removeprefix("test_") for part in parts) -def discover_test_names(module_path: Path) -> list[str]: +def _is_test_def(node: ast.AST) -> bool: + return isinstance( + node, ast.FunctionDef | ast.AsyncFunctionDef + ) and node.name.startswith("test_") + + +def discover_tests(module_path: Path) -> list[tuple[str | None, str]]: + """Return ``(class_name, function_name)`` for each test in *module_path*. + + ``class_name`` is ``None`` for module-level test functions. + """ tree = ast.parse(module_path.read_text()) - names = [] + tests: list[tuple[str | None, str]] = [] for node in tree.body: - if isinstance( - node, ast.FunctionDef | ast.AsyncFunctionDef - ) and node.name.startswith("test_"): - names.append(_normalize_test_name(node.name)) + if _is_test_def(node): + tests.append((None, node.name)) elif isinstance(node, ast.ClassDef): - for child in node.body: - if isinstance( - child, ast.FunctionDef | ast.AsyncFunctionDef - ) and child.name.startswith("test_"): - names.append(_normalize_test_name(node.name, child.name)) # noqa: PERF401 - return names + tests.extend( + (node.name, child.name) for child in node.body if _is_test_def(child) + ) + return tests + + +def _make_suite_class( + module_path: Path, suite: str, source_roots: Sequence[Path] +) -> type: + """Build a host class mirroring the structure of the in-worker test module. + + Module-level in-worker tests become methods; in-worker test classes become + nested classes with the same names, so the host node IDs mirror the + in-worker ones (``test_kv.py::TestFoo::test_bar``) and ``-k`` expressions + select the same tests on both sides. + """ + # ``__test__ = True`` makes pytest collect the class even though its name + # (``test_kv.py``) doesn't match ``python_classes``. + members: dict[str, Any] = {"__test__": True} + nested: dict[str, dict[str, Any]] = {} + for class_name, function_name in discover_tests(module_path): + if class_name is None: + key = _result_key(function_name) + members[function_name] = _make_test(suite, key, function_name, source_roots) + else: + key = _result_key(class_name, function_name) + nested.setdefault(class_name, {"__test__": True})[function_name] = ( + _make_test(suite, key, function_name, source_roots) + ) + for class_name, class_members in nested.items(): + members[class_name] = type(class_name, (), class_members) + return type(module_path.name, (), members) def register_in_worker_suites( @@ -294,11 +329,15 @@ def register_in_worker_suites( src_dir: Path, *, marks: dict[str, pytest.MarkDecorator] | None = None, - class_name: Callable[[str], str] | None = None, source_roots: Sequence[Path] = (), ) -> None: """Expose each in-worker test as an individual host-side pytest test. + Each ``test_.py`` in *src_dir* is registered in *namespace* under + its file name, so host node IDs mirror the in-worker ones, e.g. + ``tests/test_bindings.py::test_kv.py::test_get[3.12]`` for the in-worker + ``test_kv.py::test_get``. + ``source_roots`` lists extra host directories (besides ``src_dir``) that hold copies of code running inside the worker, e.g. a package's source tree that gets vendored into ``python_modules``. Traceback frames from @@ -307,19 +346,7 @@ def register_in_worker_suites( roots = (src_dir, *source_roots) for module_path in sorted(src_dir.glob("test_*.py")): suite = module_path.stem[len("test_") :] - generated_class_name = ( - class_name(suite) - if class_name - else "".join(part.title() for part in suite.split("_")) - ) - suite_cls = type( - f"Test{generated_class_name}", - (), - { - f"test_{name}": _make_test(suite, name, roots) - for name in discover_test_names(module_path) - }, - ) + suite_cls = _make_suite_class(module_path, suite, roots) if marks and suite in marks: suite_cls = marks[suite](suite_cls) - namespace[suite_cls.__name__] = suite_cls + namespace[module_path.name] = suite_cls From 77fab85359778f216c85cfddeb3bed9b13417e7b Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 22 Sep 2026 17:30:10 -0700 Subject: [PATCH 2/2] chore(testlib): Forward host pytest arguments to the in-worker pytest run The host sends its invocation arguments as repeated `?arg=` query parameters on `/run-tests/`, and `TestRunner.run_suite` appends them to `pytest.main`. So `-x`, `-v`, `--tb`, `-W`, `-k` etc. now apply inside the worker too. For `-k`, the host also sends the keywords that only exist on the host as `?kw=`; `ExtraKeywordsPlugin` attaches them to the in-worker items so the same expression selects the same tests on both sides. Tge FastAPI test worker uses `TestRunner` directly so it builds a `RunSuiteRequest` from the query string. --- AGENTS.md | 2 +- .../fastapi-tests/src/worker.py | 6 +- packages/testlib/AGENTS.md | 19 ++- packages/testlib/testlib/entrypoint.py | 65 ++++++++- packages/testlib/testlib/host.py | 123 +++++++++++++++++- 5 files changed, 200 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e6b32e81..bde24088 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,7 +32,7 @@ For django-cf conventions, see `packages/django-cf/AGENTS.md`. ### `packages/testlib` -For how in-worker test suites are exposed as host tests, see `packages/testlib/AGENTS.md`. +For how in-worker test suites are exposed as host tests and how pytest arguments are forwarded into the worker, see `packages/testlib/AGENTS.md`. ## Build System & Commands diff --git a/packages/runtime-sdk/tests/web-frameworks-test/fastapi-tests/src/worker.py b/packages/runtime-sdk/tests/web-frameworks-test/fastapi-tests/src/worker.py index 45de65bb..07985850 100644 --- a/packages/runtime-sdk/tests/web-frameworks-test/fastapi-tests/src/worker.py +++ b/packages/runtime-sdk/tests/web-frameworks-test/fastapi-tests/src/worker.py @@ -32,7 +32,7 @@ from pydantic import BaseModel from starlette.background import BackgroundTask from starlette.middleware.gzip import GZipMiddleware -from testlib.entrypoint import TestRunner +from testlib.entrypoint import RunSuiteRequest, TestRunner import asgi @@ -594,7 +594,9 @@ def fastapi_app(self): @app.get("/run-tests/{suite_name:path}") async def run_suite(suite_name: str, request: Request): runner = TestRunner(request.scope["env"], extra_plugins=[FastAPIAppPlugin()]) - result = runner.run_suite(suite_name) + result = runner.run_suite( + suite_name, RunSuiteRequest.from_query_params(request.query_params) + ) return JSONResponse(result.payload, status_code=result.status) diff --git a/packages/testlib/AGENTS.md b/packages/testlib/AGENTS.md index ac5e4362..235d48b5 100644 --- a/packages/testlib/AGENTS.md +++ b/packages/testlib/AGENTS.md @@ -10,7 +10,7 @@ via `../packages/testlib` in `[tool.uv.sources]`. | Module | Runs on | Purpose | |---|---|---| -| `testlib/host.py` | host | `dev_server`, `pywrangler_sync`, `register_in_worker_suites` | +| `testlib/host.py` | host | `dev_server`, `pywrangler_sync`, `register_in_worker_suites`, arg/keyword forwarding | | `testlib/entrypoint.py` | worker | `TestRunner`, `TestRunnerEntrypoint` (`/run-tests/`, `/health`), `ResultCollector` | | `testlib/tracebacks.py` | both | Pickle worker exceptions and remap their frames onto host source roots | @@ -28,9 +28,26 @@ via `../packages/testlib` in `[tool.uv.sources]`. - The suite is run once per `dev_server` (`functools.cache` on `get_suite_results`); each host test just looks up its result. +## Forwarding from the outer to the inner pytest run + +- `worker_pytest_args(config)` forwards `config.invocation_params.args` minus + positional targets and `HOST_ONLY_OPTIONS` (currently `-m/--markexpr`). + `addopts` are never forwarded. Sent as repeated `?arg=` query params. +- `host_only_keywords(item)` sends the `-k` keywords that exist only on the host + (ancestor names such as `test_bindings.py`, suite marks, the compat-config + param such as `3.12`) as repeated `?kw=` params; `ExtraKeywordsPlugin` adds + them to in-worker items so `-k` selects the same tests on both sides. +- Workers that don't subclass `TestRunnerEntrypoint` (e.g. the FastAPI test + worker) must build a `RunSuiteRequest` from the query string themselves. + ## Conventions and pitfalls - Keep `host._result_key` and `entrypoint.ResultCollector._key` in sync. +- Add host-only options to `HOST_ONLY_OPTIONS` as they turn up (plugin options + not installed in the worker, e.g. `-n`, `--cov`, `--lf`, produce an inner + usage error surfaced as a 500). +- `-x`/`--maxfail` stops the inner session early; later host tests in that + suite then fail as "not found in results". - Python 3.12 (Pyodide 0.26.0a2) reports false passes for async in-worker tests; verify failure behaviour on 3.13+. - `pywrangler sync` in tests may need `UV_NATIVE_TLS=1` on hosts with custom diff --git a/packages/testlib/testlib/entrypoint.py b/packages/testlib/testlib/entrypoint.py index a7a490d8..4d280645 100644 --- a/packages/testlib/testlib/entrypoint.py +++ b/packages/testlib/testlib/entrypoint.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from io import StringIO from typing import Any -from urllib.parse import urlparse +from urllib.parse import parse_qs, urlparse import pytest from pyodide.webloop import WebLoop @@ -159,6 +159,48 @@ def run_pytest(pytest_args): assert exit_code == 0, f"pytest exit code {exit_code}" +class ExtraKeywordsPlugin: + """Attach host-only ``-k`` keywords to every collected item. + + The host's mirrored items carry keywords (host module name, compat-config + parameter, marks) that don't exist in the worker; adding them here makes a + forwarded ``-k`` expression select the same tests on both sides. + """ + + def __init__(self, keywords): + self.keywords = set(keywords) + + def pytest_itemcollected(self, item): + item.extra_keyword_matches.update(self.keywords) + + +@dataclass +class RunSuiteRequest: + """Parameters the host sends along with a ``/run-tests/`` request.""" + + pytest_args: list + keywords: list + + @classmethod + def from_url(cls, url): + """Parse a ``/run-tests`` URL (string or ``urlparse`` result). + + The host sends pytest arguments as repeated ``arg`` and extra ``-k`` + keywords as repeated ``kw`` query parameters. + """ + query = urlparse(url).query if isinstance(url, str) else url.query + params = parse_qs(query, keep_blank_values=True) + return cls(pytest_args=params.get("arg", []), keywords=params.get("kw", [])) + + @classmethod + def from_query_params(cls, query_params): + """Build from a Starlette-style multi-dict with ``getlist``.""" + return cls( + pytest_args=query_params.getlist("arg"), + keywords=query_params.getlist("kw"), + ) + + @dataclass class TestRunnerResult: payload: Any @@ -171,14 +213,22 @@ def __init__(self, env, extra_plugins=()): self.collector = ResultCollector() self.extra_plugins = list(extra_plugins) - def plugins(self): + def plugins(self, keywords=()): return [ self.collector, EnvPlugin(self.env), + ExtraKeywordsPlugin(keywords), *self.extra_plugins, ] - def run_suite(self, suite_name): + def run_suite(self, suite_name, request=None): + """Run the ``test_`` module under pytest. + + ``request`` carries the pytest arguments and extra ``-k`` keywords + forwarded from the host pytest invocation (see ``testlib.host``). + """ + if request is None: + request = RunSuiteRequest([], []) module = f"test_{suite_name}" if importlib.util.find_spec(module) is None: return TestRunnerResult( @@ -194,8 +244,8 @@ def run_suite(self, suite_name): redirect_stderr(output), ): exit_code = pytest.main( - ["--pyargs", module, "-p", "no:cacheprovider"], - plugins=self.plugins(), + ["--pyargs", module, "-p", "no:cacheprovider", *request.pytest_args], + plugins=self.plugins(request.keywords), ) if exit_code != 0 and not self.collector.results: return TestRunnerResult( @@ -221,11 +271,12 @@ def plugins(self): return [] async def fetch(self, request): - path = urlparse(request.url).path + url = urlparse(request.url) + path = url.path if path.startswith("/run-tests/"): suite_name = path[len("/run-tests/") :] - result = self.runner.run_suite(suite_name) + result = self.runner.run_suite(suite_name, RunSuiteRequest.from_url(url)) return Response.json(result.payload, result.status) if path == "/health": return Response.json({"ok": True}) diff --git a/packages/testlib/testlib/host.py b/packages/testlib/testlib/host.py index 1edacea1..0b204df7 100644 --- a/packages/testlib/testlib/host.py +++ b/packages/testlib/testlib/host.py @@ -218,11 +218,120 @@ def dev_server( _terminate(process, teardown_timeout) +# Host pytest options that must not be forwarded to the in-worker pytest run. +# Host markers (e.g. ``hyperdrive``) don't exist inside the worker. Extend this +# as further host-only options turn up. +HOST_ONLY_OPTIONS: frozenset[str] = frozenset({"-m", "--markexpr"}) + + +def _option_name(arg: str) -> str | None: + """Return the option part of *arg* (``--tb=short`` -> ``--tb``, ``-mfoo`` -> ``-m``).""" + if arg.startswith("--"): + return arg.split("=", 1)[0] + if arg.startswith("-") and len(arg) > 1: + return arg[:2] + return None + + +def worker_pytest_args(config: pytest.Config) -> tuple[str, ...]: + """Arguments from the host pytest invocation to forward to the worker. + + Everything the host was invoked with is forwarded except positional + targets (host node IDs don't exist inside the worker; the worker runs the + suite module instead) and the options in :data:`HOST_ONLY_OPTIONS`. + Options from ``addopts`` are not part of the invocation and are not + forwarded either. + + Host node IDs mirror the in-worker ones (see ``register_in_worker_suites``) + so ``-k`` expressions on suite file, class and function names select the + same tests on both sides. Only the host module (e.g. ``test_bindings.py``) + and the compat-config parameter (e.g. ``3.12``) have no in-worker + counterpart. + """ + args = [str(arg) for arg in config.invocation_params.args] + targets = ( + set(config.args) if config.args_source is config.ArgsSource.ARGS else set() + ) + + forwarded: list[str] = [] + i = 0 + while i < len(args): + arg = args[i] + i += 1 + if arg == "--": + # Everything after ``--`` is a positional target. + break + name = _option_name(arg) + if name is None: + # Positional target, or the value of a forwarded option. + if arg not in targets: + forwarded.append(arg) + continue + if name in HOST_ONLY_OPTIONS: + # Drop the option and, if given separately, its value. + if ( + arg == name + and i < len(args) + and not args[i].startswith("-") + and args[i] not in targets + ): + i += 1 + continue + forwarded.append(arg) + return tuple(forwarded) + + +def _suite_node(item: pytest.Item) -> pytest.Class: + """The ``test_.py`` class node that *item* belongs to.""" + node: Any = item + while not isinstance(node.parent, pytest.Module): + node = node.parent + return node + + +def host_only_keywords(item: pytest.Item) -> tuple[str, ...]: + """``-k`` keywords of *item* that have no counterpart inside the worker. + + Host items mirror the in-worker node names below the suite class (see + ``register_in_worker_suites``), but additionally carry the names of their + ancestors (``tests``, ``test_bindings.py``), the compat-config parameter + (``3.12``) and any suite marks. The worker attaches these to its own items + so a ``-k`` expression selects the same tests on both sides. + """ + suite = _suite_node(item) + keywords: list[str] = [] + for node in suite.listchain()[:-1]: + if isinstance(node, pytest.Session): + continue + # Like pytest's KeywordMatcher, skip the rootdir directory node. + if isinstance(node, pytest.Directory) and isinstance( + node.parent, pytest.Session + ): + continue + keywords.append(node.name) + keywords.extend(mark.name for mark in suite.iter_markers()) + callspec = getattr(item, "callspec", None) + if callspec is not None: + keywords.append(callspec.id) + return tuple(keywords) + + @functools.cache -def get_suite_results(server: str, suite: str) -> SuiteResults | str: +def get_suite_results( + server: str, + suite: str, + args: tuple[str, ...] = (), + keywords: tuple[str, ...] = (), +) -> SuiteResults | str: + """Run *suite* in the worker and return its results. + + *args* are extra pytest arguments and *keywords* extra ``-k`` keywords for + the in-worker items (see ``worker_pytest_args`` and ``host_only_keywords``). + """ try: response = requests.get( f"{server}/run-tests/{suite}", + params={"arg": list(args), "kw": list(keywords)}, timeout=(SUITE_CONNECT_TIMEOUT, SUITE_READ_TIMEOUT), ) except requests.RequestException as error: @@ -237,15 +346,21 @@ def _make_test( ) -> Callable: """Build a host test method that reports the in-worker result *result_key*.""" - def test_fn(self: Any, dev_server: str) -> None: + def test_fn(self: Any, dev_server: str, request: pytest.FixtureRequest) -> None: # Hide this frame: the interesting traceback is the one from the worker. __tracebackhide__ = True - results = get_suite_results(dev_server, suite) + results = get_suite_results( + dev_server, + suite, + worker_pytest_args(request.config), + host_only_keywords(request.node), + ) if isinstance(results, str): pytest.fail(results) result = results.get(result_key) assert result is not None, ( - f"Test {suite}::{result_key} not found in results; " + f"Test {suite}::{result_key} not found in results " + f"(deselected, or the worker session stopped early, e.g. due to -x); " f"available keys: {sorted(results)}" ) if result["status"] == "skipped":