From cc13ad92c34604ca2583f92b2d45e3f48728258c Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:02:53 +0200 Subject: [PATCH 01/12] feat(grpc-web): Pyodide/WASM grpc-web transport for the async client Lets the async client run inside Pyodide/WebAssembly, where grpcio has no wheel and sockets do not exist: gRPC goes over grpc-web (fetch) and REST over the browser's fetch, against Weaviate core's native /v1/grpc-web endpoint (1.38.3+). - packages/web: the weaviate-client-web companion distribution (pure-Python grpc shim, GrpcWebChannel, grpc-web framing, pyfetch/httpx senders, httpx-over-fetch transport) - weaviate/__init__.py: a bare `import weaviate` bootstraps the companion under Emscripten; a missing companion raises an install hint - setup.cfg: grpcio is skipped under Emscripten - connect/base.py: grpc_path_prefix on ConnectionParams (port collision allowed with a prefix), fail-fast check at client construction for a sync client or a missing shim, grpc-web.path_prefix channel option for the shim - connect/helpers.py: under Emscripten the async helpers pin gRPC to the REST endpoint under /v1/grpc-web (Con006 when a caller's gRPC endpoint is discarded); unchanged elsewhere - connect/v4.py: sync client rejected at construction under Emscripten; the gRPC ping error is passed through so WeaviateGRPCUnavailableError can name the real cause (grpc-web branch without firewall/port advice; 404 names server < 1.38.3 or a wrong prefix); fetch failures of the pypi version check are ignored - collections/batch/async_.py: batch.stream() fails fast under grpc-web, pointing to insert_many() - embedded.py: explicit error under Emscripten; proto/v1: grpcio version fallback when dist metadata is absent (Emscripten only), drift-pinned by proto_test - CI: grpc-web package tests (3.10-3.14) and a real-Pyodide e2e job under Node Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01GUNU7QgDr9MmFZnjKY9zFN --- .github/workflows/main.yaml | 74 ++- .gitignore | 4 +- ci/pyodide-e2e/e2e.py | 209 ++++++ ci/pyodide-e2e/package.json | 8 + ci/pyodide-e2e/run.mjs | 83 +++ packages/web/README.md | 171 +++++ packages/web/pyproject.toml | 32 + .../web/src/weaviate_client_web/__init__.py | 74 +++ .../web/src/weaviate_client_web/_channel.py | 436 ++++++++++++ .../web/src/weaviate_client_web/_framing.py | 101 +++ .../src/weaviate_client_web/_httpx_fetch.py | 208 ++++++ .../web/src/weaviate_client_web/_sender.py | 60 ++ packages/web/src/weaviate_client_web/_shim.py | 283 ++++++++ packages/web/src/weaviate_client_web/py.typed | 0 packages/web/tests/conftest.py | 7 + packages/web/tests/test_framing.py | 120 ++++ packages/web/tests/test_httpx_fetch.py | 604 +++++++++++++++++ packages/web/tests/test_shim_install.py | 123 ++++ packages/web/tests/test_single_import.py | 139 ++++ packages/web/tests/test_transport.py | 621 ++++++++++++++++++ proto_test/test_proto.py | 112 +++- pyrightconfig.json | 2 +- setup.cfg | 2 +- test/test_connection_params.py | 189 ++++++ test/test_wasm_compat.py | 254 +++++++ weaviate/__init__.py | 24 +- weaviate/collections/batch/async_.py | 10 + weaviate/connect/base.py | 69 +- weaviate/connect/helpers.py | 83 ++- weaviate/connect/v4.py | 37 +- weaviate/embedded.py | 9 + weaviate/exceptions.py | 57 +- weaviate/proto/v1/__init__.py | 18 +- weaviate/warnings.py | 13 + 34 files changed, 4200 insertions(+), 36 deletions(-) create mode 100644 ci/pyodide-e2e/e2e.py create mode 100644 ci/pyodide-e2e/package.json create mode 100644 ci/pyodide-e2e/run.mjs create mode 100644 packages/web/README.md create mode 100644 packages/web/pyproject.toml create mode 100644 packages/web/src/weaviate_client_web/__init__.py create mode 100644 packages/web/src/weaviate_client_web/_channel.py create mode 100644 packages/web/src/weaviate_client_web/_framing.py create mode 100644 packages/web/src/weaviate_client_web/_httpx_fetch.py create mode 100644 packages/web/src/weaviate_client_web/_sender.py create mode 100644 packages/web/src/weaviate_client_web/_shim.py create mode 100644 packages/web/src/weaviate_client_web/py.typed create mode 100644 packages/web/tests/conftest.py create mode 100644 packages/web/tests/test_framing.py create mode 100644 packages/web/tests/test_httpx_fetch.py create mode 100644 packages/web/tests/test_shim_install.py create mode 100644 packages/web/tests/test_single_import.py create mode 100644 packages/web/tests/test_transport.py create mode 100644 test/test_connection_params.py create mode 100644 test/test_wasm_compat.py diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 489ff9504..163c70afa 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -45,11 +45,11 @@ jobs: cache: 'pip' # caching pip dependencies - run: pip install -r requirements-devel.txt - name: "Ruff lint" - run: ruff check weaviate test mock_tests integration + run: ruff check weaviate test mock_tests integration packages/web - name: "Ruff format" - run: ruff format --diff weaviate test mock_tests integration + run: ruff format --diff weaviate test mock_tests integration packages/web - name: "Flake 8" - run: flake8 weaviate test mock_tests integration + run: flake8 weaviate test mock_tests integration packages/web - name: "Check release for pypi" run: | python -m build @@ -105,6 +105,72 @@ jobs: name: coverage-report-${{ matrix.folder }} path: coverage-${{ matrix.folder }}.xml + grpc-web-tests: + name: Run gRPC-Web Package Tests + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ matrix.version }} + cache: 'pip' # caching pip dependencies + - run: | + pip install -r requirements-test.txt -r requirements-devel.txt + pip install -e . -e packages/web + - name: Run grpc-web package tests + run: pytest packages/web/tests + + pyodide-e2e: + name: Run Pyodide (WASM) e2e Tests + runs-on: ubuntu-latest + timeout-minutes: 15 + # No Python matrix: the pinned Pyodide bundle fixes the interpreter (see + # ci/pyodide-e2e/package.json for the exact pin). + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + fetch-tags: true + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + cache: 'pip' # caching pip dependencies + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: "22" + - name: Login to Docker Hub + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + if: ${{ !github.event.pull_request.head.repo.fork && github.triggering_actor != 'dependabot[bot]' }} + with: + username: ${{secrets.DOCKER_USERNAME}} + password: ${{secrets.DOCKER_PASSWORD}} + - name: Build pure wheels (base client + grpc-web) + run: | + pip install build + python -m build --wheel --outdir dist . + python -m build --wheel --outdir dist packages/web + - name: start weaviate + run: | + source ./ci/compose.sh + export WEAVIATE_VERSION=$WEAVIATE_139 + docker compose -f ci/docker-compose-async.yml up -d + wait "http://localhost:8090" + - name: Run the e2e suite inside Pyodide under Node + env: + WEAVIATE_HOST: localhost + WEAVIATE_PORT: "8090" + run: | + npm install --prefix ci/pyodide-e2e + node ci/pyodide-e2e/run.mjs dist + - name: stop weaviate + if: always() + run: docker compose -f ci/docker-compose-async.yml down --remove-orphans + proto-test: name: Run importing protos test runs-on: ubuntu-latest @@ -352,7 +418,7 @@ jobs: build-and-publish: name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI - needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test] + needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test, grpc-web-tests, pyodide-e2e] runs-on: ubuntu-latest timeout-minutes: 20 steps: diff --git a/.gitignore b/.gitignore index b4ba50e1b..395b51d6c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ venv .idea dist/ -weaviate_client.egg-info +*.egg-info/ **/__pycache__ tmp build/ @@ -27,3 +27,5 @@ scratch/ *-test.sh *.hdf5 *.jsonl +ci/pyodide-e2e/node_modules/ +ci/pyodide-e2e/package-lock.json diff --git a/ci/pyodide-e2e/e2e.py b/ci/pyodide-e2e/e2e.py new file mode 100644 index 000000000..b1f03ce9c --- /dev/null +++ b/ci/pyodide-e2e/e2e.py @@ -0,0 +1,209 @@ +"""In-Pyodide e2e for the Weaviate client over core-native grpc-web. + +Executed by ``run.mjs`` inside Pyodide under Node: the runner runs this module's code +(imports below install the grpc shim + fetch transport) and then awaits ``main()`` on +Pyodide's event loop. Plain asserts with one ``OK`` line per step so CI logs are +diagnosable; any failure exits nonzero. + +Deliberately not covered: browser/CORS behaviour (this runs under Node, no CORS layer) +and OIDC auth flows (anonymous access only). +""" + +import os +import uuid +import warnings + +import weaviate_client_web # bootstraps the grpc shim + fetch transport under Emscripten + +import grpc +import httpx +import weaviate +import weaviate.classes as wvc +from weaviate.classes.config import DataType, Property, ReferenceProperty +from weaviate.classes.data import DataReference +from weaviate.classes.query import Filter +from weaviate.classes.tenants import Tenant +from weaviate.exceptions import WeaviateBatchStreamError, WeaviateQueryError + +COLL = "PyodideE2E" +MT_COLL = "PyodideE2ETenants" +# Weaviate core serves grpc-web natively on the REST port under this prefix (default-on +# since 1.38.3), so no proxy sits between the client and the server. Under Emscripten the +# connect helpers route gRPC there themselves — nothing here selects it. +GRPC_WEB_PREFIX = "/v1/grpc-web" + + +def ok(step: str) -> None: + print(f"OK {step}", flush=True) + + +async def main() -> None: + assert weaviate_client_web.is_installed(), "grpc shim did not install under Emscripten" + assert getattr(grpc, "__weaviate_client_web_shim__", False), ( + "sys.modules['grpc'] is not the shim" + ) + # REST must run through the package's own fetch transport, not Pyodide's bundled + # httpx transport (which cannot read the null body of HEAD / 204 responses). + assert weaviate_client_web.is_fetch_transport_installed(), "fetch transport not installed" + assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False + ), "httpx.AsyncHTTPTransport is not the package's fetch transport" + ok("self-check: package fetch transport is the active httpx transport") + + host = os.environ.get("WEAVIATE_HOST", "localhost") + port = int(os.environ.get("WEAVIATE_PORT", "8090")) + client = weaviate.use_async_with_custom( + http_host=host, + http_port=port, + http_secure=False, + grpc_host=host, + grpc_port=port, + grpc_secure=False, + ) + params = client._connection._connection_params + assert params._grpc_web_path_prefix == GRPC_WEB_PREFIX, params + assert params._grpc_target == f"{host}:{port}", params + ok("connect helper routed gRPC onto the REST endpoint under /v1/grpc-web") + + # No skip_init_checks: connect() performs the gRPC health check over grpc-web. + await client.connect() + ok("connect (health check over grpc-web)") + + try: + for name in (COLL, MT_COLL): + if await client.collections.exists(name): + await client.collections.delete(name) + + await client.collections.create( + COLL, + vector_config=wvc.config.Configure.Vectors.self_provided(), + properties=[ + Property(name="title", data_type=DataType.TEXT), + Property(name="idx", data_type=DataType.INT), + ], + references=[ReferenceProperty(name="related", target_collection=COLL)], + ) + ok("collections.create") + + coll = client.collections.get(COLL) + ret = await coll.data.insert_many([{"title": f"article {i}", "idx": i} for i in range(50)]) + assert not ret.has_errors and len(ret.uuids) == 50, f"insert_many errors: {ret.errors}" + ok("insert_many (BatchObjects) = 50") + + res = await coll.query.fetch_objects(limit=100) + assert len(res.objects) == 50, f"fetch_objects got {len(res.objects)}" + ok("query.fetch_objects = 50") + + res = await coll.query.bm25("article", limit=5) + assert len(res.objects) == 5, f"bm25 got {len(res.objects)}" + ok("query.bm25 limit=5 = 5") + + res = await coll.query.fetch_objects( + filters=Filter.by_property("idx").less_than(10), limit=100 + ) + assert len(res.objects) == 10, f"filtered fetch_objects got {len(res.objects)}" + ok("query.fetch_objects filtered idx<10 = 10") + + agg = await coll.aggregate.over_all(total_count=True) + assert agg.total_count == 50, f"aggregate total_count {agg.total_count}" + agg = await coll.aggregate.over_all( + return_metrics=[wvc.query.Metrics("idx").integer(minimum=True, maximum=True)] + ) + idx = agg.properties["idx"] + assert idx.minimum == 0 and idx.maximum == 49, agg.properties + ok("aggregate count=50 min=0 max=49") + + # REST calls answered without a body (HEAD 204/404, PATCH/DELETE 204) and the + # batch-references path, which reads httpx's response.elapsed. + first, second, last = ret.uuids[0], ret.uuids[1], ret.uuids[49] + assert await coll.data.exists(first) is True + assert await coll.data.exists(uuid.uuid4()) is False + ok("data.exists (HEAD 204 / 404) = True / False") + + await coll.data.update(uuid=first, properties={"title": "article 0 (updated)"}) + obj = await coll.query.fetch_object_by_id(first) + assert obj is not None and obj.properties["title"] == "article 0 (updated)", obj + ok("data.update (PATCH 204) -> fetch_object_by_id sees the update") + + refs = await coll.data.reference_add_many( + [ + DataReference( + from_property="related", from_uuid=ret.uuids[i], to_uuid=ret.uuids[i + 1] + ) + for i in range(5) + ] + ) + assert not refs.has_errors, f"reference_add_many errors: {refs.errors}" + assert refs.elapsed_seconds >= 0, refs + ok("data.reference_add_many (REST /batch/references) = 5") + + await coll.data.reference_delete(from_uuid=first, from_property="related", to=second) + ok("data.reference_delete (DELETE 204)") + + assert await coll.data.delete_by_id(last) is True + assert await coll.data.exists(last) is False + # deleting a missing object answers 204 or 404 depending on the server topology; + # either way it is a body-less response the transport must handle + assert isinstance(await coll.data.delete_by_id(last), bool) + ok("data.delete_by_id (DELETE 204; repeat -> 204/404) = True, then bool") + + await client.collections.create( + MT_COLL, + vector_config=wvc.config.Configure.Vectors.self_provided(), + properties=[Property(name="title", data_type=DataType.TEXT)], + multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=True), + ) + mt = client.collections.get(MT_COLL) + await mt.tenants.create([Tenant(name="t1"), Tenant(name="t2")]) + tenants = await mt.tenants.get() + assert set(tenants.keys()) == {"t1", "t2"}, f"TenantsGet: {set(tenants.keys())}" + ok("multi-tenant create + TenantsGet = {t1, t2}") + + assert await mt.tenants.exists("t1") is True + assert await mt.tenants.exists("t404") is False + ok("tenants.exists (HEAD 200 / 404) = True / False") + + t1 = mt.with_tenant("t1") + ret = await t1.data.insert_many([{"title": f"tenant doc {i}"} for i in range(10)]) + assert not ret.has_errors and len(ret.uuids) == 10, f"tenant insert_many: {ret.errors}" + agg = await t1.aggregate.over_all(total_count=True) + assert agg.total_count == 10, f"tenant aggregate {agg.total_count}" + ok("per-tenant insert_many = 10, aggregate = 10") + + dm = await t1.data.delete_many(where=Filter.by_property("title").like("tenant*")) + assert dm.successful == 10, f"delete_many successful={dm.successful}" + agg = await t1.aggregate.over_all(total_count=True) + assert agg.total_count == 0, f"post-delete aggregate {agg.total_count}" + ok("per-tenant delete_many (BatchDelete) = 10 -> aggregate = 0") + + try: + await client.collections.get("DoesNotExistXyz").query.fetch_objects(limit=1) + raise AssertionError("expected WeaviateQueryError for nonexistent collection") + except WeaviateQueryError as e: + assert "DoesNotExistXyz" in str(e), str(e) + ok("error mapping: nonexistent collection -> WeaviateQueryError names the collection") + + try: + async with client.batch.stream() as batch: + await batch.add_object(collection=COLL, properties={"title": "x", "idx": 999}) + raise AssertionError("batch.stream() did not raise under grpc-web") + except WeaviateBatchStreamError as e: + assert "grpc-web" in str(e) and "insert_many" in str(e), str(e) + ok("batch.stream() -> WeaviateBatchStreamError (clear message)") + + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + async with client.batch.experimental() as batch: + await batch.add_object(collection=COLL, properties={"title": "x", "idx": 999}) + raise AssertionError("batch.experimental() did not raise under grpc-web") + except WeaviateBatchStreamError: + ok("batch.experimental() -> WeaviateBatchStreamError") + + for name in (COLL, MT_COLL): + await client.collections.delete(name) + ok("cleanup") + finally: + await client.close() + + print("PYODIDE E2E: ALL STEPS OK", flush=True) diff --git a/ci/pyodide-e2e/package.json b/ci/pyodide-e2e/package.json new file mode 100644 index 000000000..fbf78f303 --- /dev/null +++ b/ci/pyodide-e2e/package.json @@ -0,0 +1,8 @@ +{ + "name": "weaviate-pyodide-e2e", + "private": true, + "description": "Runs the weaviate-client e2e suite inside Pyodide (WASM) under Node", + "dependencies": { + "pyodide": "314.0.4" + } +} diff --git a/ci/pyodide-e2e/run.mjs b/ci/pyodide-e2e/run.mjs new file mode 100644 index 000000000..3342c9b1a --- /dev/null +++ b/ci/pyodide-e2e/run.mjs @@ -0,0 +1,83 @@ +// Runs the Weaviate Python client e2e suite (e2e.py) inside Pyodide (WASM) under Node. +// +// Usage: node run.mjs +// must contain exactly the two locally-built pure wheels: +// weaviate_client-*.whl and weaviate_client_web-*.whl. +// Env: WEAVIATE_HOST (default localhost), WEAVIATE_PORT (default 8090). +// +// The pinned `pyodide` npm package fixes the interpreter (the 314.x line bundles +// CPython 3.14), so there is no Python version matrix here. micropip installs the two +// local wheels; transitive deps resolve from the Pyodide distribution +// (pydantic/pydantic_core/cryptography ship wasm builds there — pydantic_core has no +// wasm wheel on PyPI) or from PyPI as pure wheels (protobuf), and the base client's +// `grpcio; sys_platform != "emscripten"` marker correctly skips grpcio. +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { loadPyodide } from "pyodide"; + +if (!process.argv[2]) { + console.error("usage: node run.mjs "); + process.exit(2); +} +const wheelsDir = resolve(process.argv[2]); +const here = dirname(fileURLToPath(import.meta.url)); + +const wheels = readdirSync(wheelsDir) + .filter((f) => f.endsWith(".whl")) + .sort(); // installs weaviate_client before weaviate_client_web, which depends on it +const prefixes = ["weaviate_client-", "weaviate_client_web-"]; +if ( + wheels.length !== 2 || + !prefixes.every((p) => wheels.some((w) => w.startsWith(p))) +) { + console.error( + `expected exactly one weaviate_client-*.whl and one weaviate_client_web-*.whl in ${wheelsDir}, found: ${JSON.stringify(wheels)}`, + ); + process.exit(2); +} + +const pyodide = await loadPyodide({ + env: { + WEAVIATE_HOST: process.env.WEAVIATE_HOST ?? "localhost", + WEAVIATE_PORT: process.env.WEAVIATE_PORT ?? "8090", + }, +}); +console.log( + `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, +); + +await pyodide.loadPackage("micropip"); +const micropip = pyodide.pyimport("micropip"); +// anyio (needed because Pyodide's httpx recipe drops it, while authlib imports it +// directly) resolves from the companion wheel's `anyio ; sys_platform == "emscripten"` +// marker — no explicit install here, so the marker stays proven. + +pyodide.FS.mkdirTree("/wheels"); +pyodide.mountNodeFS("/wheels", wheelsDir); +for (const wheel of wheels) { + console.log(`micropip install ${wheel}`); + await micropip.install(`emfs:/wheels/${wheel}`); +} + +// Single-import check: the FIRST weaviate-side import in this interpreter is a bare +// `import weaviate` — the base client must bootstrap the companion (and the shim) itself. +pyodide.runPython(` +import sys +assert "weaviate_client_web" not in sys.modules +import weaviate +assert getattr(sys.modules.get("grpc"), "__weaviate_client_web_shim__", False), \\ + "bare 'import weaviate' did not install the grpc shim" +print("OK bare 'import weaviate' bootstrapped the grpc shim") +`); + +// Define e2e.py's globals (imports run here, installing the grpc shim), then await +// main() on Pyodide's event loop — asyncio.run() cannot be used inside Pyodide. +pyodide.runPython(readFileSync(resolve(here, "e2e.py"), "utf8")); +try { + await pyodide.runPythonAsync("await main()"); +} catch (err) { + console.error(err); + process.exit(1); +} diff --git a/packages/web/README.md b/packages/web/README.md new file mode 100644 index 000000000..582ab2796 --- /dev/null +++ b/packages/web/README.md @@ -0,0 +1,171 @@ +# weaviate-client-web + +A grpc-web / WebAssembly (Pyodide) transport for the +[Weaviate Python client](https://github.com/weaviate/weaviate-python-client), so the +client's **async** gRPC data path can run inside a browser (marimo notebooks, Pyodide, +WASM workers) where there is no socket and no `grpcio` wheel. + +It is built from the same repository as `weaviate-client` and reuses its generated +protobuf stubs — it does **not** fork code generation. + +Requires Weaviate ≥ 1.38.3 (the first release to serve grpc-web natively) or a grpc-web +transcoder in front of an older server. Pyodide ≥ 0.27 recommended; verified on +Pyodide 314.0.4 (CPython 3.14). + +## How it works + +Under Pyodide there is no `grpcio` Emscripten wheel, and `import weaviate` hard-imports +`grpc` at module load. This package installs a small pure-Python `grpc` shim into +`sys.modules` **before** `import weaviate`, which: + +- satisfies every import-time `import grpc` / `from grpc(.aio) import ...` in the base + client and its generated `*_pb2_grpc` stubs; +- provides `grpc.aio.Channel` as a real base class, so the grpc-web channel + (`GrpcWebChannel`) subclasses it and the client's `isinstance(..., grpc.aio.Channel)` + assertions pass; +- satisfies the generated v6300 stub's version gate + (`grpc.__version__` / `grpc._utilities.first_version_is_lower`). + +The `GrpcWebChannel` frames unary RPCs as grpc-web (a 5-byte header + protobuf payload) +and POSTs them via `pyodide.http.pyfetch`. Call metadata (API key / OIDC bearer) is +folded into `fetch` headers. + +The target is not configurable: under Emscripten the connect helpers +(`use_async_with_local`, `use_async_with_weaviate_cloud`, `use_async_with_custom`) pin +gRPC to the **REST** endpoint — same host, port and TLS — under Weaviate's own +`/v1/grpc-web` base path, so gRPC and REST share one origin and no proxy is needed. That +is deliberate: native gRPC cannot work under WASM at all, so grpc-web on the REST +listener is not a choice that could be wrong. The TypeScript `@weaviate/web` client makes +the same call, dropping `grpcHost`/`grpcPort`/`grpcSecure` from its options entirely. + +A grpc-web transcoder on a separate endpoint (Envoy, +[connectrpc/vanguard](https://github.com/connectrpc/vanguard-go)) is therefore not +reachable through the helpers. If you need one — e.g. in front of a Weaviate older than +1.38.3 — build the connection parameters yourself: + +```python +from weaviate import WeaviateAsyncClient +from weaviate.connect import ConnectionParams + +client = WeaviateAsyncClient( + ConnectionParams.from_params( + http_host="weaviate.example.com", http_port=443, http_secure=True, + grpc_host="transcoder.example.com", grpc_port=443, grpc_secure=True, + # add grpc_path_prefix="/base/path" if the transcoder is not at the root + ) +) +``` + +For REST (`is_ready`, collection config, `/batch/references`, …) the package patches +`httpx.AsyncHTTPTransport` with its own `pyfetch`-based transport. It does so even on +Pyodide builds whose bundled httpx has a JS-fetch transport of its own: that transport +cannot read the null body of HEAD requests and 204 responses (`data.exists`, +`data.delete_by_id`, `tenants.exists`, …) and does not enforce the client's per-request +timeouts. + +## Usage + +With this package installed, a plain `import weaviate` is all you need — under +Emscripten the base client imports `weaviate_client_web` itself before anything else, +which installs the shim and the fetch transport (and raises a clear error if the package +is missing). Against Weaviate ≥ 1.38.3: + +```python +import weaviate # bootstraps weaviate_client_web automatically under Emscripten + +client = weaviate.use_async_with_local(port=8080) +await client.connect() # runs the gRPC health check over grpc-web +collection = client.collections.get("Article") +await collection.query.near_text("hello", limit=3) +``` + +Nothing selects grpc-web: `use_async_with_local()`, `use_async_with_weaviate_cloud()` and +`use_async_with_custom()` all route gRPC onto the REST endpoint under `/v1/grpc-web` when +they run under Emscripten, and behave exactly as before everywhere else. + +```python +client = weaviate.use_async_with_weaviate_cloud( + cluster_url="rAnD0mD1g1t5.something.weaviate.cloud", + auth_credentials=weaviate.classes.init.Auth.api_key("my-api-key"), +) +``` + +`use_async_with_custom()` still requires `grpc_host`/`grpc_port`/`grpc_secure` — Python +cannot drop required parameters on one platform the way TypeScript drops them from a +type. Pass the HTTP values; anything else is overridden with them and warned about +(`Con006`), so a browser client never silently points somewhere it cannot reach. + +```python +client = weaviate.use_async_with_custom( + http_host="localhost", http_port=8080, http_secure=False, + grpc_host="localhost", grpc_port=8080, grpc_secure=False, # = the HTTP endpoint +) +``` + +Pass `headers={...}` / `auth_credentials=...` as usual for API keys, OIDC or WCD. + +Importing the companion explicitly first also works and remains the explicit form: + +```python +import weaviate_client_web # installs the grpc shim under Emscripten (no-op elsewhere) +import weaviate +``` + +## Supported / unsupported + +| Feature | Kind | Status | +|----------------------------------------------------------|-----------------|--------| +| Search, Aggregate, TenantsGet, BatchObjects, BatchDelete | unary gRPC | ✅ works over grpc-web | +| Health check (`/grpc.health.v1.Health/Check`) | unary gRPC | ✅ runs on `connect()` over grpc-web | +| REST (`is_ready`, config, `/batch/references`, …) | REST | ✅ via the package's own fetch transport | +| API-key auth (`Auth.api_key`) | header | ✅ | +| OIDC auth (`client_credentials` / `client_password` / `bearer_token`) | REST | ✅ token fetch + asyncio-task refresh (no threads) | +| Bulk insert: `collection.data.insert_many()` | unary gRPC | ✅ the supported bulk path under WASM | +| `batch.stream()` / `batch.experimental()` (BatchStream) | bidi streaming | ❌ not possible over grpc-web/fetch — raises immediately; use `insert_many()` | +| `batch.dynamic()` / `fixed_size()` / `rate_limit()` | sync-client API | ❌ these only exist on the sync client, which is unsupported under WASM | +| Embedded Weaviate (`use_async_with_embedded`) | subprocess | ❌ raises "not supported under WebAssembly/Pyodide" | +| Synchronous client | — | ❌ async-only under WASM | +| Weaviate Agents: `AsyncQueryAgent` `run/ask/search` | REST | ✅ via fetch | +| Weaviate Agents: `ask_stream` / `research_stream` (SSE) | REST streaming | ⚠️ degraded: the fetch transport buffers the whole response, so events arrive only when the run completes (and long runs can hit the request timeout) | +| Weaviate Agents: sync `QueryAgent`, `TransformationAgent`, `PersonalizationAgent` | REST sync | ❌ no async flavour exists | + +## Configuration not honored in the browser + +`fetch` manages connections itself, so several knobs are accepted but have no effect +under WASM: + +- `AdditionalConfig.proxies` / `trust_env` proxy environment variables (the browser + cannot proxy fetch requests per-client), +- connection-pool sizing and `session_pool_max_retries`, +- `GrpcConfig.credentials` (custom CA bundles — the browser's trust store decides TLS), +- `GrpcConfig.channel_options`, including `grpc.max_send_message_length` / + `grpc.max_receive_message_length` (only `grpc-web.path_prefix` is consumed). The + practical message-size ceiling is the server's `grpcMaxMessageSize` (reported by + `/v1/meta`); exceeding it surfaces as `RESOURCE_EXHAUSTED`, +- `Proxies.grpc` / `GRPC_PROXY`. + +## CORS requirements (browsers) + +Weaviate ≥ 1.38.3 serves the CORS headers below for its `/v1/grpc-web` endpoint itself, +with no configuration. Its request-header list is a **closed allowlist**: custom +`headers={...}` that are not on it fail the browser's preflight. Cross-origin +deployments that go through a grpc-web transcoder or a proxy must configure CORS there: + +- allow every request header the client sends: `content-type`, `x-grpc-web`, + `x-user-agent`, `grpc-timeout`, `x-weaviate-client`, `authorization` (when auth is + used) and `x-weaviate-cluster-url` (Weaviate Cloud); +- expose the grpc-web status headers on responses: + `Access-Control-Expose-Headers: grpc-status, grpc-message` — without this, + trailers-only error responses (e.g. a bad API key) are reported as + `INTERNAL: grpc-web response contained no message frame` instead of the real error; +- note that a CORS-blocked request is indistinguishable from a network failure in the + browser (`TypeError: Failed to fetch`), and is retried as UNAVAILABLE. + +## Testing on CPython + +`weaviate_client_web.install(force=True)` installs the shim on a normal CPython +interpreter (run it in a fresh process, before importing `weaviate`). Inject a sender +with `weaviate_client_web.set_sender(...)` (e.g. `make_httpx_sender()`) to exercise the +transport against an Envoy/vanguard transcoder without a browser. +`install_fetch_transport(force=True)` likewise patches httpx on CPython, given an +importable `pyodide.http` stand-in. diff --git a/packages/web/pyproject.toml b/packages/web/pyproject.toml new file mode 100644 index 000000000..95792c315 --- /dev/null +++ b/packages/web/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = ["setuptools>=65", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "weaviate-client-web" +description = "grpc-web / WASM (Pyodide) transport for the Weaviate Python client" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "BSD-3-Clause" } +authors = [{ name = "Weaviate", email = "hello@weaviate.io" }] +keywords = ["weaviate", "grpc-web", "pyodide", "wasm", "emscripten"] +# Version is kept in lockstep with weaviate-client. TODO(lockstep): derive from the same +# git tag via setuptools_scm and assert the built versions match in CI before publishing. +version = "0.0.1.dev0" +# Deliberately depends on weaviate-client WITHOUT grpcio (grpcio is excluded under +# Emscripten by the `sys_platform != "emscripten"` marker in the base package's deps). +dependencies = [ + "weaviate-client", + # Pyodide's bundled httpx build omits anyio, but authlib imports it directly. + 'anyio ; sys_platform == "emscripten"', +] + +[project.urls] +Source = "https://github.com/weaviate/weaviate-python-client" +Tracker = "https://github.com/weaviate/weaviate-python-client/issues" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +weaviate_client_web = ["py.typed"] diff --git a/packages/web/src/weaviate_client_web/__init__.py b/packages/web/src/weaviate_client_web/__init__.py new file mode 100644 index 000000000..2e9217991 --- /dev/null +++ b/packages/web/src/weaviate_client_web/__init__.py @@ -0,0 +1,74 @@ +"""grpc-web / WASM transport for the Weaviate Python client. + +Under Pyodide/Emscripten there is no ``grpcio`` wheel. Importing this package installs a +pure-Python ``grpc`` shim into ``sys.modules`` (and forces the pure-Python protobuf +runtime) so that the subsequent ``import weaviate`` succeeds and its async gRPC data path +runs over grpc-web (``fetch``) instead of HTTP/2 sockets; REST runs through the package's +own ``fetch``-based httpx transport. + +Usage under Pyodide against Weaviate >= 1.38.3, which serves grpc-web on its REST port +under ``/v1/grpc-web`` (with this package installed, a bare ``import weaviate`` suffices — +the base client imports this package itself under Emscripten before anything else):: + + import weaviate + + client = weaviate.use_async_with_local(port=8080) + await client.connect() + +There is nothing to select. Under Emscripten ``use_async_with_local``, +``use_async_with_weaviate_cloud`` and ``use_async_with_custom`` all pin gRPC to the REST +endpoint under ``/v1/grpc-web``, because native gRPC is impossible there — the same +contract as the TypeScript ``@weaviate/web`` client. ``use_async_with_custom`` still +requires ``grpc_host``/``grpc_port``/``grpc_secure``; give it the HTTP values, or it +warns that it overrode them. + +An explicit ``import weaviate_client_web`` before ``import weaviate`` also works and +remains the explicit form. The shim is installed automatically only under Emscripten, so +importing this package on a normal CPython install never clobbers a real, working +``grpcio``. Async clients only — the synchronous client is not supported in the browser. +""" + +import os +import sys + +from ._shim import StatusCode, install, is_installed + +__all__ = [ + "install", + "is_installed", + "install_fetch_transport", + "uninstall_fetch_transport", + "is_fetch_transport_installed", + "set_sender", + "make_httpx_sender", + "GrpcWebChannel", + "StatusCode", +] + + +def _bootstrap() -> None: + if sys.platform == "emscripten": + # The pure-Python protobuf runtime always works; the upb C-extension may not be + # present. Set before ``import weaviate`` (which imports protobuf) so it takes + # effect. ``setdefault`` lets a user override it explicitly. + os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") + install() + # The REST path needs fetch too: httpx/httpcore open raw sockets, which do + # not exist under WASM. Imported lazily so CPython imports stay light. + from ._httpx_fetch import install_fetch_transport + + install_fetch_transport() + + +_bootstrap() + +# Imported after the bootstrap. These modules pull their grpc base classes directly from +# ``._shim`` (not via ``sys.modules['grpc']``), so importing them is safe regardless of +# whether the shim was installed. +from ._channel import GrpcWebChannel, set_sender # noqa: E402 +from ._httpx_fetch import ( # noqa: E402 + install_fetch_transport, + is_fetch_transport_installed, + uninstall_fetch_transport, +) +from ._sender import make_httpx_sender # noqa: E402 diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py new file mode 100644 index 000000000..af899849c --- /dev/null +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -0,0 +1,436 @@ +"""The grpc-web channel and multicallables. + +:class:`GrpcWebChannel` implements the small slice of the ``grpc.aio`` channel interface +that ``weaviate``'s generated stub and ``ConnectionV4`` actually use — ``unary_unary``, +``stream_stream`` and ``close`` — by framing requests as grpc-web and POSTing them via a +pluggable async sender. It subclasses the shim's ``grpc.aio.Channel`` (:class:`AioChannel`) +so the ``isinstance(..., grpc.aio.Channel)`` assertions in ``connect/v4.py`` hold. + +Only unary RPCs are supported (Search, Aggregate, TenantsGet, BatchObjects, +BatchReferences, BatchDelete, and the unary health check). ``stream_stream`` (the bidi +``BatchStream`` used by opt-in server-side batching) cannot work over grpc-web/fetch and +raises a clear error. +""" + +import asyncio +import base64 +import math +import sys +import urllib.parse +from typing import Any, Callable, Dict, List, Optional + +from ._framing import TruncatedFrameError, UnknownFrameFlagError, encode_message, split_response +from ._sender import Sender, pyfetch_sender +from ._shim import AioChannel, AioRpcError, StatusCode, status_from_int + +# Module-level default sender; overridable for tests / non-browser runtimes. +_default_sender: Sender = pyfetch_sender + + +def set_sender(sender: Sender) -> None: + """Override the default async sender used by new channels (tests/integration).""" + global _default_sender + _default_sender = sender + + +def get_sender() -> Sender: + return _default_sender + + +# grpc-timeout is at most 8 digits plus a unit; anything longer is rejected by the server. +_GRPC_TIMEOUT_MAX = 100_000_000 + + +def _encode_timeout(seconds: Optional[float]) -> Optional[str]: + """Encode a timeout as a grpc-timeout header value; ``None`` means no deadline. + + ``None``, non-finite values and anything beyond 99,999,999 minutes (~190 years) carry + no deadline. Rounds up so we never advertise a shorter deadline than requested (which + would risk premature server-side cancellation), moving to a coarser unit + (m -> S -> M) to stay within 8 digits. Hours are never emitted: grpc-web transcoders + (vanguard) reject any H value above 8H with HTTP 400. + """ + if seconds is None or not math.isfinite(seconds): + return None + for amount, unit in ((seconds * 1000, "m"), (seconds, "S"), (seconds / 60, "M")): + value = max(1, math.ceil(amount)) + if value < _GRPC_TIMEOUT_MAX: + return f"{value}{unit}" + return None + + +def _fold_metadata(headers: Dict[str, str], metadata: Any) -> None: + """Fold gRPC call metadata (``[(key, value), ...]``) into fetch headers. + + Binary ``-bin`` keys are base64-encoded as grpc-web requires. + """ + if not metadata: + return + for key, value in metadata: + name = key.lower() + if name.endswith("-bin"): + raw = value if isinstance(value, (bytes, bytearray)) else str(value).encode() + text = base64.b64encode(raw).decode("ascii") + else: + text = value if isinstance(value, str) else str(value) + # This path bypasses h11/grpcio's header validation, so keep their defence here. + if any(c in name or c in text for c in ("\r", "\n", "\0")): + raise ValueError(f"Illegal character in gRPC metadata {name!r}") + headers[name] = text + + +def _header_lookup(headers: Dict[str, str], name: str) -> Optional[str]: + target = name.lower() + for key, value in headers.items(): + if key.lower() == target: + return value + return None + + +class _UnaryUnaryMultiCallable: + """Awaitable multicallable bound by ``WeaviateStub.__init__``. + + Called as ``await mc(request, metadata=..., timeout=...)`` (and, for the health + check, as ``mc(request, timeout=...)`` with no metadata). + """ + + def __init__( + self, + channel: "GrpcWebChannel", + path: str, + request_serializer: Callable[[Any], bytes], + response_deserializer: Callable[[bytes], Any], + ) -> None: + self._channel = channel + self._path = path + self._serialize = request_serializer + self._deserialize = response_deserializer + + async def __call__( + self, + request: Any, + *, + metadata: Any = None, + timeout: Optional[float] = None, + credentials: Any = None, + wait_for_ready: Any = None, + compression: Any = None, + ) -> Any: + payload = self._serialize(request) + return await self._channel._unary(self._path, payload, self._deserialize, metadata, timeout) + + +class _UnsupportedStreamMultiCallable: + """Placeholder for ``stream_stream`` (bidirectional streaming). + + Calling it raises immediately, before the ``async for`` in ``connect/v4.py`` begins + iterating. + """ + + def __init__(self, path: str) -> None: + self._path = path + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + # NOTE: do not recommend batch.dynamic()/fixed_size()/rate_limit() here — those + # are sync-client-only APIs and do not exist on the async client, which is the + # only client supported under WASM. + raise RuntimeError( + f"Bidirectional streaming RPC {self._path!r} (server-side batching / " + "BatchStream) is not supported over grpc-web/fetch. Use " + "collection.data.insert_many() instead of batch.stream()." + ) + + +class GrpcWebChannel(AioChannel): + """grpc-web/fetch implementation of the async grpc channel slice the client uses.""" + + def __init__( + self, + target: Optional[str], + secure: bool, + options: Any = None, + path_prefix: str = "", + sender: Optional[Sender] = None, + ) -> None: + if not target: + raise ValueError("GrpcWebChannel requires a target (host:port)") + scheme = "https" if secure else "http" + self._base_url = f"{scheme}://{target}" + # Normalize to a single leading slash and no trailing slash; "" == native path. + cleaned = (path_prefix or "").strip("/") + self._path_prefix = f"/{cleaned}" if cleaned else "" + self._sender: Sender = sender or get_sender() + + def unary_unary( + self, + method: str, + request_serializer: Callable[[Any], bytes], + response_deserializer: Callable[[bytes], Any], + _registered_method: bool = False, + ) -> _UnaryUnaryMultiCallable: + return _UnaryUnaryMultiCallable(self, method, request_serializer, response_deserializer) + + def stream_stream( + self, + method: str, + request_serializer: Callable[[Any], bytes], + response_deserializer: Callable[[bytes], Any], + _registered_method: bool = False, + ) -> _UnsupportedStreamMultiCallable: + return _UnsupportedStreamMultiCallable(method) + + async def close(self, grace: Optional[float] = None) -> None: + # Nothing to tear down: each call is an independent fetch. + return None + + async def _unary( + self, + path: str, + payload: bytes, + deserialize: Callable[[bytes], Any], + metadata: Any, + timeout: Optional[float], + ) -> Any: + headers: Dict[str, str] = { + "content-type": "application/grpc-web+proto", + "accept": "application/grpc-web+proto", + "x-grpc-web": "1", + "x-user-agent": "weaviate-client-web", + } + _fold_metadata(headers, metadata) + grpc_timeout = _encode_timeout(timeout) + if grpc_timeout is None: + timeout = None # None / non-finite: no deadline, server- or client-side + else: + headers["grpc-timeout"] = grpc_timeout + + url = self._base_url + self._path_prefix + path + framed = encode_message(payload) + + # Send. Enforce a client-side deadline (the grpc-timeout header is server-side + # only; pyfetch ignores its timeout arg, so without this a stalled request could + # hang forever). Any transport/parse failure is surfaced as AioRpcError; the only + # non-gRPC error a caller can see is the ValueError from metadata validation + # above, raised before any I/O (as native grpcio does). + try: + send = self._sender(url, headers, framed, timeout) + if timeout is not None: + status, resp_headers, body = await asyncio.wait_for(send, timeout) + else: + status, resp_headers, body = await send + except AioRpcError: + raise + except asyncio.TimeoutError as exc: + raise AioRpcError( + code=StatusCode.DEADLINE_EXCEEDED, + details=f"grpc-web request to {path} timed out after {timeout}s", + ) from exc + except Exception as exc: # network/transport failure -> retryable UNAVAILABLE + # str() of transport errors can be empty (e.g. httpx.ConnectError) — always + # include the exception type so failures stay diagnosable + detail = f"{type(exc).__name__}: {exc}" if str(exc) else repr(exc) + details = f"grpc-web transport error for {path}: {detail}" + if not self._path_prefix and sys.platform == "emscripten": + details += " " + _no_path_prefix_hint() + raise AioRpcError(code=StatusCode.UNAVAILABLE, details=details) from exc + + try: + return self._handle_response(status, resp_headers, body, deserialize, url) + except AioRpcError: + raise + except Exception as exc: # malformed framing / status / payload + raise AioRpcError( + code=StatusCode.INTERNAL, + details=f"malformed grpc-web response for {path}: {exc}", + ) from exc + + @staticmethod + def _handle_response( + http_status: int, + resp_headers: Dict[str, str], + body: bytes, + deserialize: Callable[[bytes], Any], + url: str = "", + ) -> Any: + # A frame-parse failure must never decide the outcome by itself: real error + # responses carry non-grpc-web bodies (Weaviate's 404 JSON, an nginx page), and + # the HTTP status, URL and the server's own text must survive into the error. + messages: List[bytes] = [] + trailers: Dict[str, str] = {} + frame_error: Optional[BaseException] = None + if body: + try: + messages, trailers = split_response(body) + except Exception as exc: + frame_error = exc + + raw_status = trailers.get("grpc-status") + if raw_status is None: + raw_status = _header_lookup(resp_headers, "grpc-status") + raw_message = ( + trailers.get("grpc-message") or _header_lookup(resp_headers, "grpc-message") or "" + ) + message = urllib.parse.unquote(raw_message) + + if raw_status is None: + # No grpc-status anywhere AND either a non-200 or a body that is not + # grpc-web framing: a gRPC service did not answer this request at all. + if http_status != 200 or frame_error is not None: + raise _frame_error_to_rpc(http_status, url, body, frame_error) + if messages: + # Every grpc-web unary response must carry a grpc-status (trailer frame + # or header); a proxy that drops the trailer must not read as success. + raise AioRpcError( + code=StatusCode.INTERNAL, + details="grpc-web response missing grpc-status trailers", + ) + code = StatusCode.OK + else: + code = status_from_int(int(raw_status)) + + if code is not StatusCode.OK: + raise AioRpcError(code=code, details=message) + if frame_error is not None: + # grpc-status said OK but the body will not parse — report what actually + # came back rather than a bare "no message frame". + raise _frame_error_to_rpc(http_status, url, body, frame_error) + if len(messages) > 1: + raise AioRpcError( + code=StatusCode.INTERNAL, + details=f"unary grpc-web response carried {len(messages)} message frames", + ) + if not messages: + details = "grpc-web response contained no message frame" + if raw_status is None: + # HTTP 200, no body frames, and no grpc-status anywhere: the classic + # signature of a trailers-only error response whose grpc-status / + # grpc-message headers were stripped by CORS in the browser. + details += ( + " and no grpc-status was visible. If this is a cross-origin browser " + "request, configure the grpc-web proxy to send " + "'Access-Control-Expose-Headers: grpc-status, grpc-message' so " + "trailers-only error responses are readable." + ) + raise AioRpcError(code=StatusCode.INTERNAL, details=details) + return deserialize(messages[0]) + + +_BODY_EXCERPT_LIMIT = 200 + + +def _body_excerpt(body: bytes, limit: int = _BODY_EXCERPT_LIMIT) -> str: + """Render a short, printable, one-line excerpt of a response body for error details. + + The body here is whatever a server or proxy sent — JSON, HTML, or binary — so decode + leniently and drop non-printables: building an error detail must never itself raise. + """ + if not body: + return "" + text = body[:limit].decode("utf-8", "replace") + text = " ".join("".join(ch if ch.isprintable() else " " for ch in text).split()) + if not text: + return f"<{len(body)} non-printable bytes>" + return text + ("..." if len(body) > limit else "") + + +def _no_path_prefix_hint() -> str: + # Lazy import: this module is imported while ``weaviate/__init__`` is still + # bootstrapping the shim under Emscripten. + from weaviate.exceptions import GRPC_WEB_MIN_SERVER_VERSION, GRPC_WEB_SERVER_PATH_PREFIX + + return ( + "(no grpc_path_prefix set — under WebAssembly the connect helpers route gRPC to " + f"the REST endpoint under '{GRPC_WEB_SERVER_PATH_PREFIX}' by themselves, so use " + "one of them; hand-built ConnectionParams must set " + f"grpc_path_prefix='{GRPC_WEB_SERVER_PATH_PREFIX}' for Weaviate >= " + f"{GRPC_WEB_MIN_SERVER_VERSION}, or point grpc_host/grpc_port at a grpc-web " + "transcoder)" + ) + + +def _frame_error_to_rpc( + http_status: int, url: str, body: bytes, frame_error: Optional[BaseException] +) -> AioRpcError: + """Choose the error for a body that did not parse as grpc-web frames.""" + if http_status != 200 or isinstance(frame_error, (UnknownFrameFlagError, TruncatedFrameError)): + return _non_grpc_web_error(http_status, url, body, frame_error) + # Well-formed grpc-web up to the point of failure: a grpc-web endpoint answered but + # broke the protocol (compressed frame, message after trailer, …). + return AioRpcError( + code=StatusCode.INTERNAL, + details=f"malformed grpc-web response from {url or ''}: {frame_error}", + ) + + +def _non_grpc_web_error( + http_status: int, + url: str, + body: bytes, + frame_error: Optional[BaseException] = None, +) -> AioRpcError: + """Build the error for a response that is not a usable grpc-web response. + + Details always begin with ``HTTP `` and carry the request URL plus a body + excerpt (``weaviate/connect`` matches on that prefix). The status alone rarely + separates "endpoint missing" from "proxy misconfigured"; the server's own body + text usually does. + """ + truncated = isinstance(frame_error, TruncatedFrameError) + what = "not a grpc-web response" + if http_status == 200 and frame_error is not None: + if truncated: + what = f"the grpc-web body is truncated ({frame_error})" + else: + what = f"the body is not grpc-web framing ({frame_error})" + parts = [f"HTTP {http_status} from {url or ''}: {what}."] + + if http_status == 404: + # Two candidate causes, and the channel cannot tell them apart (it does not know + # the server version) — name both rather than guess. + parts.append( + "The grpc-web endpoint does not exist at that path: either this Weaviate " + "server predates 1.38.3, the first release to serve grpc-web natively, or " + "the configured grpc-web path prefix is wrong for the proxy in front of it. " + "Weaviate's native prefix is '/v1/grpc-web'." + ) + elif http_status == 405: + # A 405 can only come from an existing HTTP route: the prefix points at one. + parts.append( + "An HTTP route answered instead of the grpc-web endpoint (method not " + "allowed): the configured grpc-web path prefix is wrong. Weaviate's native " + "prefix is '/v1/grpc-web'." + ) + elif http_status in (502, 503, 504): + parts.append("Weaviate or the proxy in front of it is unavailable.") + elif http_status == 200 and truncated: + parts.append( + "The response was cut short — a proxy or browser buffering limit, or the " + "connection dropped mid-response." + ) + elif http_status == 200: + parts.append( + "Something other than a grpc-web endpoint answered — typically a proxy " + "error page or a single-page-app catch-all route serving index.html. Check " + "the grpc-web path prefix (Weaviate's native prefix is '/v1/grpc-web')." + ) + parts.append(f"Response body: {_body_excerpt(body)}") + + code = StatusCode.INTERNAL if http_status == 200 else _status_from_http(http_status) + return AioRpcError(code=code, details=" ".join(parts)) + + +def _status_from_http(http_status: int) -> StatusCode: + """Map an HTTP status to a gRPC status when no grpc-status is present. + + Mirrors the grpc-web spec's HTTP-to-gRPC code mapping. + """ + return { + 400: StatusCode.INTERNAL, + 401: StatusCode.UNAUTHENTICATED, + 403: StatusCode.PERMISSION_DENIED, + 404: StatusCode.UNIMPLEMENTED, + 429: StatusCode.UNAVAILABLE, + 502: StatusCode.UNAVAILABLE, + 503: StatusCode.UNAVAILABLE, + 504: StatusCode.UNAVAILABLE, + }.get(http_status, StatusCode.UNKNOWN) diff --git a/packages/web/src/weaviate_client_web/_framing.py b/packages/web/src/weaviate_client_web/_framing.py new file mode 100644 index 000000000..7e6a3dfc8 --- /dev/null +++ b/packages/web/src/weaviate_client_web/_framing.py @@ -0,0 +1,101 @@ +r"""grpc-web binary framing (``application/grpc-web+proto``). + +A grpc-web message frame is a 1-byte flag + 4-byte big-endian length + payload: + + +--------+----------------+----------------------+ + | flag | length (uint32)| payload (length bytes)| + +--------+----------------+----------------------+ + +The flag's high bit (``0x80``) marks a trailer frame whose payload is an +HTTP/1-style header block (``grpc-status: 0\\r\\ngrpc-message: ...``). The low bit +(``0x01``) marks a compressed message, which this transport neither sends nor +accepts. A unary grpc-web response body is one or more message frames followed by +exactly one trailer frame (or a "trailers-only" response carrying the status in +the HTTP headers, handled by the caller). +""" + +import struct +from typing import Dict, Iterator, List, Tuple + +_FLAG_TRAILER = 0x80 +_FLAG_COMPRESSED = 0x01 +_KNOWN_FLAGS = _FLAG_TRAILER | _FLAG_COMPRESSED +_HEADER = struct.Struct(">BI") # 1 flag byte + 4-byte big-endian length + + +class FrameError(ValueError): + """The body is not a well-formed grpc-web response.""" + + +class UnknownFrameFlagError(FrameError): + """A flag byte outside the grpc-web set: the body is not grpc-web framing (JSON, HTML, …).""" + + +class TruncatedFrameError(FrameError): + """The body ends before the length its frame header announces.""" + + +def encode_message(payload: bytes) -> bytes: + """Frame a single (uncompressed) protobuf payload for sending.""" + return _HEADER.pack(0x00, len(payload)) + payload + + +def iter_frames(buf: bytes) -> Iterator[Tuple[int, bytes]]: + """Yield ``(flag, payload)`` for each frame in a grpc-web response body.""" + off, n = 0, len(buf) + while off < n: + # Validate the flag before the length so a text body ('{', '<') is reported as + # non-grpc-web rather than as a truncated frame with a garbage length. + flag = buf[off] + if flag & ~_KNOWN_FLAGS: + raise UnknownFrameFlagError(f"unknown grpc-web frame flag 0x{flag:02x} at byte {off}") + if off + 5 > n: + raise TruncatedFrameError(f"truncated grpc-web frame header at byte {off}") + _, length = _HEADER.unpack_from(buf, off) + off += 5 + if off + length > n: + raise TruncatedFrameError( + f"truncated grpc-web frame: header announces {length} bytes, {n - off} remain" + ) + yield flag, buf[off : off + length] + off += length + + +def parse_trailers(raw: bytes) -> Dict[str, str]: + """Parse a trailer frame payload into a lower-cased header dict. + + Decoded leniently on both sides of the colon: a proxy that does not percent-encode + ``grpc-message``, or a server error quoting a UTF-8 collection / tenant / property + name, puts raw non-ASCII bytes in the trailer, and one odd key must not discard the + ``grpc-status`` travelling with it. Lines are CRLF-terminated by spec; bare LF is + accepted. + """ + out: Dict[str, str] = {} + for line in raw.split(b"\n"): + line = line.rstrip(b"\r") + if not line: + continue + key, _, value = line.partition(b":") + name = key.strip().decode("utf-8", "replace").lower() + out[name] = value.strip().decode("utf-8", "replace") + return out + + +def split_response(body: bytes) -> Tuple[List[bytes], Dict[str, str]]: + """Split a grpc-web response body into message payloads and trailers.""" + messages: List[bytes] = [] + trailers: Dict[str, str] = {} + seen_trailer = False + for flag, payload in iter_frames(body): + if flag & _FLAG_TRAILER: + trailers.update(parse_trailers(payload)) + seen_trailer = True + elif flag & _FLAG_COMPRESSED: + raise FrameError( + "compressed grpc-web message frames are not supported by this transport" + ) + elif seen_trailer: + raise FrameError("message frame after the trailer frame") + else: + messages.append(payload) + return messages, trailers diff --git a/packages/web/src/weaviate_client_web/_httpx_fetch.py b/packages/web/src/weaviate_client_web/_httpx_fetch.py new file mode 100644 index 000000000..e3f0ddf4a --- /dev/null +++ b/packages/web/src/weaviate_client_web/_httpx_fetch.py @@ -0,0 +1,208 @@ +"""fetch-based httpx transport for Pyodide/Emscripten. + +The base client's REST path uses ``httpx.AsyncClient`` with explicit +``httpx.AsyncHTTPTransport`` mounts (``weaviate/connect/v4.py``). httpcore opens raw +sockets, which do not exist under WASM, so without this module every REST call +(``is_ready``, collection config, batch references, …) fails with an empty connection +error even though the grpc-web data path works. + +Installing reroutes ``AsyncHTTPTransport.handle_async_request`` through the browser's +``fetch`` via ``pyodide.http.pyfetch`` — the same install-globally-under-Emscripten +philosophy as the grpc shim in ``_shim.py``. Responses are fully buffered, which matches +how the base client consumes them (JSON bodies, no streaming). + +It installs under Emscripten even when Pyodide's bundled httpx carries its own JS-fetch +transport (``httpx/_transports/jsfetch.py``): that transport reads ``Response.body`` +unconditionally, which is ``null`` for HEAD requests and 204 responses (``data.exists``, +``data.delete_by_id``, ``tenants.exists``, …), and it does not enforce the per-request +read timeout end-to-end. + +Known divergences from native httpx (acceptable for the weaviate client's usage): +- the browser's fetch follows redirects internally, so httpx never sees a 3xx; +- multi-value response headers (e.g. Set-Cookie) are folded into one value; +- responses are fully buffered (no streaming). +""" + +import math +import sys +from typing import Callable, Dict, Optional + +import httpx + +_installed = False +_original_handle_async_request: Optional[Callable] = None + +# Hop-by-hop / connection-managed headers that the browser's fetch controls itself. +# Browsers silently drop forbidden headers, but Node's undici (used by the CPython/Node +# test path) rejects some of them outright, so strip them before handing off. +_FETCH_MANAGED_HEADERS = { + "host", + "connection", + "accept-encoding", + "content-length", + "transfer-encoding", +} + +# Response headers describing the wire encoding of the body. fetch decompresses +# responses transparently, so the bytes handed to httpx are already plain; passing the +# original content-encoding through makes httpx run its decoders over them again and +# raise DecodingError, and the original content-length no longer matches the body. +# (Browsers usually hide content-encoding on CORS responses, which is why this never +# fired live — same-origin and Node fetch do expose it.) +_FETCH_DECODED_RESPONSE_HEADERS = { + "content-encoding", + "content-length", +} + +_TIMEOUT_HINTS = ("timeout", "timed out", "abort") + +# JS timers take a signed 32-bit millisecond delay; anything larger overflows and fires +# immediately, so a huge timeout would abort every request at once. +_MAX_ABORT_SIGNAL_MS = 2**31 - 1 + + +async def _read_request_body(request: httpx.Request) -> bytes: + try: + return request.content + except httpx.RequestNotRead: + return await request.aread() + + +def _pick_timeout(request: httpx.Request) -> Optional[float]: + """Pick the request deadline from httpx's timeout extension: the ``read`` value only. + + The base client sets ``read`` per request and passes ``read=None`` for "no deadline" + while still carrying a ``pool`` value; ``connect`` is not separable under fetch and a + pool-acquire timeout has no meaning there, so neither may stand in for ``read``. + """ + timeouts = request.extensions.get("timeout") or {} + return timeouts.get("read") + + +def _abort_signal_ms(timeout: Optional[float]) -> Optional[int]: + """Milliseconds for ``AbortSignal.timeout``; ``None`` means no client-side deadline. + + Zero, negative and non-finite timeouts all mean "no deadline". Rounded up so a + sub-millisecond timeout never becomes an immediate abort. + """ + if timeout is None or not math.isfinite(timeout) or timeout <= 0: + return None + return min(math.ceil(timeout * 1000), _MAX_ABORT_SIGNAL_MS) + + +def _map_fetch_error( + e: BaseException, request: httpx.Request, deadline_set: bool +) -> httpx.TransportError: + """Translate a pyfetch failure into httpx's exception taxonomy. + + Pyodide surfaces every JS fetch rejection (network down, DNS, CORS, CSP, an + AbortSignal firing) as OSError — or pyodide.http.AbortError, an OSError subclass — + never as an httpx exception. Without this mapping the base client cannot classify + failures (WeaviateConnectionError/WeaviateTimeoutError) and best-effort callers that + swallow httpx.RequestError break. + """ + msg = str(e) or repr(e) + if deadline_set and any(hint in msg.lower() for hint in _TIMEOUT_HINTS): + return httpx.ReadTimeout(msg, request=request) + return httpx.ConnectError(msg, request=request) + + +def _validate_header(name: str, value: str) -> None: + # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at + # send time; this transport bypasses h11, so mirror that defence here rather than + # delegating it entirely to the JS runtime's fetch. + if any(c in name or c in value for c in ("\r", "\n", "\0")): + raise httpx.LocalProtocolError(f"Illegal character in header {name!r}") + + +async def _fetch_handle_async_request( + self: httpx.AsyncHTTPTransport, request: httpx.Request +) -> httpx.Response: + from pyodide.http import pyfetch # type: ignore[import-not-found] + + headers: Dict[str, str] = {} + for k, v in request.headers.items(): + if k.lower() in _FETCH_MANAGED_HEADERS: + continue + _validate_header(k, v) + headers[k] = v + kwargs: Dict[str, object] = {} + body = await _read_request_body(request) + if body: + # fetch rejects GET/HEAD requests that carry a body + kwargs["body"] = body + + deadline_set = False + deadline_ms = _abort_signal_ms(_pick_timeout(request)) + if deadline_ms is not None: + try: + from js import AbortSignal # type: ignore[import-not-found] + + kwargs["signal"] = AbortSignal.timeout(deadline_ms) + deadline_set = True + except Exception: # pragma: no cover - AbortSignal.timeout availability varies + pass + + try: + response = await pyfetch(str(request.url), method=request.method, headers=headers, **kwargs) + # A body-less response (HEAD, 204) reads as b"": fetch resolves a null body to + # an empty ArrayBuffer. + data = await response.bytes() + except OSError as e: # incl. pyodide.http.AbortError + raise _map_fetch_error(e, request, deadline_set) from e + + try: + resp_headers = { + k: v + for k, v in dict(response.headers).items() + if k.lower() not in _FETCH_DECODED_RESPONSE_HEADERS + } + except Exception: # pragma: no cover - header shape varies across Pyodide versions + resp_headers = {} + # Hand httpx an unread stream, as its own transports do: the client reads it and + # only then stamps ``response.elapsed``, which the batch-references path relies on. + return httpx.Response( + status_code=int(response.status), + headers=resp_headers, + stream=httpx.ByteStream(data), + request=request, + ) + + +# sentinel so other packages (and uninstall) can recognise the patched method +_fetch_handle_async_request.__weaviate_fetch_shim__ = True # type: ignore[attr-defined] + + +def install_fetch_transport(force: bool = False) -> None: + """Patch ``httpx.AsyncHTTPTransport`` to send requests through ``fetch``. + + Installs only under Emscripten unless ``force=True`` (CPython testing, where a + ``pyodide`` stub must be importable). Idempotent. + """ + global _installed, _original_handle_async_request + if _installed: + return + if not force and sys.platform != "emscripten": + return + # Fail fast: the handler imports pyfetch per request, so a missing pyodide module + # would otherwise surface as a confusing ModuleNotFoundError on the first request. + from pyodide.http import pyfetch # type: ignore[import-not-found] # noqa: F401 + + _original_handle_async_request = httpx.AsyncHTTPTransport.handle_async_request + httpx.AsyncHTTPTransport.handle_async_request = _fetch_handle_async_request # type: ignore[method-assign] + _installed = True + + +def uninstall_fetch_transport() -> None: + """Restore the original ``httpx.AsyncHTTPTransport`` behaviour. No-op if not installed.""" + global _installed, _original_handle_async_request + if not _installed: + return + assert _original_handle_async_request is not None + httpx.AsyncHTTPTransport.handle_async_request = _original_handle_async_request # type: ignore[method-assign] + _original_handle_async_request = None + _installed = False + + +def is_fetch_transport_installed() -> bool: + return _installed diff --git a/packages/web/src/weaviate_client_web/_sender.py b/packages/web/src/weaviate_client_web/_sender.py new file mode 100644 index 000000000..d41f9f6c0 --- /dev/null +++ b/packages/web/src/weaviate_client_web/_sender.py @@ -0,0 +1,60 @@ +"""HTTP senders for the grpc-web transport. + +A *sender* is ``async def sender(url, headers, body, timeout) -> (status, headers, body)``. +The default uses ``pyodide.http.pyfetch`` (browser fetch); a sender can be injected for +testing or for non-browser runtimes via :func:`weaviate_client_web.set_sender`. +""" + +from typing import Awaitable, Callable, Dict, Optional, Tuple + +Sender = Callable[ + [str, Dict[str, str], bytes, Optional[float]], + Awaitable[Tuple[int, Dict[str, str], bytes]], +] + + +async def pyfetch_sender( + url: str, headers: Dict[str, str], body: bytes, timeout: Optional[float] +) -> Tuple[int, Dict[str, str], bytes]: + """Default browser sender. + + Imports ``pyodide.http`` lazily so this module stays importable on CPython (where + ``pyodide`` does not exist). ``pyfetch`` has no timeout parameter of its own; the + call deadline is enforced by ``GrpcWebChannel._unary`` via ``asyncio.wait_for``. + """ + from pyodide.http import pyfetch # type: ignore[import-not-found] + + response = await pyfetch(url, method="POST", headers=headers, body=body) + data = await response.bytes() + try: + resp_headers = dict(response.headers) + except Exception: # pragma: no cover - header shape varies across Pyodide versions + resp_headers = {} + return int(response.status), resp_headers, data + + +def make_httpx_sender(client: Optional[object] = None) -> Sender: + """Build a sender backed by ``httpx.AsyncClient`` for CPython tests/integration. + + Targets a grpc-web transcoder (Envoy / connectrpc vanguard). + """ + import httpx + + async def _send( + url: str, headers: Dict[str, str], body: bytes, timeout: Optional[float] + ) -> Tuple[int, Dict[str, str], bytes]: + owns_client = client is None + active = client or httpx.AsyncClient() + assert isinstance(active, httpx.AsyncClient) + try: + response = await active.post(url, headers=headers, content=body, timeout=timeout) + return ( + response.status_code, + {k.lower(): v for k, v in response.headers.items()}, + response.content, + ) + finally: + if owns_client: + await active.aclose() + + return _send diff --git a/packages/web/src/weaviate_client_web/_shim.py b/packages/web/src/weaviate_client_web/_shim.py new file mode 100644 index 000000000..b04b8837a --- /dev/null +++ b/packages/web/src/weaviate_client_web/_shim.py @@ -0,0 +1,283 @@ +"""A minimal pure-Python stand-in for the ``grpc`` API surface ``weaviate-client`` uses. + +It covers what ``weaviate-client`` touches at import time and on the async unary data +path. It is installed into ``sys.modules`` (as ``grpc``, ``grpc.aio``, ``grpc._utilities``, +``grpc.aio._typing``, ``grpc.experimental``) *before* ``import weaviate`` so the client +loads under Pyodide/Emscripten, where the real ``grpcio`` C-extension wheel does not +exist. The shim satisfies two contracts at once: + +1. **Import surface** — every ``import grpc`` / ``from grpc(.aio) import ...`` executed + while ``weaviate`` and its generated ``*_pb2_grpc`` stubs are imported + (``weaviate/config.py``, ``exceptions.py``, ``retry.py``, ``connect/base.py``, + ``connect/v4.py``, and the v6300 stub's ``grpc.__version__`` / + ``grpc._utilities.first_version_is_lower`` version gate). +2. **Runtime type contract** — :class:`AioChannel` becomes ``grpc.aio.Channel`` so the + real grpc-web channel (which subclasses it) passes the + ``isinstance(..., grpc.aio.Channel)`` assertions in ``connect/v4.py``; + :class:`AioRpcError` is the error the client catches and inspects via ``.code()`` / + ``.details()`` (``exceptions.py``, ``retry.py``). +""" + +import enum +import sys +import types +from typing import Any, Optional + +# grpcio reports 1.72.1 as the version that the v6300 generated stub requires; matching +# it makes the stub's import-time version gate pass. See weaviate/proto/v1/__init__.py. +FAKE_GRPC_VERSION = "1.72.1" + +_SHIM_MARKER = "__weaviate_client_web_shim__" + + +class StatusCode(enum.Enum): + """Mirror of ``grpc.StatusCode``. + + ``value`` is the canonical ``(int, str)`` tuple, matching grpcio so ``code.value[0]`` + / ``code.value[1]`` (``exceptions.py``) and ``code.name`` (``connect/v4.py``) behave + identically. + """ + + OK = (0, "ok") + CANCELLED = (1, "cancelled") + UNKNOWN = (2, "unknown") + INVALID_ARGUMENT = (3, "invalid argument") + DEADLINE_EXCEEDED = (4, "deadline exceeded") + NOT_FOUND = (5, "not found") + ALREADY_EXISTS = (6, "already exists") + PERMISSION_DENIED = (7, "permission denied") + RESOURCE_EXHAUSTED = (8, "resource exhausted") + FAILED_PRECONDITION = (9, "failed precondition") + ABORTED = (10, "aborted") + OUT_OF_RANGE = (11, "out of range") + UNIMPLEMENTED = (12, "unimplemented") + INTERNAL = (13, "internal") + UNAVAILABLE = (14, "unavailable") + DATA_LOSS = (15, "data loss") + UNAUTHENTICATED = (16, "unauthenticated") + + +_BY_NUMBER = {member.value[0]: member for member in StatusCode} + + +def status_from_int(code: int) -> StatusCode: + """Map a numeric grpc-status to a :class:`StatusCode` (``UNKNOWN`` if unmapped).""" + return _BY_NUMBER.get(code, StatusCode.UNKNOWN) + + +class RpcError(Exception): + """Stand-in for ``grpc.RpcError`` (imported by ``retry.py``).""" + + +class Call: + """Stand-in for ``grpc.Call`` (imported by ``exceptions.py`` / ``retry.py``). + + Only used for ``isinstance``/type-import purposes; the async-only WASM path raises + :class:`AioRpcError`, never a sync ``Call``. + """ + + def code(self) -> StatusCode: # pragma: no cover - never instantiated under WASM + raise NotImplementedError + + def details(self) -> str: # pragma: no cover + raise NotImplementedError + + +class AioRpcError(RpcError): + """Stand-in for ``grpc.aio.AioRpcError``. + + Raised by the grpc-web multicallable on a non-OK status; exposes the same + ``code()`` / ``details()`` surface the client uses. + """ + + def __init__( + self, + code: StatusCode, + initial_metadata: Any = None, + trailing_metadata: Any = None, + details: str = "", + debug_error_string: Optional[str] = None, + ) -> None: + self._code = code + self._details = details + self._initial_metadata = initial_metadata + self._trailing_metadata = trailing_metadata + self._debug_error_string = debug_error_string + super().__init__(f"") + + def code(self) -> StatusCode: + return self._code + + def details(self) -> str: + return self._details + + def initial_metadata(self) -> Any: + return self._initial_metadata + + def trailing_metadata(self) -> Any: + return self._trailing_metadata + + def debug_error_string(self) -> Optional[str]: + return self._debug_error_string + + +class StreamStreamCall: + """Stand-in for ``grpc.aio.StreamStreamCall`` (imported as a type by ``connect/v4.py``).""" + + +class ChannelCredentials: + """Stand-in for ``grpc.ChannelCredentials`` (imported by ``config.py``).""" + + +def ssl_channel_credentials(*_args: Any, **_kwargs: Any) -> ChannelCredentials: + return ChannelCredentials() + + +class SyncChannel: + """Stand-in for ``grpc.Channel`` (sync). + + Never instantiated under WASM — the sync channel factory raises (the WASM transport + is async-only). + """ + + +class AioChannel: + """Become ``grpc.aio.Channel``. + + The grpc-web channel subclasses this so the ``isinstance(..., grpc.aio.Channel)`` + assertions in ``connect/v4.py`` hold. + """ + + +def first_version_is_lower(_version: str, _other: str) -> bool: + """Stand-in for ``grpc._utilities.first_version_is_lower``. + + Returning ``False`` makes the v6300 stub's import-time version gate + (``weaviate_pb2_grpc.py``) pass. + """ + return False + + +_ASYNC_ONLY_MESSAGE = ( + "weaviate-client-web provides an asynchronous-only gRPC transport under " + "WebAssembly/Pyodide. Use an async client (weaviate.use_async_with_local / " + "use_async_with_weaviate_cloud / use_async_with_custom, or WeaviateAsyncClient); " + "the synchronous client is not supported in the browser." +) + + +def _sync_channel_unsupported(*_args: Any, **_kwargs: Any) -> "AioChannel": + raise RuntimeError(_ASYNC_ONLY_MESSAGE) + + +def _path_prefix_from_options(options: Any) -> str: + """Extract the ``("grpc-web.path_prefix", prefix)`` channel option, or "" if absent.""" + for item in options or (): + if isinstance(item, (tuple, list)) and len(item) == 2 and item[0] == "grpc-web.path_prefix": + return item[1] or "" + return "" + + +def _aio_secure_channel( + target: Optional[str] = None, credentials: Any = None, options: Any = None, **_kw: Any +) -> AioChannel: + from ._channel import GrpcWebChannel + + return GrpcWebChannel( + target=target, + secure=True, + options=options, + path_prefix=_path_prefix_from_options(options), + ) + + +def _aio_insecure_channel( + target: Optional[str] = None, options: Any = None, **_kw: Any +) -> AioChannel: + from ._channel import GrpcWebChannel + + return GrpcWebChannel( + target=target, + secure=False, + options=options, + path_prefix=_path_prefix_from_options(options), + ) + + +def _noop(*_args: Any, **_kwargs: Any) -> None: + """Inert stand-in for imported-but-unused server-side stub-registration helpers. + + e.g. ``grpc.unary_unary_rpc_method_handler``: imported by generated ``*_pb2_grpc`` + code, never called by the client. + """ + return None + + +def is_installed() -> bool: + return getattr(sys.modules.get("grpc"), _SHIM_MARKER, False) is True + + +def install(force: bool = False) -> bool: + """Install the shim into ``sys.modules`` as ``grpc`` and submodules. + + On normal platforms this is a no-op unless ``force=True`` — we must never clobber a + real, working ``grpcio``. Under Emscripten the bootstrap calls this automatically. + Returns ``True`` if the shim is in place afterwards. + """ + if not force and sys.platform != "emscripten": + return False + if is_installed(): + return True + + # Modules are populated via __dict__.update — dynamic module synthesis, so static + # type checkers do not flag each attribute assignment. + utilities = types.ModuleType("grpc._utilities") + utilities.__dict__["first_version_is_lower"] = first_version_is_lower + + experimental = types.ModuleType("grpc.experimental") + experimental.__dict__.update(unary_unary=_noop, stream_stream=_noop) + + aio_typing = types.ModuleType("grpc.aio._typing") + aio_typing.__dict__["ChannelArgumentType"] = Any + + aio = types.ModuleType("grpc.aio") + aio.__dict__.update( + Channel=AioChannel, + AioRpcError=AioRpcError, + StreamStreamCall=StreamStreamCall, + secure_channel=_aio_secure_channel, + insecure_channel=_aio_insecure_channel, + _typing=aio_typing, + ) + + grpc_mod = types.ModuleType("grpc") + grpc_mod.__dict__.update( + { + "__version__": FAKE_GRPC_VERSION, + _SHIM_MARKER: True, + "StatusCode": StatusCode, + "RpcError": RpcError, + "Call": Call, + "Channel": SyncChannel, + "ChannelCredentials": ChannelCredentials, + "ssl_channel_credentials": ssl_channel_credentials, + "secure_channel": _sync_channel_unsupported, + "insecure_channel": _sync_channel_unsupported, + # Imported (never called) by generated *_pb2_grpc servicer/registration code. + "unary_unary_rpc_method_handler": _noop, + "stream_stream_rpc_method_handler": _noop, + "unary_stream_rpc_method_handler": _noop, + "stream_unary_rpc_method_handler": _noop, + "method_handlers_generic_handler": _noop, + "_utilities": utilities, + "experimental": experimental, + "aio": aio, + } + ) + + sys.modules["grpc"] = grpc_mod + sys.modules["grpc._utilities"] = utilities + sys.modules["grpc.experimental"] = experimental + sys.modules["grpc.aio"] = aio + sys.modules["grpc.aio._typing"] = aio_typing + return True diff --git a/packages/web/src/weaviate_client_web/py.typed b/packages/web/src/weaviate_client_web/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/packages/web/tests/conftest.py b/packages/web/tests/conftest.py new file mode 100644 index 000000000..fe4afb09d --- /dev/null +++ b/packages/web/tests/conftest.py @@ -0,0 +1,7 @@ +import pathlib +import sys + +# Make the package importable without an editable install. +_SRC = pathlib.Path(__file__).resolve().parents[1] / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py new file mode 100644 index 000000000..fd18a35a3 --- /dev/null +++ b/packages/web/tests/test_framing.py @@ -0,0 +1,120 @@ +import struct + +import pytest + +from weaviate_client_web._framing import ( + FrameError, + TruncatedFrameError, + UnknownFrameFlagError, + encode_message, + iter_frames, + parse_trailers, + split_response, +) + + +def _frame(payload: bytes, flag: int = 0x00) -> bytes: + return struct.pack(">BI", flag, len(payload)) + payload + + +def test_encode_message_round_trip(): + framed = encode_message(b"hello") + frames = list(iter_frames(framed)) + assert frames == [(0x00, b"hello")] + + +def test_split_response_message_and_trailer(): + body = _frame(b"payload") + _frame(b"grpc-status:0\r\ngrpc-message:\r\n", 0x80) + messages, trailers = split_response(body) + assert messages == [b"payload"] + assert trailers["grpc-status"] == "0" + assert trailers["grpc-message"] == "" + + +def test_split_response_multiple_messages(): + # the splitter returns every message frame; whether more than one is acceptable is + # the channel's decision (a unary RPC rejects it) + body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:0\r\n", 0x80) + messages, trailers = split_response(body) + assert messages == [b"a", b"bb"] + assert trailers["grpc-status"] == "0" + + +def test_split_response_message_after_trailer_raises(): + body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") + with pytest.raises(FrameError, match="after the trailer"): + split_response(body) + + +def test_split_response_trailers_only(): + body = _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80) + messages, trailers = split_response(body) + assert messages == [] + assert trailers == {"grpc-status": "7", "grpc-message": "denied"} + + +def test_parse_trailers_lowercases_keys(): + parsed = parse_trailers(b"Grpc-Status:0\r\nGrpc-Message:ok\r\n") + assert parsed == {"grpc-status": "0", "grpc-message": "ok"} + + +def test_parse_trailers_keeps_status_when_message_is_not_ascii(): + # A proxy that does not percent-encode grpc-message, or a server error quoting a + # UTF-8 collection/tenant name, sends raw non-ASCII bytes. Decoding must not raise: + # the grpc-status travelling with it is the part the client acts on. + parsed = parse_trailers("grpc-status:5\r\ngrpc-message:Café not found\r\n".encode("utf-8")) + assert parsed["grpc-status"] == "5" + assert parsed["grpc-message"] == "Café not found" + + +def test_parse_trailers_keeps_status_when_message_is_invalid_utf8(): + # latin-1 (or any non-UTF-8) bytes must degrade to replacement chars, not an error + parsed = parse_trailers(b"grpc-status:9\r\ngrpc-message:tenant caf\xe9 is COLD\r\n") + assert parsed["grpc-status"] == "9" + assert parsed["grpc-message"].startswith("tenant caf") + + +def test_split_response_survives_non_ascii_trailer(): + body = _frame("grpc-status:7\r\ngrpc-message:accès refusé\r\n".encode("utf-8"), 0x80) + messages, trailers = split_response(body) + assert messages == [] + assert trailers["grpc-status"] == "7" + + +def test_parse_trailers_accepts_lf_only_lines(): + parsed = parse_trailers(b"grpc-status:0\ngrpc-message:ok\n") + assert parsed == {"grpc-status": "0", "grpc-message": "ok"} + + +def test_parse_trailers_keeps_status_when_a_key_is_not_ascii(): + # one odd key from a proxy must not throw away the whole block + parsed = parse_trailers("x-caf\u00e9:1\r\ngrpc-status:0\r\n".encode("utf-8")) + assert parsed["grpc-status"] == "0" + assert parsed["x-caf\u00e9"] == "1" + + +def test_truncated_frame_raises(): + framed = encode_message(b"hello")[:-2] + with pytest.raises(TruncatedFrameError): + list(iter_frames(framed)) + with pytest.raises(TruncatedFrameError): + list(iter_frames(b"\x00\x00\x00")) # shorter than one frame header + + +@pytest.mark.parametrize("first_byte", [b"{", b"<", b"\x02", b"\x40", b"\xff"]) +def test_unknown_frame_flag_raises(first_byte): + # a JSON / HTML body, or a flag bit this transport does not know + body = first_byte + b"\x00\x00\x00\x01x" + with pytest.raises(UnknownFrameFlagError, match="unknown grpc-web frame flag"): + list(iter_frames(body)) + + +def test_frame_errors_are_value_errors(): + assert issubclass(TruncatedFrameError, ValueError) + assert issubclass(UnknownFrameFlagError, ValueError) + + +def test_compressed_message_frame_rejected(): + body = _frame(b"x", 0x01) + with pytest.raises(FrameError, match="compressed"): + split_response(body) diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py new file mode 100644 index 000000000..63480311e --- /dev/null +++ b/packages/web/tests/test_httpx_fetch.py @@ -0,0 +1,604 @@ +"""Tests for the fetch-based httpx transport (_httpx_fetch.py). + +In-process tests call ``_fetch_handle_async_request`` directly with a fake +``pyodide.http`` module injected into ``sys.modules`` — no global monkeypatch of +``httpx.AsyncHTTPTransport`` is needed, so the real httpx in the dev environment is left +untouched. Install semantics (which DO patch the class globally) run in fresh +subprocesses, mirroring test_shim_install.py. +""" + +import asyncio +import pathlib +import subprocess +import sys +import textwrap +import types +from typing import Any, Dict, List, Optional + +import httpx +import pytest + +from weaviate_client_web._httpx_fetch import ( + _MAX_ABORT_SIGNAL_MS, + _abort_signal_ms, + _fetch_handle_async_request, +) + +_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") + + +class FakeFetchResponse: + def __init__( + self, status: int = 200, headers: Optional[Any] = None, body: Optional[bytes] = b"" + ): + self.status = status + self.headers: Any = headers or {} + self._body = body + + async def bytes(self) -> bytes: # noqa: A003 - mirrors pyodide's FetchResponse API + # a null JS body (HEAD, 204) resolves to an empty ArrayBuffer, i.e. b"" + return b"" if self._body is None else self._body + + +class FakePyfetch: + def __init__(self, response: Optional[FakeFetchResponse] = None): + self.response = response or FakeFetchResponse() + self.calls: List[Dict[str, Any]] = [] + + async def __call__(self, url: str, **kwargs: Any) -> FakeFetchResponse: + self.calls.append({"url": url, **kwargs}) + return self.response + + +@pytest.fixture +def fake_pyfetch(monkeypatch) -> FakePyfetch: + fetch = FakePyfetch() + pyodide_mod = types.ModuleType("pyodide") + http_mod = types.ModuleType("pyodide.http") + http_mod.pyfetch = fetch # type: ignore[attr-defined] + pyodide_mod.http = http_mod # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "pyodide", pyodide_mod) + monkeypatch.setitem(sys.modules, "pyodide.http", http_mod) + return fetch + + +async def _handle_async(request: httpx.Request) -> httpx.Response: + # self is unused by the handler implementation; a bare transport instance suffices + transport = httpx.AsyncHTTPTransport.__new__(httpx.AsyncHTTPTransport) + response = await _fetch_handle_async_request(transport, request) + await response.aread() # httpx.AsyncClient reads non-streamed responses the same way + return response + + +def _handle(request: httpx.Request) -> httpx.Response: + return asyncio.run(_handle_async(request)) + + +class _FetchTransport(httpx.AsyncBaseTransport): + """Route an ``httpx.AsyncClient`` through the handler without patching httpx globally.""" + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + return await _fetch_handle_async_request(self, request) # type: ignore[arg-type] + + +def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: + async def main() -> httpx.Response: + async with httpx.AsyncClient(transport=_FetchTransport()) as client: + return await client.request(method, url, **kwargs) + + return asyncio.run(main()) + + +def test_basic_get_round_trip(fake_pyfetch): + fake_pyfetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' + ) + response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + + assert response.status_code == 200 + assert response.json() == {"version": "1.30.0"} + assert response.headers["content-type"] == "application/json" + call = fake_pyfetch.calls[0] + assert call["url"] == "http://h:8080/v1/meta" + assert call["method"] == "GET" + + +def test_response_has_request_attached_for_raise_for_status(fake_pyfetch): + fake_pyfetch.response = FakeFetchResponse(status=404, body=b"") + response = _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) + with pytest.raises(httpx.HTTPStatusError): + response.raise_for_status() + + +@pytest.mark.parametrize("status", [204, 404]) +def test_head_response_without_body_yields_empty_content(fake_pyfetch, status): + # data.exists() / tenants.exists() are HEAD requests answered 204/404 with a null + # body; the transport must hand httpx an empty response, not fail on the missing body + fake_pyfetch.response = FakeFetchResponse(status=status, body=None) + response = _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == status + assert response.content == b"" + assert "body" not in fake_pyfetch.calls[0] + + +def test_delete_204_without_body_yields_empty_content(fake_pyfetch): + # data.delete_by_id() / reference_delete() are answered 204 with a null body + fake_pyfetch.response = FakeFetchResponse(status=204, body=None) + response = _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == 204 + assert response.content == b"" + + +def test_body_less_response_through_async_client(fake_pyfetch): + # the full httpx.AsyncClient path (stream wrapping + read) on a body-less response + fake_pyfetch.response = FakeFetchResponse(status=204, body=None) + response = _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") + assert response.status_code == 204 + assert response.content == b"" + + +def test_response_through_async_client_exposes_elapsed_and_content(fake_pyfetch): + # the batch-references path reads ``res.elapsed``, which httpx only sets after it + # has read/closed a stream-backed response; a pre-loaded body never gets one + payload = b'[{"result": {"status": "SUCCESS"}}]' + fake_pyfetch.response = FakeFetchResponse(status=200, body=payload) + response = _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") + assert response.content == payload + assert response.json() == [{"result": {"status": "SUCCESS"}}] + assert response.elapsed.total_seconds() >= 0 + + +def test_fetch_managed_request_headers_stripped(fake_pyfetch): + request = httpx.Request( + "POST", + "http://h:8080/v1/objects", + headers={ + "authorization": "Bearer k", + "content-type": "application/json", + "host": "h:8080", + "connection": "keep-alive", + "accept-encoding": "gzip", + "transfer-encoding": "chunked", + }, + content=b"{}", + ) + _handle(request) + sent = fake_pyfetch.calls[0]["headers"] + assert sent["authorization"] == "Bearer k" + assert sent["content-type"] == "application/json" + for managed in ("host", "connection", "accept-encoding", "content-length", "transfer-encoding"): + assert managed not in sent + + +def test_get_without_body_omits_body_kwarg(fake_pyfetch): + # fetch rejects GET/HEAD requests that carry a body, so the kwarg must be absent + _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) + assert "body" not in fake_pyfetch.calls[0] + + +def test_post_body_passed(fake_pyfetch): + _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) + assert fake_pyfetch.calls[0]["body"] == b'{"query": "x"}' + + +def test_delete_with_body_passed(fake_pyfetch): + # the REST batch-delete path sends DELETE with a JSON body + _handle(httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}')) + assert fake_pyfetch.calls[0]["body"] == b'{"match": {}}' + + +def test_query_string_preserved_in_url(fake_pyfetch): + _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) + assert fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" + + +def test_content_encoding_stripped_from_response(fake_pyfetch): + # fetch hands back ALREADY-decompressed bytes; if the original content-encoding + # header were passed through, httpx.Response would gunzip a second time and raise + # DecodingError. content-length is stale for the same reason. + fake_pyfetch.response = FakeFetchResponse( + status=200, + headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, + body=b'{"version": "1.30.0"}', + ) + response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.json() == {"version": "1.30.0"} + assert "content-encoding" not in response.headers + assert response.headers["x-other"] == "kept" + + +def test_unreadable_response_headers_tolerated(fake_pyfetch): + class BadHeaders: + def keys(self): + raise TypeError("header shape varies across Pyodide versions") + + fake_pyfetch.response = FakeFetchResponse(status=200, body=b"ok") + fake_pyfetch.response.headers = BadHeaders() + response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.status_code == 200 + assert response.content == b"ok" + + +class _AbortSignalRecorder: + def __init__(self): + self.timeouts: List[int] = [] + + def timeout(self, ms: int): + self.timeouts.append(ms) + return f"signal-{ms}" + + +@pytest.fixture +def fake_abort_signal(monkeypatch) -> _AbortSignalRecorder: + recorder = _AbortSignalRecorder() + js_mod = types.ModuleType("js") + js_mod.AbortSignal = recorder # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "js", js_mod) + return recorder + + +def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request: + request = httpx.Request("GET", "http://h:8080/v1/meta") + request.extensions["timeout"] = timeouts + return request + + +def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_signal): + # mirrors what weaviate's AsyncClient puts in extensions: connect/read/write/pool + _handle(_request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0})) + assert fake_abort_signal.timeouts == [30000] + assert fake_pyfetch.calls[0]["signal"] == "signal-30000" + + +def test_read_none_means_no_deadline_even_with_pool_and_connect_set( + fake_pyfetch, fake_abort_signal +): + # what the base client hands over for a non-finite request timeout: read=None with the + # session pool timeout still set; falling back to pool/connect would abort a long + # insert after 5 s + _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5})) + _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) + assert fake_abort_signal.timeouts == [] + assert all("signal" not in c for c in fake_pyfetch.calls) + + +def test_read_timeout_alone_sets_the_deadline(fake_pyfetch, fake_abort_signal): + _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) + assert fake_abort_signal.timeouts == [7000] + assert fake_pyfetch.calls[0]["signal"] == "signal-7000" + + +def test_no_timeout_extension_sends_no_signal(fake_pyfetch, fake_abort_signal): + _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] + + +def test_missing_js_module_degrades_to_no_signal(fake_pyfetch): + # off-browser (no js module) the AbortSignal import fails; the request must still go out + assert "js" not in sys.modules + response = _handle( + _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) + ) + assert response.status_code == 200 + assert "signal" not in fake_pyfetch.calls[0] + + +def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): + # an explicit read=0 must not fall through to the 5s connect timeout, nor become an + # immediate AbortSignal.timeout(0) + _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] + + +@pytest.mark.parametrize( + "timeout,expected_ms", + [ + (None, None), + (0, None), + (-1, None), + (float("inf"), None), + (float("nan"), None), + (0.0001, 1), # rounds up: never an immediate AbortSignal.timeout(0) + (30.0, 30_000), + (1e8, _MAX_ABORT_SIGNAL_MS), + (1e10, _MAX_ABORT_SIGNAL_MS), + ], +) +def test_abort_signal_ms_bounds(timeout, expected_ms): + assert _abort_signal_ms(timeout) == expected_ms + + +def test_infinite_timeout_sends_no_signal(fake_pyfetch, fake_abort_signal): + # an inf read deadline reaching the transport: no signal, not an OverflowError + _handle( + _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) + ) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] + + +def test_huge_timeout_is_capped_to_int32_ms(fake_pyfetch, fake_abort_signal): + # setTimeout delays above 2^31-1 ms overflow and fire at once, aborting the request + _handle(_request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None})) + assert fake_abort_signal.timeouts == [_MAX_ABORT_SIGNAL_MS] + assert fake_pyfetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" + + +class RaisingPyfetch: + def __init__(self, exc: BaseException): + self.exc = exc + + async def __call__(self, url: str, **kwargs: Any): + raise self.exc + + +def _install_raising_pyfetch(monkeypatch, exc: BaseException) -> None: + pyodide_mod = types.ModuleType("pyodide") + http_mod = types.ModuleType("pyodide.http") + http_mod.pyfetch = RaisingPyfetch(exc) # type: ignore[attr-defined] + pyodide_mod.http = http_mod # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "pyodide", pyodide_mod) + monkeypatch.setitem(sys.modules, "pyodide.http", http_mod) + + +def test_fetch_failure_maps_to_httpx_connect_error(monkeypatch): + # pyodide surfaces JS fetch rejections as OSError; the base client can only classify + # httpx exceptions (WeaviateConnectionError etc.), so the shim must translate + _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) + with pytest.raises(httpx.ConnectError, match="Failed to fetch") as excinfo: + _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert isinstance(excinfo.value.__cause__, OSError) + + +def test_fetch_abort_with_deadline_maps_to_read_timeout(monkeypatch, fake_abort_signal): + # AbortSignal.timeout firing surfaces as an OSError subclass mentioning the abort; + # with a deadline set this must classify as a timeout, not a connection error + _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) + with pytest.raises(httpx.ReadTimeout, match="signal timed out"): + _handle(_request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None})) + + +def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error( + monkeypatch, fake_abort_signal +): + # nearly every weaviate request sets a read deadline; a plain network failure on + # such a request must remain a connection error, not become a timeout + _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) + with pytest.raises(httpx.ConnectError, match="Failed to fetch"): + _handle(_request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None})) + + +def test_fetch_abort_without_deadline_stays_connect_error(monkeypatch): + # the same message without a deadline set (no js module -> no signal) is not OUR + # timeout, so it must stay a connection error + _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) + assert "js" not in sys.modules + with pytest.raises(httpx.ConnectError): + _handle(_request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None})) + + +def test_empty_oserror_str_keeps_repr_detail(monkeypatch): + _install_raising_pyfetch(monkeypatch, OSError()) + with pytest.raises(httpx.ConnectError) as excinfo: + _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert "OSError" in str(excinfo.value) + + +def test_crlf_in_header_value_rejected(fake_pyfetch): + # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at + # send time; this transport bypasses h11 and must keep that defence + request = httpx.Request( + "GET", "http://h:8080/v1/meta", headers={"x-key": "val\r\nx-injected: evil"} + ) + with pytest.raises(httpx.LocalProtocolError): + _handle(request) + assert fake_pyfetch.calls == [] + + +# --------------------------------------------------------------------------- +# Install semantics: these patch httpx.AsyncHTTPTransport globally, so each +# scenario runs in a fresh subprocess (same pattern as test_shim_install.py). +# --------------------------------------------------------------------------- + +_FAKE_PYODIDE_PRELUDE = """ +import sys, types + +class _FakeResponse: + status = 200 + headers = {"content-type": "application/json"} + async def bytes(self): + return b'{"ok": true}' + +CALLS = [] +async def pyfetch(url, **kwargs): + CALLS.append((url, kwargs)) + return _FakeResponse() + +_pyodide = types.ModuleType("pyodide") +_http = types.ModuleType("pyodide.http") +_http.pyfetch = pyfetch +_pyodide.http = _http +sys.modules["pyodide"] = _pyodide +sys.modules["pyodide.http"] = _http +""" + + +def _run(body: str, prelude: str = "") -> subprocess.CompletedProcess: + script = f"import sys\nsys.path.insert(0, {_SRC!r})\n" + prelude + textwrap.dedent(body) + return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + + +def test_force_install_routes_async_client_through_pyfetch(): + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import asyncio, httpx + from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed + + install_fetch_transport(force=True) + assert is_fetch_transport_installed() + + async def main(): + async with httpx.AsyncClient() as client: + return await client.get("http://h:8080/v1/meta") + + resp = asyncio.run(main()) + assert resp.status_code == 200, resp.status_code + assert resp.json() == {"ok": True} + assert CALLS and CALLS[0][0] == "http://h:8080/v1/meta" + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_install_without_force_is_noop_off_emscripten(): + result = _run( + """ + import sys + assert sys.platform != "emscripten" + import httpx + before = httpx.AsyncHTTPTransport.handle_async_request + from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed + install_fetch_transport() + assert not is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is before + print("OK") + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_force_install_is_idempotent(): + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import httpx + from weaviate_client_web import install_fetch_transport + install_fetch_transport(force=True) + patched = httpx.AsyncHTTPTransport.handle_async_request + install_fetch_transport(force=True) + assert httpx.AsyncHTTPTransport.handle_async_request is patched + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_sync_transport_left_untouched(): + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import httpx + sync_before = httpx.HTTPTransport.handle_request + from weaviate_client_web import install_fetch_transport + install_fetch_transport(force=True) + assert httpx.HTTPTransport.handle_request is sync_before + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_uninstall_restores_original_transport(): + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import httpx + before = httpx.AsyncHTTPTransport.handle_async_request + from weaviate_client_web import ( + install_fetch_transport, + is_fetch_transport_installed, + uninstall_fetch_transport, + ) + uninstall_fetch_transport() # no-op when not installed + install_fetch_transport(force=True) + assert is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is not before + uninstall_fetch_transport() + assert not is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is before + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_patched_method_carries_sentinel(): + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import httpx + from weaviate_client_web import install_fetch_transport + assert not getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False + ) + install_fetch_transport(force=True) + assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False + ) is True + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_force_install_without_pyodide_fails_fast(): + # without a pyodide module the install must raise immediately, not let every later + # request die with a lazy ModuleNotFoundError + result = _run( + """ + import httpx + before = httpx.AsyncHTTPTransport.handle_async_request + from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed + try: + install_fetch_transport(force=True) + except ModuleNotFoundError: + assert not is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is before + print("OK") + else: + raise AssertionError("expected install to fail fast without pyodide") + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_emscripten_installs_even_when_platform_httpx_has_jsfetch(): + # Pyodide's bundled httpx ships a jsfetch transport that crashes on body-less + # responses (HEAD / 204); the shim must take over regardless of the httpx build + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import importlib.machinery, sys, types + + sys.platform = "emscripten" + fake = types.ModuleType("httpx._transports.jsfetch") + fake.__spec__ = importlib.machinery.ModuleSpec( + "httpx._transports.jsfetch", loader=None + ) + sys.modules["httpx._transports.jsfetch"] = fake + + import httpx + before = httpx.AsyncHTTPTransport.handle_async_request + from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed + install_fetch_transport() # no force: the platform alone must trigger it + assert is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is not before + assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False + ) is True + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout diff --git a/packages/web/tests/test_shim_install.py b/packages/web/tests/test_shim_install.py new file mode 100644 index 000000000..cdcbed56d --- /dev/null +++ b/packages/web/tests/test_shim_install.py @@ -0,0 +1,123 @@ +"""Shim/import tests. + +Installing the shim replaces ``sys.modules['grpc']`` process-wide, so each scenario runs +in a fresh subprocess to avoid clobbering the real ``grpc`` used by the rest of the suite. +""" + +import pathlib +import subprocess +import sys +import textwrap + +_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") + + +def _run(body: str) -> subprocess.CompletedProcess: + script = f"import sys\nsys.path.insert(0, {_SRC!r})\n" + textwrap.dedent(body) + return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + + +def test_import_weaviate_under_shim(): + result = _run( + """ + import weaviate_client_web + assert weaviate_client_web.install(force=True) is True + assert weaviate_client_web.is_installed() + + import grpc + assert getattr(grpc, "__weaviate_client_web_shim__", False) is True + assert grpc.__version__ == "1.72.1" + assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False + from grpc.aio._typing import ChannelArgumentType # noqa: F401 + + import weaviate # must not raise even though grpcio is shimmed + from weaviate.proto.v1 import weaviate_pb2_grpc + from weaviate_client_web import GrpcWebChannel + + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + assert stub.Search is not None + assert stub.BatchObjects is not None + assert stub.BatchDelete is not None + assert isinstance(ch, grpc.aio.Channel) + print("OK") + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_sync_channel_factory_raises_async_only(): + result = _run( + """ + import weaviate_client_web + weaviate_client_web.install(force=True) + import grpc + try: + grpc.insecure_channel("localhost:50051") + except RuntimeError as exc: + assert "async" in str(exc).lower() + print("OK") + else: + raise AssertionError("expected sync channel factory to raise") + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_real_proto_unary_round_trip_under_shim(): + result = _run( + """ + import asyncio + import struct + import weaviate_client_web + weaviate_client_web.install(force=True) + + import weaviate # noqa: F401 + from weaviate.proto.v1 import tenants_pb2, weaviate_pb2_grpc + + reply = tenants_pb2.TenantsGetReply() + payload = reply.SerializeToString() + + def frame(p, flag=0x00): + return struct.pack(">BI", flag, len(p)) + p + + body = frame(payload) + frame(b"grpc-status:0\\r\\n", 0x80) + + async def sender(url, headers, body_in, timeout): + assert headers["authorization"] == "Bearer k" + assert url.endswith("/weaviate.v1.Weaviate/TenantsGet") + return 200, {}, body + + weaviate_client_web.set_sender(sender) + from weaviate_client_web import GrpcWebChannel + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + + async def main(): + res = await stub.TenantsGet( + tenants_pb2.TenantsGetRequest(), + metadata=[("authorization", "Bearer k")], + timeout=5, + ) + assert isinstance(res, tenants_pb2.TenantsGetReply) + print("OK") + + asyncio.run(main()) + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_fake_grpc_version_matches_base_fallback(): + # In-process on purpose: nothing here installs the shim, we only compare the two + # copies of the pinned version. The shim advertises FAKE_GRPC_VERSION as + # grpc.__version__ and the base package falls back to _GRPCIO_FALLBACK_VERSION + # under Emscripten — the vendored stubs' version gates see both, so they must + # never drift apart. + from weaviate.proto.v1 import _GRPCIO_FALLBACK_VERSION + from weaviate_client_web._shim import FAKE_GRPC_VERSION + + assert FAKE_GRPC_VERSION == _GRPCIO_FALLBACK_VERSION diff --git a/packages/web/tests/test_single_import.py b/packages/web/tests/test_single_import.py new file mode 100644 index 000000000..562f31e0a --- /dev/null +++ b/packages/web/tests/test_single_import.py @@ -0,0 +1,139 @@ +"""Tests for the single-import hook in the base client (``weaviate/__init__.py``). + +The hook fires on ``sys.platform == "emscripten"`` and (via the companion's bootstrap) +replaces ``sys.modules['grpc']`` process-wide, so each scenario runs in a fresh +subprocess with the platform faked before ``import weaviate`` — the same pattern as +test_shim_install.py / test_httpx_fetch.py's install tests. +""" + +import pathlib +import subprocess +import sys +import textwrap + +_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") +_REPO_ROOT = str(pathlib.Path(__file__).resolve().parents[3]) + +# CPython derives the _sysconfigdata module name from sys.platform on first use, so a +# faked platform breaks any later sysconfig lookup (pydantic imports zoneinfo, which +# calls sysconfig.get_config_var). Prime the cache before faking. +_PRIME_SYSCONFIG = """ +import sysconfig + +sysconfig.get_config_vars() +""" + +# The companion's bootstrap installs the fetch transport under Emscripten and fails fast +# if pyodide.http cannot be imported, so a faked platform needs a stand-in module. +_FAKE_PYODIDE = """ +import types + +_pyodide = types.ModuleType("pyodide") +_http = types.ModuleType("pyodide.http") +_http.pyfetch = None +_pyodide.http = _http +sys.modules["pyodide"] = _pyodide +sys.modules["pyodide.http"] = _http +""" + + +def _run( + body: str, *, prelude: str = "", path_entry: str = _SRC, no_site: bool = False +) -> subprocess.CompletedProcess: + # -I -S: skip site-packages entirely (plain -I still processes the venv's .pth + # files), so nothing pip-installed is importable — only stdlib plus `path_entry`. + interp = [sys.executable, "-I", "-S"] if no_site else [sys.executable] + script = f"import sys\nsys.path.insert(0, {path_entry!r})\n" + prelude + textwrap.dedent(body) + return subprocess.run([*interp, "-c", script], capture_output=True, text=True) + + +def test_bare_import_weaviate_installs_shim_under_emscripten(): + result = _run( + prelude=_PRIME_SYSCONFIG + _FAKE_PYODIDE, + body=""" + sys.platform = "emscripten" + + import weaviate # the ONLY weaviate-side import: must bootstrap the companion + + assert "weaviate_client_web" in sys.modules, "hook did not import the companion" + import weaviate_client_web + assert weaviate_client_web.is_installed() + assert weaviate_client_web.is_fetch_transport_installed() + import grpc + assert getattr(grpc, "__weaviate_client_web_shim__", False) is True + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_without_companion_raises_clear_import_error(): + # No site-packages, so neither weaviate_client_web nor grpcio is importable; the repo + # root goes on sys.path so the weaviate package itself is still found. + result = _run( + """ + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert "weaviate-client-web" in str(e), str(e) + assert "WebAssembly/Pyodide" in str(e), str(e) + print("OK") + else: + raise AssertionError("expected ImportError without the companion") + """, + path_entry=_REPO_ROOT, + no_site=True, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_grpc_present_falls_through_silently(): + # Companion blocked but a real grpc IS importable (grpcio in the dev env): the hook + # must fall through and leave the normal import path untouched. + result = _run( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + sys.modules["weaviate_client_web"] = None # makes its import raise ImportError + + import weaviate + import grpc + + assert not getattr(grpc, "__weaviate_client_web_shim__", False) + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_broken_companion_surfaces_its_own_error(tmp_path): + # An INSTALLED companion whose import fails (here: a missing dependency of its own) + # must raise that error, not the "install weaviate-client-web" hint — the hint would + # send the user to reinstall a package that is already there. + fake_pkg = tmp_path / "weaviate_client_web" + fake_pkg.mkdir() + (fake_pkg / "__init__.py").write_text( + "raise ModuleNotFoundError(\"No module named 'anyio'\", name='anyio')\n" + ) + result = _run( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert e.name == "anyio", (e.name, str(e)) + assert "anyio" in str(e), str(e) + assert "weaviate-client-web" not in str(e), str(e) + print("OK") + else: + raise AssertionError("expected the companion's own ImportError to surface") + """, + path_entry=str(tmp_path), + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py new file mode 100644 index 000000000..70777f10c --- /dev/null +++ b/packages/web/tests/test_transport.py @@ -0,0 +1,621 @@ +"""In-process tests for the grpc-web channel/multicallable. + +These exercise the transport classes directly (they import their grpc base classes from +``weaviate_client_web._shim``, not from ``sys.modules['grpc']``), so no shim install is +needed and the real ``grpc`` in the dev environment is left untouched. +""" + +import asyncio +import struct +import sys +from typing import Dict, List, Optional, Tuple + +import pytest + +from weaviate_client_web._channel import ( + GrpcWebChannel, + _body_excerpt, + _encode_timeout, + set_sender, +) +from weaviate_client_web._shim import AioChannel, AioRpcError, StatusCode + + +def _frame(payload: bytes, flag: int = 0x00) -> bytes: + return struct.pack(">BI", flag, len(payload)) + payload + + +def _ok_response(payload: bytes) -> bytes: + return _frame(payload) + _frame(b"grpc-status:0\r\n", 0x80) + + +class FakeSender: + def __init__( + self, status: int = 200, headers: Optional[Dict[str, str]] = None, body: bytes = b"" + ): + self.status = status + self.headers = headers or {} + self.body = body + self.calls: List[Tuple[str, Dict[str, str], bytes, Optional[float]]] = [] + + async def __call__(self, url, headers, body, timeout): + self.calls.append((url, headers, body, timeout)) + return self.status, self.headers, self.body + + +def _channel(sender: FakeSender, secure: bool = False) -> GrpcWebChannel: + return GrpcWebChannel("example.com:443", secure=secure, sender=sender) + + +def test_grpcwebchannel_is_grpc_aio_channel(): + assert issubclass(GrpcWebChannel, AioChannel) + assert isinstance(_channel(FakeSender()), AioChannel) + + +def test_unary_success_round_trip(): + sender = FakeSender(body=_ok_response(b"reply-bytes")) + channel = _channel(sender) + mc = channel.unary_unary( + "/weaviate.v1.Weaviate/Search", + request_serializer=lambda x: x, + response_deserializer=lambda b: b, + _registered_method=True, + ) + + result = asyncio.run(mc(b"request-bytes", metadata=[("authorization", "Bearer k")], timeout=5)) + + assert result == b"reply-bytes" + url, headers, body, timeout = sender.calls[0] + assert url == "http://example.com:443/weaviate.v1.Weaviate/Search" + assert body == _frame(b"request-bytes") + assert headers["content-type"] == "application/grpc-web+proto" + assert headers["authorization"] == "Bearer k" + assert headers["grpc-timeout"] == "5000m" + assert timeout == 5 + + +def test_secure_channel_uses_https(): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender, secure=True) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + asyncio.run(mc(b"q")) + assert sender.calls[0][0].startswith("https://example.com:443/") + + +def test_health_call_without_metadata(): + sender = FakeSender(body=_ok_response(b"pong")) + channel = _channel(sender) + mc = channel.unary_unary("/grpc.health.v1.Health/Check", lambda x: x, lambda b: b) + # mirrors the health check in connect/v4.py — request + timeout, no metadata + assert asyncio.run(mc(b"ping", timeout=2)) == b"pong" + + +def test_error_trailer_raises_aiorpcerror(): + body = _frame(b"grpc-status:7\r\ngrpc-message:nope\r\n", 0x80) + channel = _channel(FakeSender(body=body)) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.PERMISSION_DENIED + assert excinfo.value.code().name == "PERMISSION_DENIED" + assert excinfo.value.details() == "nope" + + +def test_percent_encoded_grpc_message_decoded(): + body = _frame(b"grpc-status:5\r\ngrpc-message:not%20found\r\n", 0x80) + channel = _channel(FakeSender(body=body)) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.details() == "not found" + + +def test_trailers_only_status_in_http_headers(): + channel = _channel( + FakeSender(status=200, headers={"grpc-status": "16", "grpc-message": "auth"}, body=b"") + ) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.UNAUTHENTICATED + + +# --- non-grpc-web responses ------------------------------------------------------- +# +# Every real error response carries a body, and none of them is grpc-web framing. The +# bodies below are verbatim shapes seen in the wild (an empty error body is the one +# shape no server or proxy produces). + +# Weaviate's own 404, verbatim from a 1.39.0 server asked for the wrong prefix. +WEAVIATE_404_JSON = ( + b'{"code":404,"message":"path /grpc-web/grpc.health.v1.Health/Check was not found"}' +) +NGINX_502_HTML = ( + b"\r\n502 Bad Gateway\r\n\r\n" + b"

502 Bad Gateway

\r\n
nginx/1.27.3
\r\n" + b"\r\n\r\n" +) +NGINX_404_HTML = ( + b"\r\n404 Not Found\r\n\r\n" + b"

404 Not Found

\r\n
nginx/1.27.3
\r\n" + b"\r\n\r\n" +) +# A single-page app's catch-all route answers 200 with index.html for unknown paths. +SPA_INDEX_HTML = ( + b'\n\n \n My App\n' + b' \n' + b' \n
\n\n' +) + + +def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/Check"): + """Run one request against a canned HTTP response and return the AioRpcError.""" + channel = _channel(FakeSender(status=status, headers=headers or {}, body=body)) + mc = channel.unary_unary(path, lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + return excinfo.value + + +def test_weaviate_404_json_names_both_candidate_causes(): + # A 404 means EITHER the server predates the native /v1/grpc-web endpoint OR the + # configured path prefix is wrong. The channel cannot tell which, so it must say both. + err = _details_of(404, WEAVIATE_404_JSON, {"content-type": "application/json"}) + details = err.details() + + assert err.code() is StatusCode.UNIMPLEMENTED + assert details.startswith("HTTP 404 ") + assert "/grpc.health.v1.Health/Check" in details # the request path + assert "1.38.3" in details # candidate 1: server too old + assert "path prefix" in details # candidate 2: wrong prefix + assert "/v1/grpc-web" in details # the native prefix, spelled out + assert "was not found" in details # the server's own explanation + assert "malformed grpc-web response" not in details + + +def test_nginx_502_maps_to_unavailable_so_the_client_retries(): + # weaviate/retry.py retries UNAVAILABLE and nothing else; a gateway error arriving + # as INTERNAL is silently un-retried, which is the regression this pins. + err = _details_of(502, NGINX_502_HTML) + assert err.code() is StatusCode.UNAVAILABLE + assert err.details().startswith("HTTP 502 ") + assert "502 Bad Gateway" in err.details() + + +@pytest.mark.parametrize("status", [503, 504]) +def test_gateway_errors_are_unavailable(status): + err = _details_of(status, b"upstream down") + assert err.code() is StatusCode.UNAVAILABLE + + +def test_nginx_404_html_is_reported_as_an_http_404(): + err = _details_of(404, NGINX_404_HTML) + assert err.code() is StatusCode.UNIMPLEMENTED + assert err.details().startswith("HTTP 404 ") + assert "404 Not Found" in err.details() + assert "malformed grpc-web response" not in err.details() + + +def test_405_names_the_wrong_prefix(): + # a 405 can only come from an existing HTTP route (measured live: a prefix pointing + # at /v1/objects answers "method POST is not allowed"), so the prefix is wrong + err = _details_of(405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}') + assert err.details().startswith("HTTP 405 ") + assert "path prefix" in err.details() + assert "/v1/grpc-web" in err.details() + assert "method POST is not allowed" in err.details() + + +def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): + # a valid frame header whose payload was cut short: the endpoint IS grpc-web, so the + # SPA / path-prefix hint would send the user the wrong way + body = _ok_response(b"reply-bytes")[:-6] + err = _details_of(200, body) + assert err.code() is StatusCode.INTERNAL + assert "truncated" in err.details() + assert "cut short" in err.details() + assert "single-page-app" not in err.details() + assert "path prefix" not in err.details() + + +def test_spa_html_body_is_not_reported_as_truncated(): + # text bodies decode to an unknown flag byte and a garbage length; they must read as + # "not grpc-web framing", never as a truncated grpc-web body + err = _details_of(200, SPA_INDEX_HTML) + assert "truncated" not in err.details() + assert "not grpc-web framing" in err.details() + + +def test_message_frame_after_trailer_is_internal(): + body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") + err = _details_of(200, body) + assert err.code() is StatusCode.INTERNAL + assert "malformed grpc-web response" in err.details() + assert "after the trailer" in err.details() + assert "single-page-app" not in err.details() + + +def test_multiple_message_frames_in_unary_response_is_internal(): + # a unary RPC has exactly one message; silently taking the first would hide a proxy + # or server that streams several + body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:0\r\n", 0x80) + err = _details_of(200, body) + assert err.code() is StatusCode.INTERNAL + assert "2 message frames" in err.details() + + +def test_error_status_wins_over_multiple_message_frames(): + body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:5\r\ngrpc-message:gone\r\n", 0x80) + err = _details_of(200, body) + assert err.code() is StatusCode.NOT_FOUND + assert err.details() == "gone" + + +def test_spa_fallback_html_200_is_distinguishable_from_a_404(): + # An HTTP 200 serving index.html is the other half of a wrong path prefix: the app's + # catch-all route answers instead of Weaviate. It must not read as malformed framing. + err = _details_of(200, SPA_INDEX_HTML) + details = err.details() + + assert details.startswith("HTTP 200 ") + assert "" in details + assert "single-page-app" in details # names the actual cause + assert "malformed grpc-web response" not in details + # distinguishable from the 404 case, not the same generic message + assert details != _details_of(404, NGINX_404_HTML).details() + + +def test_401_json_body_maps_to_unauthenticated(): + err = _details_of(401, b'{"error":[{"message":"anonymous access not enabled"}]}') + assert err.code() is StatusCode.UNAUTHENTICATED + assert err.details().startswith("HTTP 401 ") + assert "anonymous access not enabled" in err.details() + + +def test_403_error_body_reaches_details(): + # regression: the response body is the most actionable part of the error and must + # survive into details() rather than being parsed as frames and discarded + err = _details_of(403, b'{"code":403,"message":"forbidden: rbac denied"}') + assert err.code() is StatusCode.PERMISSION_DENIED + assert "forbidden: rbac denied" in err.details() + + +def test_error_body_excerpt_is_capped(): + err = _details_of(500, b"E" * 5000) + details = err.details() + assert "EEEE" in details + assert details.endswith("...") + assert len(details) < 600 # the 5000-byte body is excerpted, not pasted in + + +def test_binary_error_body_does_not_break_the_error(): + # a proxy answering with a binary payload must not raise UnicodeDecodeError while + # the error message is being built + err = _details_of(502, b"\xff\xfe\x00\x01\x02") + assert err.code() is StatusCode.UNAVAILABLE + assert err.details().startswith("HTTP 502 ") + + +def test_non_200_with_valid_grpc_web_trailers_still_uses_grpc_status(): + # guard on the fix's shape: the HTTP status must not shadow a real grpc-status that + # a proxy shipped alongside a non-200 + err = _details_of(500, _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80)) + assert err.code() is StatusCode.PERMISSION_DENIED + assert err.details() == "denied" + + +def test_non_ascii_grpc_message_preserves_the_status(): + # a trailer carrying raw UTF-8 (an un-percent-encoded proxy, or an error quoting a + # collection name) must not degrade to INTERNAL and lose grpc-status + body = _frame("grpc-status:5\r\ngrpc-message:collection Café not found\r\n".encode(), 0x80) + err = _details_of(200, body) + assert err.code() is StatusCode.NOT_FOUND + assert "Caf" in err.details() + + +def test_invalid_utf8_grpc_message_preserves_the_status(): + # latin-1 bytes are not valid UTF-8; the status must still survive + body = _frame(b"grpc-status:9\r\ngrpc-message:tenant caf\xe9 is COLD\r\n", 0x80) + err = _details_of(200, body) + assert err.code() is StatusCode.FAILED_PRECONDITION + assert "tenant caf" in err.details() + + +def test_binary_metadata_base64_encoded(): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + asyncio.run(mc(b"q", metadata=[("trace-bin", b"\x00\x01\x02")])) + assert sender.calls[0][1]["trace-bin"] == "AAEC" + + +def test_stream_stream_raises_clear_error(): + channel = _channel(FakeSender()) + mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) + with pytest.raises(RuntimeError) as excinfo: + mc(request_iterator=iter([]), timeout=5, metadata=None) + assert "not supported over grpc-web" in str(excinfo.value) + + +def test_timeout_maps_to_deadline_exceeded(): + async def slow_sender(url, headers, body, timeout): + await asyncio.sleep(0.5) + return 200, {}, _ok_response(b"x") + + channel = GrpcWebChannel("h:1", secure=False, sender=slow_sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q", timeout=0.01)) + assert excinfo.value.code() is StatusCode.DEADLINE_EXCEEDED + + +def test_transport_exception_maps_to_unavailable(): + async def boom(url, headers, body, timeout): + raise ConnectionError("connection refused") + + channel = GrpcWebChannel("h:1", secure=False, sender=boom) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.UNAVAILABLE + assert "ConnectionError: connection refused" in str(excinfo.value.details()) + + +def test_transport_exception_with_empty_str_keeps_type(): + # httpx transport errors commonly stringify to '' — the detail must still name them + async def boom(url, headers, body, timeout): + raise ConnectionError() + + channel = GrpcWebChannel("h:1", secure=False, sender=boom) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert "ConnectionError" in str(excinfo.value.details()) + + +def test_empty_ok_response_hints_at_cors_expose_headers(): + # HTTP 200, empty body, no grpc-status anywhere: the shape of a trailers-only error + # whose grpc-status/grpc-message headers were stripped by CORS + channel = _channel(FakeSender(status=200, headers={}, body=b"")) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.INTERNAL + assert "Access-Control-Expose-Headers" in str(excinfo.value.details()) + + +def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): + # when grpc-status WAS visible (status 0, no frames), it is a malformed response, + # not a CORS problem — the hint must not appear + channel = _channel(FakeSender(status=200, headers={"grpc-status": "0"}, body=b"")) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.INTERNAL + assert "Access-Control-Expose-Headers" not in str(excinfo.value.details()) + + +def test_message_frame_without_grpc_status_is_internal_not_success(): + # HTTP 200 with a valid message frame but no grpc-status anywhere (e.g. a proxy + # dropped the trailer frame) must be an error, never a fabricated success + channel = _channel(FakeSender(status=200, headers={}, body=_frame(b"reply-bytes"))) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.INTERNAL + assert "missing grpc-status" in str(excinfo.value.details()) + + +def test_message_frame_with_grpc_status_header_still_succeeds(): + # trailers-only-in-headers responses (grpc-status as an HTTP header, no trailer + # frame) remain valid per the grpc-web contract + channel = _channel( + FakeSender(status=200, headers={"grpc-status": "0"}, body=_frame(b"reply-bytes")) + ) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + assert asyncio.run(mc(b"q")) == b"reply-bytes" + + +def test_stream_stream_error_recommends_insert_many_only(): + # batch.dynamic()/fixed_size()/rate_limit() do not exist on the async client (the + # only one supported under WASM), so the error must not recommend them + channel = _channel(FakeSender()) + mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) + with pytest.raises(RuntimeError) as excinfo: + mc(request_iterator=iter([]), timeout=5, metadata=None) + assert "insert_many" in str(excinfo.value) + for sync_only in ("dynamic", "fixed_size", "rate_limit"): + assert sync_only not in str(excinfo.value) + + +def test_malformed_frame_maps_to_internal(): + # A 3-byte body cannot contain even a 5-byte frame header -> framing ValueError. + channel = _channel(FakeSender(body=b"\x00\x00\x00")) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.INTERNAL + + +def test_malformed_grpc_status_maps_to_internal(): + body = _frame(b"grpc-status:notanint\r\n", 0x80) + channel = _channel(FakeSender(body=body)) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.INTERNAL + + +def test_grpc_timeout_header_rounds_up(): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + # 123.4ms must round UP to 124ms (never advertise a shorter deadline than requested). + asyncio.run(mc(b"q", timeout=0.1234)) + assert sender.calls[0][1]["grpc-timeout"] == "124m" + + +def test_body_excerpt_empty_and_non_printable(): + assert _body_excerpt(b"") == "" + assert _body_excerpt(b"\x00\x01\x02\x7f") == "<4 non-printable bytes>" + assert _body_excerpt(b"ok\x00\x01") == "ok" + + +@pytest.mark.parametrize( + "seconds,expected", + [ + (None, None), + (float("inf"), None), + (float("nan"), None), + (0, "1m"), + (0.1234, "124m"), + (5, "5000m"), + (99_999, "99999000m"), + (100_000, "100000S"), # 1e8 ms would be 9 digits + (1e8, "1666667M"), # 1e8 s would be 9 digits + (1e9, "16666667M"), + (5_999_999_940, "99999999M"), # the largest deadline that still fits in minutes + (1e10, None), # would need hours, which transcoders reject above 8H: no deadline + (1e15, None), + ], +) +def test_encode_timeout_stays_within_eight_digits(seconds, expected): + encoded = _encode_timeout(seconds) + assert encoded == expected + if encoded is not None: + assert len(encoded) <= 9 # 8 digits + unit + assert not encoded.endswith("H") + + +def test_infinite_timeout_sends_no_deadline(): + # Timeout(query=inf): neither a grpc-timeout header nor a client-side wait + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + assert asyncio.run(mc(b"q", timeout=float("inf"))) == b"x" + _, headers, _, timeout = sender.calls[0] + assert "grpc-timeout" not in headers + assert timeout is None + + +def test_huge_timeout_uses_minutes_then_no_deadline(): + # 1e8 s in milliseconds is 12 digits; the server rejects more than 8 ("timeout is + # too long", HTTP 400) and transcoders reject hour values above 8H, so past the + # minute range the request carries no deadline at all + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + asyncio.run(mc(b"q", timeout=1e8)) + asyncio.run(mc(b"q", timeout=1e9)) + asyncio.run(mc(b"q", timeout=1e10)) + assert sender.calls[0][1]["grpc-timeout"] == "1666667M" + assert sender.calls[1][1]["grpc-timeout"] == "16666667M" + assert "grpc-timeout" not in sender.calls[2][1] + assert sender.calls[2][3] is None # no client-side wait either + + +@pytest.mark.parametrize("bad", ["val\r\nx-injected: evil", "val\nx", "v\0"]) +def test_crlf_in_metadata_rejected(bad): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(ValueError, match="Illegal character"): + asyncio.run(mc(b"q", metadata=[("x-key", bad)])) + with pytest.raises(ValueError, match="Illegal character"): + asyncio.run(mc(b"q", metadata=[("x-key\r\n", "v")])) + assert sender.calls == [] + + +def _unavailable_details(monkeypatch, path_prefix, platform): + async def boom(url, headers, body, timeout): + raise ConnectionError("Failed to fetch") + + monkeypatch.setattr(sys, "platform", platform) + channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.UNAVAILABLE + return excinfo.value.details() + + +def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix(monkeypatch): + # the connect helpers always set the prefix under Emscripten, so a prefix-less channel + # here means hand-built ConnectionParams; the error must say what to do instead + details = _unavailable_details(monkeypatch, path_prefix="", platform="emscripten") + assert "grpc_path_prefix='/v1/grpc-web'" in details + assert "1.38.3" in details + assert "connect helpers" in details + + +def test_unavailable_with_path_prefix_has_no_prefix_hint(monkeypatch): + details = _unavailable_details(monkeypatch, path_prefix="/v1/grpc-web", platform="emscripten") + assert "no grpc_path_prefix" not in details + + +def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(monkeypatch): + # on CPython an empty prefix against a transcoder is the normal configuration + details = _unavailable_details(monkeypatch, path_prefix="", platform="linux") + assert "no grpc_path_prefix" not in details + + +def test_close_is_awaitable_noop(): + channel = _channel(FakeSender()) + assert asyncio.run(channel.close()) is None + + +def test_path_prefix_prepended_to_url(): + sender = FakeSender(body=_ok_response(b"r")) + channel = GrpcWebChannel( + "example.com:8090", secure=False, sender=sender, path_prefix="/grpc-web" + ) + mc = channel.unary_unary("/weaviate.v1.Weaviate/Search", lambda x: x, lambda b: b) + asyncio.run(mc(b"q")) + assert sender.calls[0][0] == "http://example.com:8090/grpc-web/weaviate.v1.Weaviate/Search" + + +@pytest.mark.parametrize( + "raw,expected_url", + [ + ("grpc-web", "http://h:1/grpc-web/svc/M"), + ("/grpc-web/", "http://h:1/grpc-web/svc/M"), + ("/a/b", "http://h:1/a/b/svc/M"), + ("", "http://h:1/svc/M"), + ], +) +def test_path_prefix_normalized_in_url(raw, expected_url): + sender = FakeSender(body=_ok_response(b"r")) + channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + asyncio.run(mc(b"q")) + assert sender.calls[0][0] == expected_url + + +def test_shim_factory_extracts_path_prefix_option(): + from weaviate_client_web._shim import _aio_insecure_channel + + with_prefix = _aio_insecure_channel( + target="h:1", + options=[("grpc.max_send_message_length", 1), ("grpc-web.path_prefix", "/grpc-web")], + ) + assert with_prefix._path_prefix == "/grpc-web" + + without_prefix = _aio_insecure_channel( + target="h:1", options=[("grpc.max_send_message_length", 1)] + ) + assert without_prefix._path_prefix == "" + + +def test_set_sender_overrides_default(): + sender = FakeSender(body=_ok_response(b"y")) + set_sender(sender) + try: + channel = GrpcWebChannel("h:1", secure=False) # no explicit sender + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + assert asyncio.run(mc(b"q")) == b"y" + finally: + # restore the real default so other tests/processes are unaffected + from weaviate_client_web._sender import pyfetch_sender + + set_sender(pyfetch_sender) diff --git a/proto_test/test_proto.py b/proto_test/test_proto.py index bedbf10c3..85c173d46 100644 --- a/proto_test/test_proto.py +++ b/proto_test/test_proto.py @@ -1,19 +1,123 @@ +import importlib +import pathlib +import re +from importlib.metadata import PackageNotFoundError, version as metadata_version + import pytest -from importlib.metadata import version as metadata_version from packaging import version +# The CI matrix deliberately installs incompatible grpcio/protobuf pairs to exercise the +# version gate in weaviate/proto/v1/__init__.py. In those cells the package raises on +# import (covered by test_proto_import), so the get_version tests below are skipped; they +# still run in every compatible cell. This check imports nothing from weaviate, so the +# test module always loads. +def _versions_incompatible() -> bool: + """Whether the installed grpcio/protobuf pair makes ``import weaviate.proto.v1`` raise.""" + try: + grpc_ver = version.parse(metadata_version("grpcio")) + pb_ver = version.parse(metadata_version("protobuf")) + except PackageNotFoundError: + return False + return (pb_ver >= version.parse("6.30.0") and grpc_ver < version.parse("1.72.0")) or ( + pb_ver >= version.parse("5.26.1") and grpc_ver < version.parse("1.63.0") + ) + + +_skip_if_incompatible = pytest.mark.skipif( + _versions_incompatible(), + reason="weaviate.proto.v1 cannot be imported with an incompatible grpcio/protobuf " + "pair (CI version-gate matrix); the gate is covered by test_proto_import and the " + "fallback is exercised in every compatible cell", +) + + def test_proto_import(): grpc_ver = version.parse(metadata_version("grpcio")) pb_ver = version.parse(metadata_version("protobuf")) - if (pb_ver >= version.parse("6.30.0") and grpc_ver < version.parse("1.72.0")) or ( pb_ver >= version.parse("5.26.1") and grpc_ver < version.parse("1.63.0") ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception) as e: import weaviate - assert "gRPC incompatibility detected" in str(exc_info.value) + + assert weaviate.version is not None + assert "WeaviateProtobufIncompatibility" in str(e.type) else: import weaviate assert weaviate.version is not None + + +@_skip_if_incompatible +def test_grpcio_metadata_fallback_under_emscripten(monkeypatch): + """Fall back for grpcio when its metadata is absent; protobuf still surfaces. + + Under Pyodide/Emscripten grpcio is excluded via an environment marker, so its + distribution metadata is missing and ``get_version`` must fall back to a working + proto variant; a genuinely missing protobuf is still surfaced, not masked. + """ + mod = importlib.import_module("weaviate.proto.v1") + + def raises(pkg: str) -> str: + raise PackageNotFoundError(pkg) + + monkeypatch.setattr(mod, "metadata_version", raises) + monkeypatch.setattr("sys.platform", "emscripten") + + assert str(mod.get_version("grpcio")) == "1.72.1" + with pytest.raises(PackageNotFoundError): + mod.get_version("protobuf") + + +@_skip_if_incompatible +def test_grpcio_missing_metadata_raises_off_emscripten(monkeypatch): + """Off Emscripten, missing grpcio metadata surfaces instead of being masked.""" + mod = importlib.import_module("weaviate.proto.v1") + + def raises(pkg: str) -> str: + raise PackageNotFoundError(pkg) + + monkeypatch.setattr(mod, "metadata_version", raises) + monkeypatch.setattr("sys.platform", "linux") + with pytest.raises(PackageNotFoundError): + mod.get_version("grpcio") + + +@_skip_if_incompatible +def test_grpcio_fallback_version_passes_every_vendored_stub_gate(): + """The Emscripten fallback version must satisfy every vendored stub's version gate. + + Under Pyodide ``get_version("grpcio")`` returns ``_GRPCIO_FALLBACK_VERSION`` and the + shim reports it as ``grpc.__version__``, so every vendored ``*_pb2_grpc.py`` whose + import-time gate (``first_version_is_lower``) rejects it would break at import. If + the protos are regenerated with a newer grpcio-tools, this fails until the fallback + (and the grpc-web shim's ``FAKE_GRPC_VERSION``) is bumped to match. + """ + try: + from grpc._utilities import first_version_is_lower + except ImportError: + pytest.skip( + "grpc._utilities.first_version_is_lower is unavailable in this grpcio; " + "newer matrix cells run the comparison" + ) + + fallback = importlib.import_module("weaviate.proto.v1")._GRPCIO_FALLBACK_VERSION + proto_root = pathlib.Path(__file__).resolve().parents[1] / "weaviate" / "proto" / "v1" + stub_files = sorted(proto_root.glob("*/v1/*_pb2_grpc.py")) + assert stub_files, "no vendored *_pb2_grpc.py stubs found" + + gate_pattern = re.compile(r"^GRPC_GENERATED_VERSION = '([^']+)'", re.MULTILINE) + gated = 0 + for stub in stub_files: + match = gate_pattern.search(stub.read_text()) + if match is None: + continue # older codegen (e.g. v4216) emits no version gate + gated += 1 + generated = match.group(1) + assert not first_version_is_lower(fallback, generated), ( + f"{stub.relative_to(proto_root)} requires grpcio>={generated} but " + f"_GRPCIO_FALLBACK_VERSION is {fallback}; bump the fallback (and the " + "grpc-web shim's FAKE_GRPC_VERSION) to match the regenerated stubs" + ) + assert gated > 0, "no stub carried a GRPC_GENERATED_VERSION gate; check the extraction regex" diff --git a/pyrightconfig.json b/pyrightconfig.json index 396d62cfd..61eb3eaa3 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,6 +1,6 @@ { "include": [ - "weaviate", "integration" + "weaviate", "integration", "packages/web/src" ], "exclude": [ diff --git a/setup.cfg b/setup.cfg index 0b5ba855a..7343116a5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,7 @@ install_requires = # When bumping authlib to >=2.0.0, remove the `authlib.jose` deprecation # warning filter implemented in `weaviate/_authlib_compat.py`. pydantic>=2.12.0,<3.0.0 - grpcio>=1.59.5,<1.80.0 + grpcio>=1.59.5,<1.80.0; sys_platform != "emscripten" protobuf>=4.21.6,<7.0.0 packaging>=21.0 python_requires = >=3.10 diff --git a/test/test_connection_params.py b/test/test_connection_params.py new file mode 100644 index 000000000..1a4df4c88 --- /dev/null +++ b/test/test_connection_params.py @@ -0,0 +1,189 @@ +import sys + +import pytest +from pydantic import ValidationError + +import weaviate.connect.base as base_mod +from weaviate.connect.base import ConnectionParams +from weaviate.exceptions import WeaviateInvalidInputError + + +def test_same_host_port_raises_without_prefix() -> None: + with pytest.raises(ValidationError, match="must be different"): + ConnectionParams.from_params( + http_host="localhost", + http_port=8090, + http_secure=False, + grpc_host="localhost", + grpc_port=8090, + grpc_secure=False, + ) + + +def test_same_host_port_allowed_with_grpc_web_prefix() -> None: + params = ConnectionParams.from_params( + http_host="localhost", + http_port=8090, + http_secure=False, + grpc_host="localhost", + grpc_port=8090, + grpc_secure=False, + grpc_path_prefix="/grpc-web", + ) + assert params._grpc_web_path_prefix == "/grpc-web" + + +def test_from_url_same_host_port_allowed_with_prefix() -> None: + params = ConnectionParams.from_url( + "http://localhost:8090", grpc_port=8090, grpc_path_prefix="/grpc-web" + ) + assert params._grpc_web_path_prefix == "/grpc-web" + + +@pytest.mark.parametrize( + "raw,expected", + [ + (None, ""), + ("", ""), + ("/", ""), + ("grpc-web", "/grpc-web"), + ("/grpc-web", "/grpc-web"), + ("grpc-web/", "/grpc-web"), + ("/a/b/", "/a/b"), + ], +) +def test_path_prefix_normalization(raw, expected) -> None: + params = ConnectionParams.from_params( + http_host="h", + http_port=8080, + http_secure=False, + grpc_host="g", + grpc_port=50051, + grpc_secure=False, + grpc_path_prefix=raw, + ) + assert params._grpc_web_path_prefix == expected + + +def _grpc_web_params() -> ConnectionParams: + return ConnectionParams.from_params( + http_host="localhost", + http_port=8090, + http_secure=False, + grpc_host="localhost", + grpc_port=8090, + grpc_secure=False, + grpc_path_prefix="/grpc-web", + ) + + +def test_grpc_channel_forwards_path_prefix_option(monkeypatch) -> None: + captured: dict = {} + + def fake_insecure_channel(target, options=None, **kwargs): + captured["target"] = target + captured["options"] = options + return "CHANNEL" + + monkeypatch.setattr(base_mod.grpc.aio, "insecure_channel", fake_insecure_channel) + + channel = _grpc_web_params()._grpc_channel(proxies={}, grpc_msg_size=None, is_async=True) + + assert channel == "CHANNEL" + assert captured["target"] == "localhost:8090" + assert ("grpc-web.path_prefix", "/grpc-web") in captured["options"] + + +def test_grpc_channel_omits_option_without_prefix(monkeypatch) -> None: + captured: dict = {} + + def fake_insecure_channel(target, options=None, **kwargs): + captured["options"] = options + return "CHANNEL" + + monkeypatch.setattr(base_mod.grpc.aio, "insecure_channel", fake_insecure_channel) + + params = ConnectionParams.from_params( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=50051, + grpc_secure=False, + ) + params._grpc_channel(proxies={}, grpc_msg_size=None, is_async=True) + + option_keys = [key for key, _ in captured["options"]] + assert "grpc-web.path_prefix" not in option_keys + + +def test_async_client_construction_rejects_prefix_without_shim(monkeypatch) -> None: + # fail at construction with actionable text, not deep inside connect() after the + # OIDC and /v1/meta round trips already succeeded + from weaviate import WeaviateAsyncClient + + monkeypatch.delattr(base_mod.grpc, "__weaviate_client_web_shim__", raising=False) + with pytest.raises(WeaviateInvalidInputError, match="weaviate-client-web"): + WeaviateAsyncClient(_grpc_web_params()) + + +def test_async_client_construction_allows_prefix_with_shim(monkeypatch) -> None: + from weaviate import WeaviateAsyncClient + + monkeypatch.setattr(base_mod.grpc, "__weaviate_client_web_shim__", True, raising=False) + client = WeaviateAsyncClient(_grpc_web_params()) + assert client._connection._connection_params._grpc_web_path_prefix == "/grpc-web" + + +def test_sync_client_construction_rejects_grpc_web_prefix() -> None: + from weaviate import WeaviateClient + + with pytest.raises(WeaviateInvalidInputError, match="async"): + WeaviateClient(_grpc_web_params()) + + +@pytest.mark.parametrize( + "call,expected", + [ + ( + lambda w: w.use_async_with_local(), + { + "http": {"host": "localhost", "port": 8080, "secure": False}, + "grpc": {"host": "localhost", "port": 50051, "secure": False}, + "grpc_path_prefix": None, + }, + ), + ( + lambda w: w.use_async_with_weaviate_cloud("abc.something.weaviate.cloud", None), + { + "http": {"host": "abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc": {"host": "grpc-abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc_path_prefix": None, + }, + ), + ( + lambda w: w.use_async_with_custom( + http_host="rest.example.com", + http_port=443, + http_secure=True, + grpc_host="grpc.example.com", + grpc_port=443, + grpc_secure=True, + ), + { + "http": {"host": "rest.example.com", "port": 443, "secure": True}, + "grpc": {"host": "grpc.example.com", "port": 443, "secure": True}, + "grpc_path_prefix": None, + }, + ), + ], +) +def test_helper_params_off_emscripten_are_unchanged(call, expected) -> None: + # the no-regression pin: off Emscripten the helpers build exactly the params they + # always did; grpc_path_prefix is new and stays None (native gRPC) + import weaviate + + assert sys.platform != "emscripten" + params = call(weaviate)._connection._connection_params + assert params.model_dump() == expected + assert params._grpc_web_path_prefix == "" diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py new file mode 100644 index 000000000..455776078 --- /dev/null +++ b/test/test_wasm_compat.py @@ -0,0 +1,254 @@ +"""Unit tests for WASM/Pyodide-compatibility behaviour that runs on CPython too. + +Under Emscripten there are no subprocesses, no threads and no sockets — these tests pin +the guards, the grpc-web auto-routing and the grpc-web diagnostics added for that +environment without needing a browser. +""" + +import asyncio +import sys + +import grpc +import pytest +from grpc.aio import AioRpcError, Metadata + +from weaviate import WeaviateClient +from weaviate.collections.batch.async_ import _BatchBaseAsync +from weaviate.connect.base import ConnectionParams +from weaviate.connect.v4 import _ConnectionBase +from weaviate.embedded import _EmbeddedBase +from weaviate.exceptions import ( + WeaviateBatchStreamError, + WeaviateGRPCUnavailableError, + WeaviateStartUpError, +) +from weaviate.util import _ServerVersion + + +def test_embedded_raises_explicit_error_under_emscripten(monkeypatch) -> None: + # without the guard, the Emscripten socket emulation makes the port probe + # "succeed" and embedded misreports that Weaviate is already listening + monkeypatch.setattr(sys, "platform", "emscripten") + with pytest.raises(WeaviateStartUpError, match="WebAssembly/Pyodide"): + _EmbeddedBase.check_supported_platform() + + +def test_sync_client_construction_raises_async_only_under_emscripten(monkeypatch) -> None: + # without the guard the sync client constructs fine and the first REST call fails + # with an opaque ConnectError; the clear async-only error must win, at construction + monkeypatch.setattr(sys, "platform", "emscripten") + with pytest.raises(WeaviateStartUpError, match="async client"): + WeaviateClient(connection_params=ConnectionParams.from_url("http://localhost:8080", 50051)) + + +def test_batch_stream_fails_fast_when_grpc_web_shim_active(monkeypatch) -> None: + # over grpc-web the BatchStream RPC would die inside the background tasks (silent + # drop / endless flush); _start must raise before any task is created + monkeypatch.setattr(grpc, "__weaviate_client_web_shim__", True, raising=False) + batch = object.__new__(_BatchBaseAsync) # the guard runs before any attribute access + with pytest.raises(WeaviateBatchStreamError, match="insert_many"): + asyncio.run(batch._start()) + + +# --- grpc-web diagnostics ------------------------------------------------------------- + + +def _connection(prefix=None) -> _ConnectionBase: + conn = object.__new__(_ConnectionBase) + conn._client = None + conn._grpc_channel = None + conn._weaviate_version = _ServerVersion.from_string("1.36.0") + conn._connection_params = ConnectionParams.from_url( + "http://localhost:8080", + grpc_port=8080 if prefix else 50051, + grpc_path_prefix=prefix, + ) + return conn + + +def _ping_exception(conn: _ConnectionBase, error: Exception) -> None: + getattr(conn, "_ConnectionBase__handle_ping_exception")(error) # noqa: B009 + + +def test_grpc_web_404_names_the_two_real_causes_and_drops_firewall_advice() -> None: + # over grpc-web there is no separate gRPC port and no firewall: REST just succeeded + # against this very host:port. A 404 means the path was not routed. + conn = _connection(prefix="/grpc-web") + error = AioRpcError( + grpc.StatusCode.UNIMPLEMENTED, + Metadata(), + Metadata(), + details="HTTP 404 for /grpc-web/grpc.health.v1.Health/Check: 404 page not found", + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + assert "firewall" not in msg + assert "port (localhost:8080) are correct" not in msg + assert "UNIMPLEMENTED" in msg # the real code, not swallowed + assert "HTTP 404 for /grpc-web/grpc.health.v1.Health/Check" in msg # ... and details + assert "/grpc-web" in msg # the prefix that was actually used + assert "1.38.3" in msg # candidate 1: server too old ... + assert "v1.36.0" in msg # ... shown against the observed server version + assert "/v1/grpc-web" in msg # candidate 2: wrong prefix + + +def test_grpc_web_non_404_error_still_omits_the_native_port_advice() -> None: + conn = _connection(prefix="/grpc-web") + error = AioRpcError( + grpc.StatusCode.UNAVAILABLE, Metadata(), Metadata(), details="HTTP 502 for /grpc-web/..." + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + assert "firewall" not in msg + assert "UNAVAILABLE" in msg + assert "HTTP 502" in msg + assert "skip_init_checks=True" in msg # the still-useful advice is kept + + +def test_native_grpc_message_keeps_its_advice_and_gains_the_real_status() -> None: + conn = _connection() + error = AioRpcError( + grpc.StatusCode.UNAVAILABLE, Metadata(), Metadata(), details="failed to connect" + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + # unchanged guidance for native gRPC ... + assert "The gRPC traffic at the specified port is blocked by a firewall." in msg + assert "Please check that the server address and port (localhost:50051) are correct." in msg + # ... plus the error that was previously discarded + assert "UNAVAILABLE" in msg + assert "failed to connect" in msg + + +def test_non_grpc_ping_error_is_still_reported() -> None: + # not every ping failure is an RpcError; those must not lose the generic advice + conn = _connection() + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, ValueError("boom")) + assert "blocked by a firewall" in str(excinfo.value) + + +# --- grpc-web auto-routing under Emscripten ------------------------------------------- +# +# Native gRPC is impossible under WASM (no sockets, no grpcio wheel), so the async connect +# helpers pin gRPC to the REST endpoint under Weaviate's own grpc-web base path — the same +# contract as the TypeScript @weaviate/web client's webify(). Nothing selects it. + +GRPC_WEB_PREFIX = "/v1/grpc-web" + + +@pytest.fixture +def emscripten(monkeypatch): + """Fake Emscripten, with the grpc-web shim marked active. + + Under real Pyodide ``import weaviate`` installs the shim itself; here only the + routing decision is under test, not the environment check that guards it. + """ + import weaviate.connect.base as base_mod + + monkeypatch.setattr(sys, "platform", "emscripten") + monkeypatch.setattr(base_mod.grpc, "__weaviate_client_web_shim__", True, raising=False) + + +def _params(client) -> ConnectionParams: + return client._connection._connection_params + + +def _assert_grpc_rides_rest(client) -> None: + params = _params(client) + assert params.grpc.model_dump() == params.http.model_dump() + assert params._grpc_web_path_prefix == GRPC_WEB_PREFIX + assert params._grpc_target == f"{params.http.host}:{params.http.port}" + + +def test_use_async_with_local_routes_grpc_to_rest_under_emscripten(emscripten) -> None: + import weaviate + + _assert_grpc_rides_rest(weaviate.use_async_with_local(host="localhost", port=8290)) + assert _params(weaviate.use_async_with_local()).model_dump() == { + "http": {"host": "localhost", "port": 8080, "secure": False}, + "grpc": {"host": "localhost", "port": 8080, "secure": False}, + "grpc_path_prefix": GRPC_WEB_PREFIX, + } + + +def test_use_async_with_weaviate_cloud_routes_grpc_to_the_cluster_host(emscripten) -> None: + # WCD serves grpc-web on the cluster's own REST endpoint, not on grpc- + import weaviate + + client = weaviate.use_async_with_weaviate_cloud("abc.something.weaviate.cloud", None) + _assert_grpc_rides_rest(client) + assert _params(client).model_dump() == { + "http": {"host": "abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc": {"host": "abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc_path_prefix": GRPC_WEB_PREFIX, + } + + +def test_use_async_with_custom_routes_grpc_to_rest_under_emscripten(emscripten) -> None: + import weaviate + + _assert_grpc_rides_rest( + weaviate.use_async_with_custom( + http_host="wv.example.com", + http_port=443, + http_secure=True, + grpc_host="wv.example.com", + grpc_port=443, + grpc_secure=True, + ) + ) + + +def test_matching_grpc_arguments_are_not_warned_about(emscripten, recwarn) -> None: + # the documented WASM shape: gRPC arguments equal to the HTTP ones. Nothing is + # discarded, so warning here would just train users to ignore the warning. + import weaviate + + weaviate.use_async_with_custom( + http_host="localhost", + http_port=8290, + http_secure=False, + grpc_host="localhost", + grpc_port=8290, + grpc_secure=False, + ) + weaviate.use_async_with_local(port=8290) + weaviate.use_async_with_weaviate_cloud("abc.something.weaviate.cloud", None) + assert [str(w.message) for w in recwarn] == [] + + +def test_overridden_grpc_arguments_are_warned_about(emscripten) -> None: + # Python cannot drop required parameters the way TypeScript drops them from a type, + # so a WASM caller must pass something. Overriding keeps the client usable, but it + # must never look like the endpoint they gave was honoured. + import weaviate + + with pytest.warns(UserWarning, match="Con006") as record: + client = weaviate.use_async_with_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="grpc.example.com", + grpc_port=50051, + grpc_secure=True, + ) + msg = str(record[0].message) + assert "grpc.example.com:50051" in msg # what was discarded ... + assert "localhost:8080" in msg # ... and what is used instead + assert "WebAssembly" in msg # ... and why + _assert_grpc_rides_rest(client) + + +def test_an_explicit_local_grpc_port_is_warned_about_but_the_default_is_not(emscripten) -> None: + import weaviate + + with pytest.warns(UserWarning, match="Con006"): + client = weaviate.use_async_with_local(port=8080, grpc_port=8081) + _assert_grpc_rides_rest(client) diff --git a/weaviate/__init__.py b/weaviate/__init__.py index f3b38dab5..20ccff232 100644 --- a/weaviate/__init__.py +++ b/weaviate/__init__.py @@ -1,7 +1,29 @@ """Weaviate Python Client Library used to interact with a Weaviate instance.""" -import os import sys + +# Must run before every other import: under Pyodide there is no grpcio wheel, and importing +# the companion installs the pure-Python grpc shim that everything below resolves against. +if sys.platform == "emscripten": + try: + import weaviate_client_web # noqa: F401 + except ImportError as exc: + from importlib.util import find_spec + + # Only an absent companion earns the install hint; a companion that is present + # but fails to import (a broken dependency of its own) must surface that error. + if not (isinstance(exc, ModuleNotFoundError) and exc.name == "weaviate_client_web"): + raise + if find_spec("grpc") is None: + raise ImportError( + "weaviate requires the weaviate-client-web package under " + "WebAssembly/Pyodide: there is no grpcio wheel for Emscripten, and " + "weaviate-client-web provides the grpc-web (fetch) transport in its " + "place. Install it (e.g. micropip.install('weaviate-client-web')) and " + "import weaviate again." + ) from exc + +import os from importlib.metadata import PackageNotFoundError, version from typing import Any diff --git a/weaviate/collections/batch/async_.py b/weaviate/collections/batch/async_.py index c63ec2106..5f510439c 100644 --- a/weaviate/collections/batch/async_.py +++ b/weaviate/collections/batch/async_.py @@ -37,6 +37,7 @@ ReferenceToMulti, ) from weaviate.collections.classes.types import WeaviateProperties +from weaviate.connect.base import _grpc_web_shim_active from weaviate.connect.executor import aresult from weaviate.connect.v4 import ConnectionAsync from weaviate.exceptions import ( @@ -133,6 +134,15 @@ def __all_tasks_alive(self) -> bool: return self.__bg_tasks is not None and self.__bg_tasks.all_alive() async def _start(self): + if _grpc_web_shim_active(): + # fail fast and loud: over grpc-web the BatchStream RPC raises inside the + # background tasks, where it would otherwise surface as a silent drop or a + # never-ending flush() + raise WeaviateBatchStreamError( + "batch.stream() requires bidirectional gRPC streaming, which is not " + "possible over grpc-web/fetch (WebAssembly/Pyodide). Use " + "collection.data.insert_many() instead." + ) self.__number_of_nodes = await self.__cluster.get_number_of_nodes() async def loop_wrapper() -> None: diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index 99607e3ae..fa83a7126 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -9,6 +9,7 @@ from pydantic import BaseModel, field_validator, model_validator from weaviate.config import GrpcConfig, Proxies +from weaviate.exceptions import WeaviateInvalidInputError from weaviate.types import NUMBER from weaviate.util import is_weaviate_domain @@ -20,6 +21,17 @@ MAX_GRPC_MESSAGE_LENGTH = 104858000 # 10mb, needs to be synchronized with GRPC server +def _grpc_web_shim_active() -> bool: + """Whether the 'weaviate-client-web' shim has replaced the grpc module. + + The shim (used under WASM/Pyodide, where there is no grpcio wheel) routes unary RPCs + over grpc-web/fetch and cannot do bidirectional streaming. The marker attribute is + the documented contract between the two packages — keep all sniffs going through + this helper. + """ + return getattr(grpc, "__weaviate_client_web_shim__", False) is True + + class ProtocolParams(BaseModel): host: str port: int @@ -47,9 +59,19 @@ def is_gcp(self) -> bool: class ConnectionParams(BaseModel): http: ProtocolParams grpc: ProtocolParams + # Optional base-path prefix for a grpc-web endpoint served on the REST host:port + # (e.g. "/grpc-web"). None/"" means native gRPC. When set, sharing the REST + # host:port is permitted and the prefix is forwarded to the grpc-web transport. + grpc_path_prefix: Optional[str] = None @classmethod - def from_url(cls, url: str, grpc_port: int, grpc_secure: bool = False) -> "ConnectionParams": + def from_url( + cls, + url: str, + grpc_port: int, + grpc_secure: bool = False, + grpc_path_prefix: Optional[str] = None, + ) -> "ConnectionParams": parsed_url = urlparse(url) if parsed_url.scheme not in ["http", "https"]: raise ValueError(f"Unsupported scheme: {parsed_url.scheme}") @@ -69,6 +91,7 @@ def from_url(cls, url: str, grpc_port: int, grpc_secure: bool = False) -> "Conne port=grpc_port, secure=grpc_secure or parsed_url.scheme == "https", ), + grpc_path_prefix=grpc_path_prefix, ) @classmethod @@ -80,6 +103,7 @@ def from_params( grpc_host: str, grpc_port: int, grpc_secure: bool, + grpc_path_prefix: Optional[str] = None, ) -> "ConnectionParams": return cls( http=ProtocolParams( @@ -92,6 +116,7 @@ def from_params( port=grpc_port, secure=grpc_secure, ), + grpc_path_prefix=grpc_path_prefix, ) def is_gcp_on_wcd(self) -> bool: @@ -99,7 +124,10 @@ def is_gcp_on_wcd(self) -> bool: @model_validator(mode="after") def _check_port_collision(self: T) -> T: - if self.http.host == self.grpc.host and self.http.port == self.grpc.port: + same_endpoint = self.http.host == self.grpc.host and self.http.port == self.grpc.port + # grpc-web can be multiplexed onto the REST port under a base-path prefix, so a + # shared host:port is only a conflict for native gRPC (no prefix configured). + if same_endpoint and self._grpc_web_path_prefix == "": raise ValueError("http.port and grpc.port must be different if using the same host") return self @@ -111,6 +139,39 @@ def _grpc_address(self) -> Tuple[str, int]: def _grpc_target(self) -> str: return f"{self.grpc.host}:{self.grpc.port}" + @property + def _grpc_web_path_prefix(self) -> str: + """Return the normalized grpc-web base-path prefix; "" means native gRPC. + + A configured prefix is returned with a single leading slash and no trailing + slash (e.g. "grpc-web/" -> "/grpc-web"); empty/None -> "" (native gRPC). + """ + cleaned = (self.grpc_path_prefix or "").strip("/") + return f"/{cleaned}" if cleaned else "" + + def _check_grpc_web_usable(self, is_async: bool) -> None: + """Fail fast on a grpc-web prefix this process cannot honour; a no-op for native gRPC. + + A native grpcio channel would silently ignore the ``grpc-web.path_prefix`` option + and route over native gRPC, so the shim (which consumes it) must be in place. + """ + if self._grpc_web_path_prefix == "": + return + if not is_async: + raise WeaviateInvalidInputError( + "grpc_path_prefix (grpc-web) is only supported for async clients; " + "use use_async_with_custom(...) / WeaviateAsyncClient" + ) + if not _grpc_web_shim_active(): + raise WeaviateInvalidInputError( + "grpc_path_prefix enables grpc-web, which requires the " + "'weaviate-client-web' package (it installs a grpc shim before " + "'import weaviate'); it is not active in this environment. Under Pyodide a " + "plain `import weaviate` activates it; on CPython call " + "weaviate_client_web.install(force=True) and set_sender(make_httpx_sender()) " + "before importing weaviate (intended for integration testing)." + ) + def _grpc_channel( self, proxies: Dict[str, str], @@ -134,6 +195,10 @@ def _grpc_channel( if grpc_config is not None and grpc_config.channel_options is not None: options.extend(grpc_config.channel_options) + # nothing is added for native gRPC, so its channel options stay byte-for-byte unchanged + if (prefix := self._grpc_web_path_prefix) != "": + options.append(("grpc-web.path_prefix", prefix)) + if is_async: mod = grpc.aio else: diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index 29faaa3c2..726767d0e 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -1,5 +1,6 @@ """Helper functions for creating new WeaviateClient or WeaviateAsyncClient instances in common scenarios.""" +import sys from typing import Dict, Optional, Tuple, Union from urllib.parse import urlparse @@ -17,10 +18,45 @@ from weaviate.config import AdditionalConfig from weaviate.connect.base import ConnectionParams, ProtocolParams from weaviate.embedded import WEAVIATE_VERSION, EmbeddedOptions +from weaviate.exceptions import GRPC_WEB_SERVER_PATH_PREFIX from weaviate.util import docstring_deprecated from weaviate.validator import _validate_input, _ValidateArgument from weaviate.warnings import _Warnings +# The native-gRPC port a local Weaviate exposes by default. Doubles as the sentinel for +# "the caller did not pick a gRPC port of their own" in use_async_with_local(). +_LOCAL_GRPC_PORT_DEFAULT = 50051 + + +def _webify( + http: ProtocolParams, grpc: ProtocolParams, *, grpc_chosen_by_caller: bool +) -> ConnectionParams: + """Build connection params, routing gRPC over grpc-web under WebAssembly. + + Under Emscripten there is no grpcio wheel and no socket, so native gRPC cannot work + at all; grpc-web on the REST listener is the only transport that can. gRPC is + therefore pinned to the HTTP endpoint under Weaviate's own grpc-web base path, which + is what the TypeScript ``@weaviate/web`` client does (its ``webify()``). Everywhere + else this is the identity: ``grpc`` is used exactly as given. + + ``grpc_chosen_by_caller`` says whether ``grpc`` came from the caller rather than from + a convention of the helper's own; discarding a caller's endpoint warns, so nobody is + left believing an endpoint was honoured when it was not. + """ + if sys.platform != "emscripten": + # grpc_path_prefix passed explicitly: it keeps the constructor arguments (and so + # pydantic's echo of them in a validation error) identical to what callers saw + # before grpc-web existed. + return ConnectionParams(http=http, grpc=grpc, grpc_path_prefix=None) + + web_grpc = ProtocolParams(host=http.host, port=http.port, secure=http.secure) + if grpc_chosen_by_caller and web_grpc != grpc: + _Warnings.grpc_endpoint_forced_to_grpc_web( + requested=f"{grpc.host}:{grpc.port}", + effective=f"{web_grpc.host}:{web_grpc.port}", + ) + return ConnectionParams(http=http, grpc=web_grpc, grpc_path_prefix=GRPC_WEB_SERVER_PATH_PREFIX) + def __parse_weaviate_cloud_cluster_url(cluster_url: str) -> Tuple[str, str]: _validate_input(_ValidateArgument([str], "cluster_url", cluster_url)) @@ -384,6 +420,10 @@ def use_async_with_weaviate_cloud( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. + Under WebAssembly/Pyodide gRPC runs over grpc-web on the cluster's own REST endpoint + (443/TLS) rather than the separate ``grpc-`` host, because native gRPC cannot work + there. Nothing to configure: the cluster serves grpc-web itself. + Args: cluster_url: The WCD cluster URL or hostname to connect to. Usually in the form: rAnD0mD1g1t5.something.weaviate.cloud auth_credentials: The credentials to use for authentication with your Weaviate instance. This can be an API key, in which case pass a string or use `weaviate.classes.init.Auth.api_key()`, @@ -420,9 +460,11 @@ def use_async_with_weaviate_cloud( """ cluster_url, grpc_host = __parse_weaviate_cloud_cluster_url(cluster_url) return WeaviateAsyncClient( - connection_params=ConnectionParams( + connection_params=_webify( http=ProtocolParams(host=cluster_url, port=443, secure=True), grpc=ProtocolParams(host=grpc_host, port=443, secure=True), + # the grpc- host is this helper's own convention, never caller input + grpc_chosen_by_caller=False, ), auth_client_secret=__parse_auth_credentials(auth_credentials), additional_headers=headers, @@ -446,10 +488,15 @@ def use_async_with_local( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. + Under WebAssembly/Pyodide gRPC runs over grpc-web on the REST listener, because native + gRPC cannot work there. ``grpc_port`` is then replaced by ``port``; if you passed a + ``grpc_port`` of your own it is discarded and a ``UserWarning`` says so. + Args: host: The host to use for the underlying REST and GraphQL API calls. port: The port to use for the underlying REST and GraphQL API calls. - grpc_port: The port to use for the underlying gRPC API. + grpc_port: The port to use for the underlying gRPC API. Ignored under + WebAssembly/Pyodide, where gRPC shares the REST ``port`` over grpc-web. headers: Additional headers to include in the requests, e.g. API keys for Cloud vectorization. additional_config: This includes many additional, rarely used config options. use wvc.init.AdditionalConfig() to configure. skip_init_checks: Whether to skip the initialization checks when connecting to Weaviate. @@ -486,9 +533,11 @@ def use_async_with_local( >>> # The connection is automatically closed when the context is exited. """ return WeaviateAsyncClient( - connection_params=ConnectionParams( + connection_params=_webify( http=ProtocolParams(host=host, port=port, secure=False), grpc=ProtocolParams(host=host, port=grpc_port, secure=False), + # the default port is this helper's convention; anything else was chosen + grpc_chosen_by_caller=grpc_port != _LOCAL_GRPC_PORT_DEFAULT, ), additional_headers=headers, additional_config=additional_config, @@ -596,13 +645,23 @@ def use_async_with_custom( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. + Under WebAssembly/Pyodide gRPC runs over grpc-web on the REST listener, because native + gRPC cannot work there (no sockets, no ``grpcio`` wheel). ``grpc_host``, ``grpc_port`` + and ``grpc_secure`` are then replaced by ``http_host``, ``http_port`` and + ``http_secure``; if what you passed differed, it is discarded and a ``UserWarning`` + names both endpoints. This mirrors the TypeScript ``@weaviate/web`` client, which + removes those three options from its API altogether. + Args: http_host: The host to use for the underlying REST and GraphQL API calls. http_port: The port to use for the underlying REST and GraphQL API calls. http_secure: Whether to use https for the underlying REST and GraphQL API calls. - grpc_host: The host to use for the underlying gRPC API. - grpc_port: The port to use for the underlying gRPC API. - grpc_secure: Whether to use a secure channel for the underlying gRPC API. + grpc_host: The host to use for the underlying gRPC API. Ignored under + WebAssembly/Pyodide, where gRPC shares the REST endpoint over grpc-web. + grpc_port: The port to use for the underlying gRPC API. Ignored under + WebAssembly/Pyodide, where gRPC shares the REST endpoint over grpc-web. + grpc_secure: Whether to use a secure channel for the underlying gRPC API. Ignored + under WebAssembly/Pyodide, where gRPC shares the REST endpoint over grpc-web. headers: Additional headers to include in the requests, e.g. API keys for Cloud vectorization. additional_config: This includes many additional, rarely used config options. use wvc.init.AdditionalConfig() to configure. auth_credentials: The credentials to use for authentication with your Weaviate instance. This can be an API key, in which case pass a string or use `weaviate.classes.init.Auth.api_key()`, @@ -645,13 +704,11 @@ def use_async_with_custom( >>> # The connection is automatically closed when the context is exited. """ return WeaviateAsyncClient( - ConnectionParams.from_params( - http_host=http_host, - http_port=http_port, - http_secure=http_secure, - grpc_host=grpc_host, - grpc_port=grpc_port, - grpc_secure=grpc_secure, + _webify( + http=ProtocolParams(host=http_host, port=http_port, secure=http_secure), + grpc=ProtocolParams(host=grpc_host, port=grpc_port, secure=grpc_secure), + # all three gRPC arguments are required here, so they are always caller input + grpc_chosen_by_caller=True, ), auth_client_secret=__parse_auth_credentials(auth_credentials), additional_headers=headers, diff --git a/weaviate/connect/v4.py b/weaviate/connect/v4.py index 60214a8d8..f0a717a74 100644 --- a/weaviate/connect/v4.py +++ b/weaviate/connect/v4.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import sys import time from copy import copy from dataclasses import dataclass, field @@ -145,6 +146,17 @@ def __init__( self._connection_params = connection_params self._grpc_stub: Optional[weaviate_pb2_grpc.WeaviateStub] = None self._grpc_channel: Union[AsyncChannel, SyncChannel, None] = None + if sys.platform == "emscripten" and isinstance(self, ConnectionSync): + # fail at construction, before the first REST call surfaces an opaque + # ConnectError; _client/_grpc_channel are already set, so __del__ stays quiet + raise WeaviateStartUpError( + "The synchronous client is not supported under WebAssembly/Pyodide. " + "Use an async client (weaviate.use_async_with_local / " + "use_async_with_weaviate_cloud / use_async_with_custom, or " + "WeaviateAsyncClient) instead." + ) + # a grpc-web prefix this process cannot honour fails here, not deep inside connect() + connection_params._check_grpc_web_usable(is_async=not isinstance(self, ConnectionSync)) self.timeout_config = timeout_config self.__connection_config = connection_config self.__trust_env = trust_env @@ -339,15 +351,27 @@ async def execute(): def __handle_ping_response(self, res: health_weaviate_pb2.WeaviateHealthCheckResponse) -> None: if res.status != health_weaviate_pb2.WeaviateHealthCheckResponse.SERVING: raise WeaviateGRPCUnavailableError( - f"v{self.server_version}", self._connection_params._grpc_address + f"v{self.server_version}", + self._connection_params._grpc_address, + grpc_path_prefix=self.__grpc_web_prefix(), ) return None def __handle_ping_exception(self, e: Exception) -> None: + # pass the error along: its code()/details() are the only thing that says what + # actually went wrong, and the generic advice is wrong in grpc-web mode (no + # separate gRPC port, no firewall — REST just succeeded against this endpoint) raise WeaviateGRPCUnavailableError( - f"v{self.server_version}", self._connection_params._grpc_address + f"v{self.server_version}", + self._connection_params._grpc_address, + grpc_path_prefix=self.__grpc_web_prefix(), + error=e, ) from e + def __grpc_web_prefix(self) -> Optional[str]: + """The configured grpc-web base path, or None when this is native gRPC.""" + return self._connection_params._grpc_web_path_prefix or None + @property def grpc_stub(self) -> Optional[weaviate_pb2_grpc.WeaviateStub]: if not self.is_connected(): @@ -789,8 +813,11 @@ async def _execute() -> None: async with AsyncClient() as client: res = await client.get(PYPI_PACKAGE_URL, timeout=self.timeout_config.init) return resp(res) - except RequestError: - pass # ignore any errors related to requests, it is a best-effort warning + except (RequestError, OSError): + # ignore any errors related to requests, it is a best-effort warning. + # OSError covers fetch failures under Pyodide/WASM, where a page CSP + # commonly blocks pypi.org — that must not fail connect(). + pass return _execute() @@ -798,7 +825,7 @@ async def _execute() -> None: with Client() as client: res = client.get(PYPI_PACKAGE_URL, timeout=self.timeout_config.init) return resp(res) - except RequestError: + except (RequestError, OSError): pass # ignore any errors related to requests, it is a best-effort warning def delete( diff --git a/weaviate/embedded.py b/weaviate/embedded.py index a511665cc..fb5a19a15 100644 --- a/weaviate/embedded.py +++ b/weaviate/embedded.py @@ -5,6 +5,7 @@ import socket import stat import subprocess +import sys import tarfile import time import urllib.request @@ -175,6 +176,14 @@ def wait_till_listening(self) -> None: @staticmethod def check_supported_platform() -> None: + if sys.platform == "emscripten": + # without this guard the port probe below "succeeds" under Emscripten's lazy + # socket emulation and misreports that Weaviate is already listening + raise WeaviateStartUpError( + "Embedded Weaviate is not supported under WebAssembly/Pyodide: it spawns a " + "local Weaviate subprocess, and processes are unavailable in the browser. " + "Connect to a remote Weaviate instance instead." + ) if platform.system() in ["Windows"]: raise WeaviateStartUpError( f"""{platform.system()} is not supported with EmbeddedDB. Please upvote this feature request if you want diff --git a/weaviate/exceptions.py b/weaviate/exceptions.py index ce0fe6f7e..5b900f71b 100644 --- a/weaviate/exceptions.py +++ b/weaviate/exceptions.py @@ -317,6 +317,24 @@ def __init__(self, data: dict): super().__init__(msg) +def _grpc_status_of( + error: Optional[BaseException], +) -> Tuple[Optional[StatusCode], Optional[str]]: + """Return the (code, details) of a gRPC error, or (None, None) if it carries none.""" + if isinstance(error, (AioRpcError, Call)): + try: + return cast(Optional[StatusCode], error.code()), error.details() + except Exception: # a half-initialized call can raise instead of answering + return None, None + return None, None + + +# first Weaviate release that serves grpc-web on the REST port +GRPC_WEB_MIN_SERVER_VERSION = "1.38.3" +# the base path Weaviate itself serves grpc-web from +GRPC_WEB_SERVER_PATH_PREFIX = "/v1/grpc-web" + + class WeaviateGRPCUnavailableError(WeaviateBaseError): """Is raised when a gRPC-backed query is made with no gRPC connection present.""" @@ -324,7 +342,44 @@ def __init__( self, weaviate_version: str = "", grpc_address: Tuple[str, int] = ("not provided", 0), + grpc_path_prefix: Optional[str] = None, + error: Optional[BaseException] = None, ) -> None: + code, details = _grpc_status_of(error) + observed = "" + if code is not None or details: + code_name = code.name if code is not None else "unknown status" + observed = ( + f"\nThe gRPC call failed with: {code_name}{f' - {details}' if details else ''}\n" + ) + + if grpc_path_prefix: + # grpc-web multiplexes gRPC onto the REST host:port under a base path: there + # is no separate gRPC port to unblock, and the client has already talked to + # this exact endpoint over REST — so no firewall/wrong-port advice here. + address = f"{grpc_address[0]}:{grpc_address[1]}" + if code is StatusCode.UNIMPLEMENTED: + reason = f"""The server did not route the grpc-web path '{grpc_path_prefix}' at {address}. Either: +- the server is too old: grpc-web is served from Weaviate {GRPC_WEB_MIN_SERVER_VERSION} onwards, and this server reports {weaviate_version or "an unknown version"}, or +- the grpc-web base path is wrong: Weaviate serves grpc-web at '{GRPC_WEB_SERVER_PATH_PREFIX}'. The connect helpers set it themselves; only hand-built ConnectionParams choose it (grpc_path_prefix). +""" + else: + reason = f"""This error could be due to one of several reasons: +- grpc-web is not enabled or is incorrectly configured on the server at {address}. +- your connection is unstable or has a high latency. In this case you can: + - increase init-timeout in `weaviate.use_async_with_custom(additional_config=wvc.init.AdditionalConfig(timeout=wvc.init.Timeout(init=X)))` + - disable startup checks by connecting using `skip_init_checks=True` +""" + msg = f""" +Weaviate {weaviate_version} makes use of a high-speed gRPC API as well as a REST API. +Unfortunately, the gRPC health check against Weaviate could not be completed. + +This client speaks grpc-web (base path '{grpc_path_prefix}'), which carries gRPC over the REST endpoint {address}; there is no separate gRPC port. + +{reason}{observed}""" + super().__init__(msg) + return + if grpc_address[0] == "not provided": grpc_msg = "Please check the server address and port." else: @@ -340,7 +395,7 @@ def __init__( - your connection is unstable or has a high latency. In this case you can: - increase init-timeout in `weaviate.connect_to_local(additional_config=wvc.init.AdditionalConfig(timeout=wvc.init.Timeout(init=X)))` - disable startup checks by connecting using `skip_init_checks=True` -""" +{observed}""" super().__init__(msg) diff --git a/weaviate/proto/v1/__init__.py b/weaviate/proto/v1/__init__.py index 09171e683..62b0910f4 100644 --- a/weaviate/proto/v1/__init__.py +++ b/weaviate/proto/v1/__init__.py @@ -1,3 +1,4 @@ +import sys import warnings @@ -11,12 +12,23 @@ from packaging import version -from importlib.metadata import version as metadata_version +from importlib.metadata import PackageNotFoundError, version as metadata_version from weaviate.exceptions import WeaviateProtobufIncompatibility -def get_version(pkg: str)-> version.Version: - return version.parse(metadata_version(pkg)) +# grpcio version assumed under Pyodide/Emscripten, where grpcio has no wheel (excluded by +# the `sys_platform != "emscripten"` marker in setup.cfg) and the grpc module is the +# weaviate-client-web shim. Restricted to grpcio AND Emscripten so a broken grpcio install +# elsewhere still raises PackageNotFoundError, and a missing protobuf is never masked. +_GRPCIO_FALLBACK_VERSION = "1.72.1" + +def get_version(pkg: str) -> version.Version: + try: + return version.parse(metadata_version(pkg)) + except PackageNotFoundError: + if pkg == "grpcio" and sys.platform == "emscripten": + return version.parse(_GRPCIO_FALLBACK_VERSION) + raise pb_version, grpc_version = get_version("protobuf"), get_version("grpcio") if pb_version >= version.parse("6.30.0"): diff --git a/weaviate/warnings.py b/weaviate/warnings.py index 1c0a1ae0b..d69027fbb 100644 --- a/weaviate/warnings.py +++ b/weaviate/warnings.py @@ -325,6 +325,19 @@ def grpc_max_msg_size_not_found() -> None: stacklevel=1, ) + @staticmethod + def grpc_endpoint_forced_to_grpc_web(requested: str, effective: str) -> None: + warnings.warn( + message=f"""Con006: The gRPC endpoint you gave ({requested}) was overridden with {effective}. + + Under WebAssembly/Pyodide there is no socket and no grpcio wheel, so native gRPC cannot be used at all; + gRPC runs over grpc-web on the REST listener, which is the endpoint above. Pass gRPC arguments matching + the HTTP ones to silence this warning. A grpc-web transcoder on a separate endpoint is not reachable + through these helpers - build weaviate.connect.ConnectionParams yourself if you need one.""", + category=UserWarning, + stacklevel=1, + ) + @staticmethod def unknown_permission_encountered(permission: Any) -> None: warnings.warn( From ffc23bd23d2c765384c1ae3e2233686bf68644d6 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:43:14 +0200 Subject: [PATCH 02/12] docs(grpc-web): simplify the base-client comments and docstrings Comment/docstring wording only, no code changes: shorter sentences, plainer words (shim -> replacement, honour -> use, multiplexed -> shares, discarded -> ignored, REST listener -> REST endpoint), same meaning. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01GUNU7QgDr9MmFZnjKY9zFN --- weaviate/__init__.py | 8 +++--- weaviate/collections/batch/async_.py | 5 ++-- weaviate/connect/base.py | 31 +++++++++++---------- weaviate/connect/helpers.py | 40 +++++++++++++--------------- weaviate/connect/v4.py | 18 ++++++------- weaviate/embedded.py | 4 +-- weaviate/exceptions.py | 8 +++--- weaviate/proto/v1/__init__.py | 8 +++--- 8 files changed, 59 insertions(+), 63 deletions(-) diff --git a/weaviate/__init__.py b/weaviate/__init__.py index 20ccff232..fbd0fdabf 100644 --- a/weaviate/__init__.py +++ b/weaviate/__init__.py @@ -2,16 +2,16 @@ import sys -# Must run before every other import: under Pyodide there is no grpcio wheel, and importing -# the companion installs the pure-Python grpc shim that everything below resolves against. +# Must run before every other import: under Pyodide there is no grpcio, so importing +# weaviate-client-web first installs the pure-Python grpc replacement the imports below use. if sys.platform == "emscripten": try: import weaviate_client_web # noqa: F401 except ImportError as exc: from importlib.util import find_spec - # Only an absent companion earns the install hint; a companion that is present - # but fails to import (a broken dependency of its own) must surface that error. + # only a missing package gets the install hint; if it is installed but fails to + # import (e.g. one of its own dependencies is broken), show that error instead if not (isinstance(exc, ModuleNotFoundError) and exc.name == "weaviate_client_web"): raise if find_spec("grpc") is None: diff --git a/weaviate/collections/batch/async_.py b/weaviate/collections/batch/async_.py index 5f510439c..07083fe9b 100644 --- a/weaviate/collections/batch/async_.py +++ b/weaviate/collections/batch/async_.py @@ -135,9 +135,8 @@ def __all_tasks_alive(self) -> bool: async def _start(self): if _grpc_web_shim_active(): - # fail fast and loud: over grpc-web the BatchStream RPC raises inside the - # background tasks, where it would otherwise surface as a silent drop or a - # never-ending flush() + # fail early: over grpc-web the BatchStream RPC would fail inside the background + # tasks, which shows up as silently dropped objects or a flush() that never ends raise WeaviateBatchStreamError( "batch.stream() requires bidirectional gRPC streaming, which is not " "possible over grpc-web/fetch (WebAssembly/Pyodide). Use " diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index fa83a7126..0f7d40bd1 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -22,12 +22,11 @@ def _grpc_web_shim_active() -> bool: - """Whether the 'weaviate-client-web' shim has replaced the grpc module. + """Whether the 'weaviate-client-web' package has replaced the grpc module. - The shim (used under WASM/Pyodide, where there is no grpcio wheel) routes unary RPCs - over grpc-web/fetch and cannot do bidirectional streaming. The marker attribute is - the documented contract between the two packages — keep all sniffs going through - this helper. + That replacement (used under WASM/Pyodide, where grpcio is not available) sends unary + RPCs over grpc-web/fetch and cannot do bidirectional streaming. The marker attribute + is the agreed contract between the two packages; check it only through this helper. """ return getattr(grpc, "__weaviate_client_web_shim__", False) is True @@ -59,9 +58,9 @@ def is_gcp(self) -> bool: class ConnectionParams(BaseModel): http: ProtocolParams grpc: ProtocolParams - # Optional base-path prefix for a grpc-web endpoint served on the REST host:port - # (e.g. "/grpc-web"). None/"" means native gRPC. When set, sharing the REST - # host:port is permitted and the prefix is forwarded to the grpc-web transport. + # Optional base path of a grpc-web endpoint on the REST host:port (e.g. "/grpc-web"). + # None/"" means native gRPC. When set, gRPC may share the REST host:port and the + # prefix is passed on to the grpc-web transport. grpc_path_prefix: Optional[str] = None @classmethod @@ -125,8 +124,8 @@ def is_gcp_on_wcd(self) -> bool: @model_validator(mode="after") def _check_port_collision(self: T) -> T: same_endpoint = self.http.host == self.grpc.host and self.http.port == self.grpc.port - # grpc-web can be multiplexed onto the REST port under a base-path prefix, so a - # shared host:port is only a conflict for native gRPC (no prefix configured). + # with a grpc-web prefix gRPC may share the REST host:port; without one (native + # gRPC) the same host:port is a conflict if same_endpoint and self._grpc_web_path_prefix == "": raise ValueError("http.port and grpc.port must be different if using the same host") return self @@ -141,19 +140,19 @@ def _grpc_target(self) -> str: @property def _grpc_web_path_prefix(self) -> str: - """Return the normalized grpc-web base-path prefix; "" means native gRPC. + """The normalized grpc-web base path; "" means native gRPC. - A configured prefix is returned with a single leading slash and no trailing - slash (e.g. "grpc-web/" -> "/grpc-web"); empty/None -> "" (native gRPC). + A set prefix comes back with one leading slash and no trailing slash + (e.g. "grpc-web/" -> "/grpc-web"); empty/None -> "". """ cleaned = (self.grpc_path_prefix or "").strip("/") return f"/{cleaned}" if cleaned else "" def _check_grpc_web_usable(self, is_async: bool) -> None: - """Fail fast on a grpc-web prefix this process cannot honour; a no-op for native gRPC. + """Fail early on a grpc-web prefix this client cannot use; does nothing for native gRPC. A native grpcio channel would silently ignore the ``grpc-web.path_prefix`` option - and route over native gRPC, so the shim (which consumes it) must be in place. + and use native gRPC, so the replacement grpc module must be in place. """ if self._grpc_web_path_prefix == "": return @@ -195,7 +194,7 @@ def _grpc_channel( if grpc_config is not None and grpc_config.channel_options is not None: options.extend(grpc_config.channel_options) - # nothing is added for native gRPC, so its channel options stay byte-for-byte unchanged + # only grpc-web adds an option; native gRPC channel options are unchanged if (prefix := self._grpc_web_path_prefix) != "": options.append(("grpc-web.path_prefix", prefix)) diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index 726767d0e..deb80c8d7 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -23,30 +23,28 @@ from weaviate.validator import _validate_input, _ValidateArgument from weaviate.warnings import _Warnings -# The native-gRPC port a local Weaviate exposes by default. Doubles as the sentinel for -# "the caller did not pick a gRPC port of their own" in use_async_with_local(). +# Default gRPC port of a local Weaviate. use_async_with_local() also uses it to tell +# whether the caller picked a gRPC port of their own. _LOCAL_GRPC_PORT_DEFAULT = 50051 def _webify( http: ProtocolParams, grpc: ProtocolParams, *, grpc_chosen_by_caller: bool ) -> ConnectionParams: - """Build connection params, routing gRPC over grpc-web under WebAssembly. + """Build connection params; under WebAssembly, route gRPC over grpc-web. - Under Emscripten there is no grpcio wheel and no socket, so native gRPC cannot work - at all; grpc-web on the REST listener is the only transport that can. gRPC is - therefore pinned to the HTTP endpoint under Weaviate's own grpc-web base path, which - is what the TypeScript ``@weaviate/web`` client does (its ``webify()``). Everywhere - else this is the identity: ``grpc`` is used exactly as given. + Under Emscripten there is no grpcio and no sockets, so native gRPC cannot work; the + only option is grpc-web on the REST endpoint. gRPC is therefore pointed at the HTTP + host/port under Weaviate's own grpc-web base path, the same thing the TypeScript + ``@weaviate/web`` client does in its ``webify()``. On every other platform ``grpc`` + is used exactly as given. ``grpc_chosen_by_caller`` says whether ``grpc`` came from the caller rather than from - a convention of the helper's own; discarding a caller's endpoint warns, so nobody is - left believing an endpoint was honoured when it was not. + a default of the helper. If a caller's endpoint is replaced, a warning says so. """ if sys.platform != "emscripten": - # grpc_path_prefix passed explicitly: it keeps the constructor arguments (and so - # pydantic's echo of them in a validation error) identical to what callers saw - # before grpc-web existed. + # grpc_path_prefix=None is passed explicitly so the constructor call (and pydantic's + # error output for it) looks exactly as it did before grpc-web existed return ConnectionParams(http=http, grpc=grpc, grpc_path_prefix=None) web_grpc = ProtocolParams(host=http.host, port=http.port, secure=http.secure) @@ -420,7 +418,7 @@ def use_async_with_weaviate_cloud( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. - Under WebAssembly/Pyodide gRPC runs over grpc-web on the cluster's own REST endpoint + Under WebAssembly/Pyodide gRPC goes over grpc-web on the cluster's own REST endpoint (443/TLS) rather than the separate ``grpc-`` host, because native gRPC cannot work there. Nothing to configure: the cluster serves grpc-web itself. @@ -463,7 +461,7 @@ def use_async_with_weaviate_cloud( connection_params=_webify( http=ProtocolParams(host=cluster_url, port=443, secure=True), grpc=ProtocolParams(host=grpc_host, port=443, secure=True), - # the grpc- host is this helper's own convention, never caller input + # the grpc- host is the helper's default, not caller input grpc_chosen_by_caller=False, ), auth_client_secret=__parse_auth_credentials(auth_credentials), @@ -488,9 +486,9 @@ def use_async_with_local( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. - Under WebAssembly/Pyodide gRPC runs over grpc-web on the REST listener, because native + Under WebAssembly/Pyodide gRPC goes over grpc-web on the REST endpoint, because native gRPC cannot work there. ``grpc_port`` is then replaced by ``port``; if you passed a - ``grpc_port`` of your own it is discarded and a ``UserWarning`` says so. + ``grpc_port`` of your own it is ignored and a ``UserWarning`` says so. Args: host: The host to use for the underlying REST and GraphQL API calls. @@ -536,7 +534,7 @@ def use_async_with_local( connection_params=_webify( http=ProtocolParams(host=host, port=port, secure=False), grpc=ProtocolParams(host=host, port=grpc_port, secure=False), - # the default port is this helper's convention; anything else was chosen + # the default port comes from the helper; anything else the caller chose grpc_chosen_by_caller=grpc_port != _LOCAL_GRPC_PORT_DEFAULT, ), additional_headers=headers, @@ -645,10 +643,10 @@ def use_async_with_custom( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. - Under WebAssembly/Pyodide gRPC runs over grpc-web on the REST listener, because native + Under WebAssembly/Pyodide gRPC goes over grpc-web on the REST endpoint, because native gRPC cannot work there (no sockets, no ``grpcio`` wheel). ``grpc_host``, ``grpc_port`` and ``grpc_secure`` are then replaced by ``http_host``, ``http_port`` and - ``http_secure``; if what you passed differed, it is discarded and a ``UserWarning`` + ``http_secure``; if what you passed differed, it is ignored and a ``UserWarning`` names both endpoints. This mirrors the TypeScript ``@weaviate/web`` client, which removes those three options from its API altogether. @@ -707,7 +705,7 @@ def use_async_with_custom( _webify( http=ProtocolParams(host=http_host, port=http_port, secure=http_secure), grpc=ProtocolParams(host=grpc_host, port=grpc_port, secure=grpc_secure), - # all three gRPC arguments are required here, so they are always caller input + # all three gRPC arguments are required here, so they are caller input grpc_chosen_by_caller=True, ), auth_client_secret=__parse_auth_credentials(auth_credentials), diff --git a/weaviate/connect/v4.py b/weaviate/connect/v4.py index f0a717a74..e94cbc003 100644 --- a/weaviate/connect/v4.py +++ b/weaviate/connect/v4.py @@ -147,15 +147,15 @@ def __init__( self._grpc_stub: Optional[weaviate_pb2_grpc.WeaviateStub] = None self._grpc_channel: Union[AsyncChannel, SyncChannel, None] = None if sys.platform == "emscripten" and isinstance(self, ConnectionSync): - # fail at construction, before the first REST call surfaces an opaque - # ConnectError; _client/_grpc_channel are already set, so __del__ stays quiet + # fail here, at construction, instead of with an unclear ConnectError on the + # first REST call; _client/_grpc_channel are already set, so __del__ does not warn raise WeaviateStartUpError( "The synchronous client is not supported under WebAssembly/Pyodide. " "Use an async client (weaviate.use_async_with_local / " "use_async_with_weaviate_cloud / use_async_with_custom, or " "WeaviateAsyncClient) instead." ) - # a grpc-web prefix this process cannot honour fails here, not deep inside connect() + # a grpc-web prefix this client cannot use fails here, not deep inside connect() connection_params._check_grpc_web_usable(is_async=not isinstance(self, ConnectionSync)) self.timeout_config = timeout_config self.__connection_config = connection_config @@ -358,9 +358,9 @@ def __handle_ping_response(self, res: health_weaviate_pb2.WeaviateHealthCheckRes return None def __handle_ping_exception(self, e: Exception) -> None: - # pass the error along: its code()/details() are the only thing that says what - # actually went wrong, and the generic advice is wrong in grpc-web mode (no - # separate gRPC port, no firewall — REST just succeeded against this endpoint) + # pass the error on: its code()/details() say what actually went wrong, and the + # generic advice does not apply to grpc-web (no separate gRPC port, no firewall; + # REST just worked against this same endpoint) raise WeaviateGRPCUnavailableError( f"v{self.server_version}", self._connection_params._grpc_address, @@ -814,9 +814,9 @@ async def _execute() -> None: res = await client.get(PYPI_PACKAGE_URL, timeout=self.timeout_config.init) return resp(res) except (RequestError, OSError): - # ignore any errors related to requests, it is a best-effort warning. - # OSError covers fetch failures under Pyodide/WASM, where a page CSP - # commonly blocks pypi.org — that must not fail connect(). + # ignore any request error, this is a best-effort warning. OSError covers + # fetch failures under Pyodide/WASM, where the page's CSP often blocks + # pypi.org; that must not fail connect(). pass return _execute() diff --git a/weaviate/embedded.py b/weaviate/embedded.py index fb5a19a15..731560547 100644 --- a/weaviate/embedded.py +++ b/weaviate/embedded.py @@ -177,8 +177,8 @@ def wait_till_listening(self) -> None: @staticmethod def check_supported_platform() -> None: if sys.platform == "emscripten": - # without this guard the port probe below "succeeds" under Emscripten's lazy - # socket emulation and misreports that Weaviate is already listening + # without this check the port probe below "succeeds" under Emscripten's fake + # sockets and wrongly reports that Weaviate is already running raise WeaviateStartUpError( "Embedded Weaviate is not supported under WebAssembly/Pyodide: it spawns a " "local Weaviate subprocess, and processes are unavailable in the browser. " diff --git a/weaviate/exceptions.py b/weaviate/exceptions.py index 5b900f71b..fdeffacf1 100644 --- a/weaviate/exceptions.py +++ b/weaviate/exceptions.py @@ -331,7 +331,7 @@ def _grpc_status_of( # first Weaviate release that serves grpc-web on the REST port GRPC_WEB_MIN_SERVER_VERSION = "1.38.3" -# the base path Weaviate itself serves grpc-web from +# the base path Weaviate serves grpc-web on GRPC_WEB_SERVER_PATH_PREFIX = "/v1/grpc-web" @@ -354,9 +354,9 @@ def __init__( ) if grpc_path_prefix: - # grpc-web multiplexes gRPC onto the REST host:port under a base path: there - # is no separate gRPC port to unblock, and the client has already talked to - # this exact endpoint over REST — so no firewall/wrong-port advice here. + # grpc-web shares the REST host:port under a base path: there is no separate + # gRPC port to open, and REST already worked against this endpoint, so no + # firewall/wrong-port advice here address = f"{grpc_address[0]}:{grpc_address[1]}" if code is StatusCode.UNIMPLEMENTED: reason = f"""The server did not route the grpc-web path '{grpc_path_prefix}' at {address}. Either: diff --git a/weaviate/proto/v1/__init__.py b/weaviate/proto/v1/__init__.py index 62b0910f4..be18e3402 100644 --- a/weaviate/proto/v1/__init__.py +++ b/weaviate/proto/v1/__init__.py @@ -16,10 +16,10 @@ from weaviate.exceptions import WeaviateProtobufIncompatibility -# grpcio version assumed under Pyodide/Emscripten, where grpcio has no wheel (excluded by -# the `sys_platform != "emscripten"` marker in setup.cfg) and the grpc module is the -# weaviate-client-web shim. Restricted to grpcio AND Emscripten so a broken grpcio install -# elsewhere still raises PackageNotFoundError, and a missing protobuf is never masked. +# grpcio version to assume under Pyodide/Emscripten, where grpcio is not installed (see the +# sys_platform marker in setup.cfg) and the grpc module comes from weaviate-client-web. +# Limited to grpcio AND Emscripten: a broken grpcio install elsewhere still raises +# PackageNotFoundError, and a missing protobuf is never hidden. _GRPCIO_FALLBACK_VERSION = "1.72.1" def get_version(pkg: str) -> version.Version: From 61df36c84561e95a1078190beda262abfb24db9f Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:06:15 +0200 Subject: [PATCH 03/12] fix(grpc-web): address review findings on timeouts, diagnostics and Con006 - _encode_timeout/_abort_signal_ms: a huge finite timeout (e.g. 1e308) overflowed to infinity in the millisecond multiplication and raised OverflowError; both now compare against their cap before multiplying (no deadline / capped, as documented) - the "server too old / wrong grpc-web path" diagnosis now also requires the channel's synthetic "HTTP 404"/"HTTP 405" marker in the details, so a genuine UNIMPLEMENTED from a routed endpoint gets the generic message instead; the string contract is noted on both sides - Con006 prints grpc:// / grpcs:// schemes on both endpoints, so a mismatch in grpc_secure alone no longer shows two identical strings Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01GUNU7QgDr9MmFZnjKY9zFN --- .../web/src/weaviate_client_web/_channel.py | 6 ++++ .../src/weaviate_client_web/_httpx_fetch.py | 4 +++ packages/web/tests/test_httpx_fetch.py | 1 + packages/web/tests/test_transport.py | 1 + test/test_wasm_compat.py | 36 +++++++++++++++++++ weaviate/connect/helpers.py | 9 +++-- weaviate/exceptions.py | 7 +++- 7 files changed, 61 insertions(+), 3 deletions(-) diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py index af899849c..428c4f8f8 100644 --- a/packages/web/src/weaviate_client_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -52,6 +52,10 @@ def _encode_timeout(seconds: Optional[float]) -> Optional[str]: """ if seconds is None or not math.isfinite(seconds): return None + if seconds / 60 >= _GRPC_TIMEOUT_MAX: + # past the minute range there is no encodable deadline; checked before the + # multiplication below, which overflows to infinity for huge finite values + return None for amount, unit in ((seconds * 1000, "m"), (seconds, "S"), (seconds / 60, "M")): value = max(1, math.ceil(amount)) if value < _GRPC_TIMEOUT_MAX: @@ -382,6 +386,8 @@ def _non_grpc_web_error( what = f"the grpc-web body is truncated ({frame_error})" else: what = f"the body is not grpc-web framing ({frame_error})" + # the base client keys its "wrong path / server too old" diagnosis on this + # "HTTP " text (weaviate.exceptions.WeaviateGRPCUnavailableError) parts = [f"HTTP {http_status} from {url or ''}: {what}."] if http_status == 404: diff --git a/packages/web/src/weaviate_client_web/_httpx_fetch.py b/packages/web/src/weaviate_client_web/_httpx_fetch.py index e3f0ddf4a..b8f4be69c 100644 --- a/packages/web/src/weaviate_client_web/_httpx_fetch.py +++ b/packages/web/src/weaviate_client_web/_httpx_fetch.py @@ -87,6 +87,10 @@ def _abort_signal_ms(timeout: Optional[float]) -> Optional[int]: """ if timeout is None or not math.isfinite(timeout) or timeout <= 0: return None + if timeout >= _MAX_ABORT_SIGNAL_MS / 1000: + # compared before the multiplication below, which overflows to infinity for + # huge finite values + return _MAX_ABORT_SIGNAL_MS return min(math.ceil(timeout * 1000), _MAX_ABORT_SIGNAL_MS) diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py index 63480311e..c34f8e35d 100644 --- a/packages/web/tests/test_httpx_fetch.py +++ b/packages/web/tests/test_httpx_fetch.py @@ -304,6 +304,7 @@ def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): (30.0, 30_000), (1e8, _MAX_ABORT_SIGNAL_MS), (1e10, _MAX_ABORT_SIGNAL_MS), + (1e308, _MAX_ABORT_SIGNAL_MS), # finite, but *1000 overflows: capped, not an error ], ) def test_abort_signal_ms_bounds(timeout, expected_ms): diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index 70777f10c..e1a43492b 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -478,6 +478,7 @@ def test_body_excerpt_empty_and_non_printable(): (5_999_999_940, "99999999M"), # the largest deadline that still fits in minutes (1e10, None), # would need hours, which transcoders reject above 8H: no deadline (1e15, None), + (1e308, None), # finite, but *1000 overflows to infinity: must not raise ], ) def test_encode_timeout_stays_within_eight_digits(seconds, expected): diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py index 455776078..5398a825a 100644 --- a/test/test_wasm_compat.py +++ b/test/test_wasm_compat.py @@ -94,6 +94,23 @@ def test_grpc_web_404_names_the_two_real_causes_and_drops_firewall_advice() -> N assert "/v1/grpc-web" in msg # candidate 2: wrong prefix +def test_grpc_web_genuine_unimplemented_is_not_diagnosed_as_a_wrong_path() -> None: + # a routed grpc-web endpoint can itself return UNIMPLEMENTED (e.g. the health + # service is missing); only the channel's synthetic HTTP 404/405 means "not routed" + conn = _connection(prefix="/grpc-web") + error = AioRpcError( + grpc.StatusCode.UNIMPLEMENTED, Metadata(), Metadata(), details="Method not implemented" + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + assert "too old" not in msg + assert "1.38.3" not in msg + assert "UNIMPLEMENTED" in msg # the real status is still reported + assert "skip_init_checks=True" in msg # the generic grpc-web advice applies + + def test_grpc_web_non_404_error_still_omits_the_native_port_advice() -> None: conn = _connection(prefix="/grpc-web") error = AioRpcError( @@ -246,6 +263,25 @@ def test_overridden_grpc_arguments_are_warned_about(emscripten) -> None: _assert_grpc_rides_rest(client) +def test_a_secure_only_grpc_mismatch_is_visible_in_the_warning(emscripten) -> None: + # host:port alone would print two identical endpoints; the scheme shows what differed + import weaviate + + with pytest.warns(UserWarning, match="Con006") as record: + client = weaviate.use_async_with_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=8080, + grpc_secure=True, + ) + msg = str(record[0].message) + assert "grpcs://localhost:8080" in msg # what was discarded ... + assert "grpc://localhost:8080" in msg # ... and what is used instead + _assert_grpc_rides_rest(client) + + def test_an_explicit_local_grpc_port_is_warned_about_but_the_default_is_not(emscripten) -> None: import weaviate diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index deb80c8d7..80f81d711 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -28,6 +28,11 @@ _LOCAL_GRPC_PORT_DEFAULT = 50051 +def _grpc_endpoint_str(params: ProtocolParams) -> str: + """The endpoint as shown in Con006; the scheme makes a secure-only mismatch visible.""" + return f"{'grpcs' if params.secure else 'grpc'}://{params.host}:{params.port}" + + def _webify( http: ProtocolParams, grpc: ProtocolParams, *, grpc_chosen_by_caller: bool ) -> ConnectionParams: @@ -50,8 +55,8 @@ def _webify( web_grpc = ProtocolParams(host=http.host, port=http.port, secure=http.secure) if grpc_chosen_by_caller and web_grpc != grpc: _Warnings.grpc_endpoint_forced_to_grpc_web( - requested=f"{grpc.host}:{grpc.port}", - effective=f"{web_grpc.host}:{web_grpc.port}", + requested=_grpc_endpoint_str(grpc), + effective=_grpc_endpoint_str(web_grpc), ) return ConnectionParams(http=http, grpc=web_grpc, grpc_path_prefix=GRPC_WEB_SERVER_PATH_PREFIX) diff --git a/weaviate/exceptions.py b/weaviate/exceptions.py index fdeffacf1..d046e776d 100644 --- a/weaviate/exceptions.py +++ b/weaviate/exceptions.py @@ -358,7 +358,12 @@ def __init__( # gRPC port to open, and REST already worked against this endpoint, so no # firewall/wrong-port advice here address = f"{grpc_address[0]}:{grpc_address[1]}" - if code is StatusCode.UNIMPLEMENTED: + # the grpc-web channel reports an unrouted path as UNIMPLEMENTED with the + # HTTP status in the details (see weaviate_client_web._channel); a genuine + # UNIMPLEMENTED from a routed endpoint must not get the wrong-path diagnosis + if code is StatusCode.UNIMPLEMENTED and any( + marker in (details or "") for marker in ("HTTP 404", "HTTP 405") + ): reason = f"""The server did not route the grpc-web path '{grpc_path_prefix}' at {address}. Either: - the server is too old: grpc-web is served from Weaviate {GRPC_WEB_MIN_SERVER_VERSION} onwards, and this server reports {weaviate_version or "an unknown version"}, or - the grpc-web base path is wrong: Weaviate serves grpc-web at '{GRPC_WEB_SERVER_PATH_PREFIX}'. The connect helpers set it themselves; only hand-built ConnectionParams choose it (grpc_path_prefix). From e5e474a958bee95d6162eac8be01cc815ff40423 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 14 Sep 2026 15:36:39 +0200 Subject: [PATCH 04/12] refactor(grpc-web): top-level pyodide imports, [grpc-web] extra, in-Pyodide unit tests The web package now imports pyodide at module scope and is importable only under Emscripten/Pyodide. The base client gains a grpc-web extra (weaviate-client[grpc-web], marker-gated to Emscripten) as the documented install path. The unit tests stay in packages/web/tests but run inside Pyodide via ci/pyodide-e2e/units.mjs (async-native, no pytest); a conftest keeps CPython pytest from collecting them, and the CPython-only testing seams (install force flags, make_httpx_sender) are removed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- .github/workflows/main.yaml | 30 +- ci/pyodide-e2e/units.mjs | 154 ++++ packages/web/README.md | 42 +- .../web/src/weaviate_client_web/__init__.py | 30 +- .../web/src/weaviate_client_web/_channel.py | 4 +- .../src/weaviate_client_web/_httpx_fetch.py | 23 +- .../web/src/weaviate_client_web/_sender.py | 42 +- packages/web/src/weaviate_client_web/_shim.py | 10 +- packages/web/tests/conftest.py | 13 +- packages/web/tests/harness.py | 53 ++ packages/web/tests/runner.py | 42 ++ packages/web/tests/test_framing.py | 26 +- packages/web/tests/test_httpx_fetch.py | 694 +++++++----------- packages/web/tests/test_shim_install.py | 164 ++--- packages/web/tests/test_single_import.py | 139 ---- packages/web/tests/test_transport.py | 313 ++++---- setup.cfg | 6 + weaviate/__init__.py | 5 +- weaviate/connect/base.py | 9 +- 19 files changed, 866 insertions(+), 933 deletions(-) create mode 100644 ci/pyodide-e2e/units.mjs create mode 100644 packages/web/tests/harness.py create mode 100644 packages/web/tests/runner.py delete mode 100644 packages/web/tests/test_single_import.py diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 163c70afa..44e52d6d0 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -105,28 +105,8 @@ jobs: name: coverage-report-${{ matrix.folder }} path: coverage-${{ matrix.folder }}.xml - grpc-web-tests: - name: Run gRPC-Web Package Tests - runs-on: ubuntu-latest - timeout-minutes: 5 - strategy: - fail-fast: false - matrix: - version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: ${{ matrix.version }} - cache: 'pip' # caching pip dependencies - - run: | - pip install -r requirements-test.txt -r requirements-devel.txt - pip install -e . -e packages/web - - name: Run grpc-web package tests - run: pytest packages/web/tests - pyodide-e2e: - name: Run Pyodide (WASM) e2e Tests + name: Run Pyodide (WASM) Unit + e2e Tests runs-on: ubuntu-latest timeout-minutes: 15 # No Python matrix: the pinned Pyodide bundle fixes the interpreter (see @@ -154,6 +134,12 @@ jobs: pip install build python -m build --wheel --outdir dist . python -m build --wheel --outdir dist packages/web + - name: Run the unit suite inside Pyodide under Node + # The grpc-web package imports pyodide at module scope, so its unit tests only + # run here. Needs no running Weaviate. + run: | + npm install --prefix ci/pyodide-e2e + node ci/pyodide-e2e/units.mjs dist - name: start weaviate run: | source ./ci/compose.sh @@ -418,7 +404,7 @@ jobs: build-and-publish: name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI - needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test, grpc-web-tests, pyodide-e2e] + needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test, pyodide-e2e] runs-on: ubuntu-latest timeout-minutes: 20 steps: diff --git a/ci/pyodide-e2e/units.mjs b/ci/pyodide-e2e/units.mjs new file mode 100644 index 000000000..61ba72c2c --- /dev/null +++ b/ci/pyodide-e2e/units.mjs @@ -0,0 +1,154 @@ +// Runs the weaviate_client_web unit suite (packages/web/tests) inside Pyodide (WASM) +// under Node, plus bootstrap scenarios that each need a fresh interpreter. No running +// Weaviate is needed — everything is driven through fake senders / a fake pyfetch. +// +// Usage: node units.mjs +// must contain exactly the two locally-built pure wheels: +// weaviate_client-*.whl and weaviate_client_web-*.whl (same layout as run.mjs). +// +// The package imports pyodide at module scope, so this harness is the only place its +// unit tests can run; scenarios needing a clean import state (bootstrap and +// install-hint semantics) get one loadPyodide() each. Deliberately untested: the +// fall-through in weaviate/__init__.py when a real grpc module exists — no grpcio +// wheel exists for Emscripten, so that branch is CPython-only defence and cannot be +// exercised here. +import { readdirSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { loadPyodide } from "pyodide"; + +if (!process.argv[2]) { + console.error("usage: node units.mjs "); + process.exit(2); +} +const wheelsDir = resolve(process.argv[2]); +const here = dirname(fileURLToPath(import.meta.url)); + +const wheels = readdirSync(wheelsDir) + .filter((f) => f.endsWith(".whl")) + .sort(); // installs weaviate_client before weaviate_client_web, which depends on it +const prefixes = ["weaviate_client-", "weaviate_client_web-"]; +if ( + wheels.length !== 2 || + !prefixes.every((p) => wheels.some((w) => w.startsWith(p))) +) { + console.error( + `expected exactly one weaviate_client-*.whl and one weaviate_client_web-*.whl in ${wheelsDir}, found: ${JSON.stringify(wheels)}`, + ); + process.exit(2); +} + +// Fresh interpreter with micropip ready and the wheels dir mounted; `only` restricts +// which of the two wheels get installed (the install-hint scenarios need the base +// client without its companion). +async function freshPyodide(only = prefixes) { + const pyodide = await loadPyodide({}); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + pyodide.FS.mkdirTree("/wheels"); + pyodide.mountNodeFS("/wheels", wheelsDir); + for (const wheel of wheels) { + if (only.some((p) => wheel.startsWith(p))) { + await micropip.install(`emfs:/wheels/${wheel}`); + } + } + return pyodide; +} + +async function scenario(name, pyodide, code) { + try { + pyodide.runPython(code); + console.log(`OK scenario: ${name}`); + } catch (err) { + console.error(`FAIL scenario: ${name}`); + console.error(err); + process.exit(1); + } +} + +// --- fresh-interpreter bootstrap scenarios ----------------------------------------- + +await scenario( + "bare 'import weaviate' bootstraps the companion", + await freshPyodide(), + ` +import sys +assert "weaviate_client_web" not in sys.modules +import weaviate # the ONLY weaviate-side import: must bootstrap the companion +assert "weaviate_client_web" in sys.modules, "hook did not import the companion" +import weaviate_client_web +import grpc +import httpx +assert weaviate_client_web.is_installed() +assert weaviate_client_web.is_fetch_transport_installed() +assert getattr(grpc, "__weaviate_client_web_shim__", False) is True +assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False +) is True +`, +); + +await scenario( + "missing companion raises the install hint", + await freshPyodide(["weaviate_client-"]), + ` +try: + import weaviate +except ImportError as e: + assert "weaviate-client-web" in str(e), str(e) + assert "weaviate-client[grpc-web]" in str(e), str(e) + assert "WebAssembly/Pyodide" in str(e), str(e) +else: + raise AssertionError("expected ImportError without the companion") +`, +); + +await scenario( + "a broken companion surfaces its own error, not the install hint", + await freshPyodide(["weaviate_client-"]), + ` +# An INSTALLED companion whose import fails (here: a missing dependency of its own) +# must raise that error — the "install weaviate-client[grpc-web]" hint would send the +# user to reinstall a package that is already there. +import pathlib, sys +pkg = pathlib.Path("/broken/weaviate_client_web") +pkg.mkdir(parents=True) +(pkg / "__init__.py").write_text( + "raise ModuleNotFoundError(\\"No module named 'anyio'\\", name='anyio')\\n" +) +sys.path.insert(0, "/broken") +try: + import weaviate +except ImportError as e: + assert e.name == "anyio", (e.name, str(e)) + assert "anyio" in str(e), str(e) + assert "grpc-web" not in str(e), str(e) +else: + raise AssertionError("expected the companion's own ImportError to surface") +`, +); + +// --- the main unit suite ----------------------------------------------------------- + +const testsDir = resolve(here, "../../packages/web/tests"); +const pyodide = await freshPyodide(); +console.log( + `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, +); +pyodide.FS.mkdirTree("/units"); +pyodide.mountNodeFS("/units", testsDir); +try { + await pyodide.runPythonAsync(` +import sys +sys.path.insert(0, "/units") +import runner +await runner.main() +`); +} catch (err) { + console.error(err); + process.exit(1); +} +// The interpreters loaded above keep live handles on the Node event loop, so the +// process does not exit on its own. +process.exit(0); diff --git a/packages/web/README.md b/packages/web/README.md index 582ab2796..c17e7f8df 100644 --- a/packages/web/README.md +++ b/packages/web/README.md @@ -12,6 +12,23 @@ Requires Weaviate ≥ 1.38.3 (the first release to serve grpc-web natively) or a transcoder in front of an older server. Pyodide ≥ 0.27 recommended; verified on Pyodide 314.0.4 (CPython 3.14). +## Installation + +Install through the base client's `grpc-web` extra: + +```python +import micropip +await micropip.install("weaviate-client[grpc-web]") +``` + +The extra carries a `sys_platform == "emscripten"` marker, so the same requirement is a +no-op on CPython — one requirements list works everywhere. Installing the companion +directly (`micropip.install("weaviate-client-web")`) works too. + +This package is defined for its environment: it imports `pyodide` at module scope and is +therefore only importable under Emscripten/Pyodide. On CPython the base client never +imports it (and the extra never installs it). + ## How it works Under Pyodide there is no `grpcio` Emscripten wheel, and `import weaviate` hard-imports @@ -81,7 +98,7 @@ await collection.query.near_text("hello", limit=3) Nothing selects grpc-web: `use_async_with_local()`, `use_async_with_weaviate_cloud()` and `use_async_with_custom()` all route gRPC onto the REST endpoint under `/v1/grpc-web` when -they run under Emscripten, and behave exactly as before everywhere else. +they run under Emscripten, and use native gRPC everywhere else. ```python client = weaviate.use_async_with_weaviate_cloud( @@ -161,11 +178,20 @@ deployments that go through a grpc-web transcoder or a proxy must configure CORS - note that a CORS-blocked request is indistinguishable from a network failure in the browser (`TypeError: Failed to fetch`), and is retried as UNAVAILABLE. -## Testing on CPython +## Testing + +Because the package imports `pyodide` at module scope, its unit tests run inside +Pyodide. From the repository root: + +```sh +python -m build --wheel --outdir dist . +python -m build --wheel --outdir dist packages/web +npm install --prefix ci/pyodide-e2e +node ci/pyodide-e2e/units.mjs dist # unit suite + bootstrap scenarios, no Weaviate needed +node ci/pyodide-e2e/run.mjs dist # e2e suite, needs a running Weaviate (see ci/) +``` -`weaviate_client_web.install(force=True)` installs the shim on a normal CPython -interpreter (run it in a fresh process, before importing `weaviate`). Inject a sender -with `weaviate_client_web.set_sender(...)` (e.g. `make_httpx_sender()`) to exercise the -transport against an Envoy/vanguard transcoder without a browser. -`install_fetch_transport(force=True)` likewise patches httpx on CPython, given an -importable `pyodide.http` stand-in. +`units.mjs` runs the unit suite in `packages/web/tests/` (driven entirely through fake +senders and a fake `pyfetch`) plus fresh-interpreter bootstrap scenarios; `run.mjs` +runs the e2e suite against a live Weaviate. A `conftest.py` keeps pytest from +collecting the test modules on CPython, where they cannot import. diff --git a/packages/web/src/weaviate_client_web/__init__.py b/packages/web/src/weaviate_client_web/__init__.py index 2e9217991..d573b7af3 100644 --- a/packages/web/src/weaviate_client_web/__init__.py +++ b/packages/web/src/weaviate_client_web/__init__.py @@ -23,14 +23,22 @@ warns that it overrode them. An explicit ``import weaviate_client_web`` before ``import weaviate`` also works and -remains the explicit form. The shim is installed automatically only under Emscripten, so -importing this package on a normal CPython install never clobbers a real, working -``grpcio``. Async clients only — the synchronous client is not supported in the browser. +remains the explicit form. This package is defined for its environment: it imports +``pyodide`` at module scope, so it is only importable under Emscripten/Pyodide — on +CPython the base client never imports it, and the ``weaviate-client[grpc-web]`` extra +does not install it there (platform marker). Async clients only — the synchronous client +is not supported in the browser. """ import os import sys +from ._channel import GrpcWebChannel, set_sender +from ._httpx_fetch import ( + install_fetch_transport, + is_fetch_transport_installed, + uninstall_fetch_transport, +) from ._shim import StatusCode, install, is_installed __all__ = [ @@ -40,7 +48,6 @@ "uninstall_fetch_transport", "is_fetch_transport_installed", "set_sender", - "make_httpx_sender", "GrpcWebChannel", "StatusCode", ] @@ -54,21 +61,8 @@ def _bootstrap() -> None: os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") install() # The REST path needs fetch too: httpx/httpcore open raw sockets, which do - # not exist under WASM. Imported lazily so CPython imports stay light. - from ._httpx_fetch import install_fetch_transport - + # not exist under WASM. install_fetch_transport() _bootstrap() - -# Imported after the bootstrap. These modules pull their grpc base classes directly from -# ``._shim`` (not via ``sys.modules['grpc']``), so importing them is safe regardless of -# whether the shim was installed. -from ._channel import GrpcWebChannel, set_sender # noqa: E402 -from ._httpx_fetch import ( # noqa: E402 - install_fetch_transport, - is_fetch_transport_installed, - uninstall_fetch_transport, -) -from ._sender import make_httpx_sender # noqa: E402 diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py index 428c4f8f8..64e6d06be 100644 --- a/packages/web/src/weaviate_client_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -23,12 +23,12 @@ from ._sender import Sender, pyfetch_sender from ._shim import AioChannel, AioRpcError, StatusCode, status_from_int -# Module-level default sender; overridable for tests / non-browser runtimes. +# Module-level default sender; overridable for tests. _default_sender: Sender = pyfetch_sender def set_sender(sender: Sender) -> None: - """Override the default async sender used by new channels (tests/integration).""" + """Override the default async sender used by new channels (tests).""" global _default_sender _default_sender = sender diff --git a/packages/web/src/weaviate_client_web/_httpx_fetch.py b/packages/web/src/weaviate_client_web/_httpx_fetch.py index b8f4be69c..b9d400465 100644 --- a/packages/web/src/weaviate_client_web/_httpx_fetch.py +++ b/packages/web/src/weaviate_client_web/_httpx_fetch.py @@ -21,6 +21,10 @@ - the browser's fetch follows redirects internally, so httpx never sees a 3xx; - multi-value response headers (e.g. Set-Cookie) are folded into one value; - responses are fully buffered (no streaming). + +Like the rest of this package, this module imports ``pyodide`` at module scope and is +therefore only importable under Emscripten/Pyodide (or with a ``pyodide`` stand-in +pre-installed in ``sys.modules``). """ import math @@ -28,13 +32,14 @@ from typing import Callable, Dict, Optional import httpx +from pyodide.http import pyfetch # type: ignore[import-not-found] _installed = False _original_handle_async_request: Optional[Callable] = None # Hop-by-hop / connection-managed headers that the browser's fetch controls itself. -# Browsers silently drop forbidden headers, but Node's undici (used by the CPython/Node -# test path) rejects some of them outright, so strip them before handing off. +# Browsers silently drop forbidden headers, but Node's undici (behind pyfetch in the +# Node-based test harness) rejects some of them outright, so strip them before handing off. _FETCH_MANAGED_HEADERS = { "host", "connection", @@ -122,8 +127,6 @@ def _validate_header(name: str, value: str) -> None: async def _fetch_handle_async_request( self: httpx.AsyncHTTPTransport, request: httpx.Request ) -> httpx.Response: - from pyodide.http import pyfetch # type: ignore[import-not-found] - headers: Dict[str, str] = {} for k, v in request.headers.items(): if k.lower() in _FETCH_MANAGED_HEADERS: @@ -177,21 +180,17 @@ async def _fetch_handle_async_request( _fetch_handle_async_request.__weaviate_fetch_shim__ = True # type: ignore[attr-defined] -def install_fetch_transport(force: bool = False) -> None: +def install_fetch_transport() -> None: """Patch ``httpx.AsyncHTTPTransport`` to send requests through ``fetch``. - Installs only under Emscripten unless ``force=True`` (CPython testing, where a - ``pyodide`` stub must be importable). Idempotent. + Installs only under Emscripten (elsewhere httpx's own socket transports work and + must be left in place). Idempotent. """ global _installed, _original_handle_async_request if _installed: return - if not force and sys.platform != "emscripten": + if sys.platform != "emscripten": return - # Fail fast: the handler imports pyfetch per request, so a missing pyodide module - # would otherwise surface as a confusing ModuleNotFoundError on the first request. - from pyodide.http import pyfetch # type: ignore[import-not-found] # noqa: F401 - _original_handle_async_request = httpx.AsyncHTTPTransport.handle_async_request httpx.AsyncHTTPTransport.handle_async_request = _fetch_handle_async_request # type: ignore[method-assign] _installed = True diff --git a/packages/web/src/weaviate_client_web/_sender.py b/packages/web/src/weaviate_client_web/_sender.py index d41f9f6c0..fd2a3488a 100644 --- a/packages/web/src/weaviate_client_web/_sender.py +++ b/packages/web/src/weaviate_client_web/_sender.py @@ -2,11 +2,17 @@ A *sender* is ``async def sender(url, headers, body, timeout) -> (status, headers, body)``. The default uses ``pyodide.http.pyfetch`` (browser fetch); a sender can be injected for -testing or for non-browser runtimes via :func:`weaviate_client_web.set_sender`. +testing via :func:`weaviate_client_web.set_sender`. + +Like the rest of this package, this module imports ``pyodide`` at module scope and is +therefore only importable under Emscripten/Pyodide (or with a ``pyodide`` stand-in +pre-installed in ``sys.modules``). """ from typing import Awaitable, Callable, Dict, Optional, Tuple +from pyodide.http import pyfetch # type: ignore[import-not-found] + Sender = Callable[ [str, Dict[str, str], bytes, Optional[float]], Awaitable[Tuple[int, Dict[str, str], bytes]], @@ -18,12 +24,9 @@ async def pyfetch_sender( ) -> Tuple[int, Dict[str, str], bytes]: """Default browser sender. - Imports ``pyodide.http`` lazily so this module stays importable on CPython (where - ``pyodide`` does not exist). ``pyfetch`` has no timeout parameter of its own; the - call deadline is enforced by ``GrpcWebChannel._unary`` via ``asyncio.wait_for``. + ``pyfetch`` has no timeout parameter of its own; the call deadline is enforced by + ``GrpcWebChannel._unary`` via ``asyncio.wait_for``. """ - from pyodide.http import pyfetch # type: ignore[import-not-found] - response = await pyfetch(url, method="POST", headers=headers, body=body) data = await response.bytes() try: @@ -31,30 +34,3 @@ async def pyfetch_sender( except Exception: # pragma: no cover - header shape varies across Pyodide versions resp_headers = {} return int(response.status), resp_headers, data - - -def make_httpx_sender(client: Optional[object] = None) -> Sender: - """Build a sender backed by ``httpx.AsyncClient`` for CPython tests/integration. - - Targets a grpc-web transcoder (Envoy / connectrpc vanguard). - """ - import httpx - - async def _send( - url: str, headers: Dict[str, str], body: bytes, timeout: Optional[float] - ) -> Tuple[int, Dict[str, str], bytes]: - owns_client = client is None - active = client or httpx.AsyncClient() - assert isinstance(active, httpx.AsyncClient) - try: - response = await active.post(url, headers=headers, content=body, timeout=timeout) - return ( - response.status_code, - {k.lower(): v for k, v in response.headers.items()}, - response.content, - ) - finally: - if owns_client: - await active.aclose() - - return _send diff --git a/packages/web/src/weaviate_client_web/_shim.py b/packages/web/src/weaviate_client_web/_shim.py index b04b8837a..0624f5259 100644 --- a/packages/web/src/weaviate_client_web/_shim.py +++ b/packages/web/src/weaviate_client_web/_shim.py @@ -217,14 +217,14 @@ def is_installed() -> bool: return getattr(sys.modules.get("grpc"), _SHIM_MARKER, False) is True -def install(force: bool = False) -> bool: +def install() -> bool: """Install the shim into ``sys.modules`` as ``grpc`` and submodules. - On normal platforms this is a no-op unless ``force=True`` — we must never clobber a - real, working ``grpcio``. Under Emscripten the bootstrap calls this automatically. - Returns ``True`` if the shim is in place afterwards. + On normal platforms this is a no-op — a real, working ``grpcio`` must be left in + place. Under Emscripten the bootstrap calls this automatically. Returns ``True`` + if the shim is in place afterwards. """ - if not force and sys.platform != "emscripten": + if sys.platform != "emscripten": return False if is_installed(): return True diff --git a/packages/web/tests/conftest.py b/packages/web/tests/conftest.py index fe4afb09d..9bc903df3 100644 --- a/packages/web/tests/conftest.py +++ b/packages/web/tests/conftest.py @@ -1,7 +1,8 @@ -import pathlib -import sys +"""Keep pytest away from this directory. -# Make the package importable without an editable install. -_SRC = pathlib.Path(__file__).resolve().parents[1] / "src" -if str(_SRC) not in sys.path: - sys.path.insert(0, str(_SRC)) +The package under test imports ``pyodide`` at module scope, so the test modules here +are only importable under Emscripten/Pyodide. They are run by +``ci/pyodide-e2e/units.mjs`` (via ``runner.py``), not by pytest. +""" + +collect_ignore_glob = ["*.py"] diff --git a/packages/web/tests/harness.py b/packages/web/tests/harness.py new file mode 100644 index 000000000..c5c8c0ad2 --- /dev/null +++ b/packages/web/tests/harness.py @@ -0,0 +1,53 @@ +"""Shared helpers for the in-Pyodide test modules (pytest is not available here).""" + +import contextlib +import sys +from typing import Any, Optional + + +class _Caught: + value: Any + + +@contextlib.contextmanager +def raises(exc_type, contains: Optional[str] = None): + """``pytest.raises`` stand-in; ``contains`` is a plain substring, not a regex.""" + caught = _Caught() + try: + yield caught + except exc_type as e: + caught.value = e + if contains is not None and contains not in str(e): + raise AssertionError(f"{e!r} does not contain {contains!r}") from e + else: + raise AssertionError(f"expected {exc_type.__name__} to be raised") + + +@contextlib.contextmanager +def patched(obj, name: str, value): + """``monkeypatch.setattr`` stand-in with restore-on-exit.""" + sentinel = object() + old = getattr(obj, name, sentinel) + setattr(obj, name, value) + try: + yield + finally: + if old is sentinel: + delattr(obj, name) + else: + setattr(obj, name, old) + + +@contextlib.contextmanager +def sys_module(name: str, module): + """Set ``sys.modules[name]`` (``None`` makes ``import name`` raise) and restore.""" + present = name in sys.modules + old = sys.modules.get(name) + sys.modules[name] = module + try: + yield + finally: + if present: + sys.modules[name] = old # type: ignore[assignment] + else: + del sys.modules[name] diff --git a/packages/web/tests/runner.py b/packages/web/tests/runner.py new file mode 100644 index 000000000..e9a6ae1fc --- /dev/null +++ b/packages/web/tests/runner.py @@ -0,0 +1,42 @@ +"""Collects and runs the test modules in this directory on Pyodide's event loop. + +Invoked by ``ci/pyodide-e2e/units.mjs``, which mounts this directory into the +interpreter and awaits :func:`main` (``asyncio.run()`` cannot be used inside Pyodide). +Test functions are ``test_*`` callables — plain or ``async`` — collected per module, +so names are qualified and cannot shadow across files. +""" + +import inspect +import traceback +from typing import List + +import test_framing +import test_httpx_fetch +import test_shim_install +import test_transport + +_MODULES = (test_framing, test_transport, test_httpx_fetch, test_shim_install) + + +async def main() -> None: + tests = [] + for module in _MODULES: + for name in sorted(vars(module)): + fn = getattr(module, name) + if name.startswith("test_") and callable(fn): + tests.append((f"{module.__name__}.{name}", fn)) + failures: List[str] = [] + for name, fn in tests: + try: + result = fn() + if inspect.iscoroutine(result): + await result + except Exception: + failures.append(name) + traceback.print_exc() + print(f"FAIL {name}", flush=True) + else: + print(f"OK {name}", flush=True) + print(f"{len(tests) - len(failures)}/{len(tests)} unit tests passed", flush=True) + if failures: + raise SystemExit(f"{len(failures)} unit test(s) failed: {', '.join(failures)}") diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py index fd18a35a3..16aeebe69 100644 --- a/packages/web/tests/test_framing.py +++ b/packages/web/tests/test_framing.py @@ -1,6 +1,8 @@ +"""grpc-web framing tests. Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``.""" + import struct -import pytest +from harness import raises from weaviate_client_web._framing import ( FrameError, @@ -42,7 +44,7 @@ def test_split_response_multiple_messages(): def test_split_response_message_after_trailer_raises(): body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") - with pytest.raises(FrameError, match="after the trailer"): + with raises(FrameError, contains="after the trailer"): split_response(body) @@ -88,25 +90,25 @@ def test_parse_trailers_accepts_lf_only_lines(): def test_parse_trailers_keeps_status_when_a_key_is_not_ascii(): # one odd key from a proxy must not throw away the whole block - parsed = parse_trailers("x-caf\u00e9:1\r\ngrpc-status:0\r\n".encode("utf-8")) + parsed = parse_trailers("x-café:1\r\ngrpc-status:0\r\n".encode("utf-8")) assert parsed["grpc-status"] == "0" - assert parsed["x-caf\u00e9"] == "1" + assert parsed["x-café"] == "1" def test_truncated_frame_raises(): framed = encode_message(b"hello")[:-2] - with pytest.raises(TruncatedFrameError): + with raises(TruncatedFrameError): list(iter_frames(framed)) - with pytest.raises(TruncatedFrameError): + with raises(TruncatedFrameError): list(iter_frames(b"\x00\x00\x00")) # shorter than one frame header -@pytest.mark.parametrize("first_byte", [b"{", b"<", b"\x02", b"\x40", b"\xff"]) -def test_unknown_frame_flag_raises(first_byte): +def test_unknown_frame_flag_raises(): # a JSON / HTML body, or a flag bit this transport does not know - body = first_byte + b"\x00\x00\x00\x01x" - with pytest.raises(UnknownFrameFlagError, match="unknown grpc-web frame flag"): - list(iter_frames(body)) + for first_byte in (b"{", b"<", b"\x02", b"\x40", b"\xff"): + body = first_byte + b"\x00\x00\x00\x01x" + with raises(UnknownFrameFlagError, contains="unknown grpc-web frame flag"): + list(iter_frames(body)) def test_frame_errors_are_value_errors(): @@ -116,5 +118,5 @@ def test_frame_errors_are_value_errors(): def test_compressed_message_frame_rejected(): body = _frame(b"x", 0x01) - with pytest.raises(FrameError, match="compressed"): + with raises(FrameError, contains="compressed"): split_response(body) diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py index c34f8e35d..c12a29d5f 100644 --- a/packages/web/tests/test_httpx_fetch.py +++ b/packages/web/tests/test_httpx_fetch.py @@ -1,30 +1,24 @@ -"""Tests for the fetch-based httpx transport (_httpx_fetch.py). +"""Tests for the fetch-based httpx transport (``_httpx_fetch.py``). -In-process tests call ``_fetch_handle_async_request`` directly with a fake -``pyodide.http`` module injected into ``sys.modules`` — no global monkeypatch of -``httpx.AsyncHTTPTransport`` is needed, so the real httpx in the dev environment is left -untouched. Install semantics (which DO patch the class globally) run in fresh -subprocesses, mirroring test_shim_install.py. +Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``. ``pyfetch`` is bound at import +time in ``_httpx_fetch``, so fakes patch that module attribute; ``from js import +AbortSignal`` is resolved per request, so a fake js module in ``sys.modules`` +intercepts it even under real Pyodide. + +The install-semantics tests at the bottom run against this interpreter's real +installation: importing ``weaviate_client_web`` bootstrapped the transport globally. """ -import asyncio -import pathlib -import subprocess -import sys -import textwrap +import contextlib import types from typing import Any, Dict, List, Optional import httpx -import pytest - -from weaviate_client_web._httpx_fetch import ( - _MAX_ABORT_SIGNAL_MS, - _abort_signal_ms, - _fetch_handle_async_request, -) +from harness import patched, raises, sys_module -_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") +import weaviate_client_web +import weaviate_client_web._httpx_fetch as _httpx_fetch +from weaviate_client_web._httpx_fetch import _MAX_ABORT_SIGNAL_MS, _abort_signal_ms class FakeFetchResponse: @@ -50,105 +44,125 @@ async def __call__(self, url: str, **kwargs: Any) -> FakeFetchResponse: return self.response -@pytest.fixture -def fake_pyfetch(monkeypatch) -> FakePyfetch: +@contextlib.contextmanager +def fake_pyfetch(): fetch = FakePyfetch() - pyodide_mod = types.ModuleType("pyodide") - http_mod = types.ModuleType("pyodide.http") - http_mod.pyfetch = fetch # type: ignore[attr-defined] - pyodide_mod.http = http_mod # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "pyodide", pyodide_mod) - monkeypatch.setitem(sys.modules, "pyodide.http", http_mod) - return fetch + with patched(_httpx_fetch, "pyfetch", fetch): + yield fetch + + +class _AbortSignalRecorder: + def __init__(self): + self.timeouts: List[int] = [] + + def timeout(self, ms: int): + self.timeouts.append(ms) + return f"signal-{ms}" + + +@contextlib.contextmanager +def fake_abort_signal(): + recorder = _AbortSignalRecorder() + js_mod = types.ModuleType("js") + js_mod.AbortSignal = recorder # type: ignore[attr-defined] + with sys_module("js", js_mod): + yield recorder + + +@contextlib.contextmanager +def missing_js_module(): + # sys.modules[name] = None makes ``import js`` raise ImportError — the closest + # in-process stand-in for an environment without the js bridge. + with sys_module("js", None): + yield -async def _handle_async(request: httpx.Request) -> httpx.Response: +async def _handle(request: httpx.Request) -> httpx.Response: # self is unused by the handler implementation; a bare transport instance suffices transport = httpx.AsyncHTTPTransport.__new__(httpx.AsyncHTTPTransport) - response = await _fetch_handle_async_request(transport, request) + response = await _httpx_fetch._fetch_handle_async_request(transport, request) await response.aread() # httpx.AsyncClient reads non-streamed responses the same way return response -def _handle(request: httpx.Request) -> httpx.Response: - return asyncio.run(_handle_async(request)) - - class _FetchTransport(httpx.AsyncBaseTransport): - """Route an ``httpx.AsyncClient`` through the handler without patching httpx globally.""" + """Route an ``httpx.AsyncClient`` through the handler without touching global state.""" async def handle_async_request(self, request: httpx.Request) -> httpx.Response: - return await _fetch_handle_async_request(self, request) # type: ignore[arg-type] - + return await _httpx_fetch._fetch_handle_async_request(self, request) # type: ignore[arg-type] -def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: - async def main() -> httpx.Response: - async with httpx.AsyncClient(transport=_FetchTransport()) as client: - return await client.request(method, url, **kwargs) - return asyncio.run(main()) +async def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: + async with httpx.AsyncClient(transport=_FetchTransport()) as client: + return await client.request(method, url, **kwargs) -def test_basic_get_round_trip(fake_pyfetch): - fake_pyfetch.response = FakeFetchResponse( - status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' - ) - response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) +async def test_basic_get_round_trip(): + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' + ) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.status_code == 200 - assert response.json() == {"version": "1.30.0"} - assert response.headers["content-type"] == "application/json" - call = fake_pyfetch.calls[0] - assert call["url"] == "http://h:8080/v1/meta" - assert call["method"] == "GET" + assert response.status_code == 200 + assert response.json() == {"version": "1.30.0"} + assert response.headers["content-type"] == "application/json" + call = fetch.calls[0] + assert call["url"] == "http://h:8080/v1/meta" + assert call["method"] == "GET" -def test_response_has_request_attached_for_raise_for_status(fake_pyfetch): - fake_pyfetch.response = FakeFetchResponse(status=404, body=b"") - response = _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) - with pytest.raises(httpx.HTTPStatusError): - response.raise_for_status() +async def test_response_has_request_attached_for_raise_for_status(): + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=404, body=b"") + response = await _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) + with raises(httpx.HTTPStatusError): + response.raise_for_status() -@pytest.mark.parametrize("status", [204, 404]) -def test_head_response_without_body_yields_empty_content(fake_pyfetch, status): +async def test_head_response_without_body_yields_empty_content(): # data.exists() / tenants.exists() are HEAD requests answered 204/404 with a null # body; the transport must hand httpx an empty response, not fail on the missing body - fake_pyfetch.response = FakeFetchResponse(status=status, body=None) - response = _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) - assert response.status_code == status - assert response.content == b"" - assert "body" not in fake_pyfetch.calls[0] + for status in (204, 404): + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=status, body=None) + response = await _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == status + assert response.content == b"" + assert "body" not in fetch.calls[0] -def test_delete_204_without_body_yields_empty_content(fake_pyfetch): +async def test_delete_204_without_body_yields_empty_content(): # data.delete_by_id() / reference_delete() are answered 204 with a null body - fake_pyfetch.response = FakeFetchResponse(status=204, body=None) - response = _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) - assert response.status_code == 204 - assert response.content == b"" + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=204, body=None) + response = await _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == 204 + assert response.content == b"" -def test_body_less_response_through_async_client(fake_pyfetch): +async def test_body_less_response_through_async_client(): # the full httpx.AsyncClient path (stream wrapping + read) on a body-less response - fake_pyfetch.response = FakeFetchResponse(status=204, body=None) - response = _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") - assert response.status_code == 204 - assert response.content == b"" + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=204, body=None) + response = await _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") + assert response.status_code == 204 + assert response.content == b"" -def test_response_through_async_client_exposes_elapsed_and_content(fake_pyfetch): +async def test_response_through_async_client_exposes_elapsed_and_content(): # the batch-references path reads ``res.elapsed``, which httpx only sets after it # has read/closed a stream-backed response; a pre-loaded body never gets one payload = b'[{"result": {"status": "SUCCESS"}}]' - fake_pyfetch.response = FakeFetchResponse(status=200, body=payload) - response = _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") - assert response.content == payload - assert response.json() == [{"result": {"status": "SUCCESS"}}] - assert response.elapsed.total_seconds() >= 0 + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=200, body=payload) + response = await _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") + assert response.content == payload + assert response.json() == [{"result": {"status": "SUCCESS"}}] + assert response.elapsed.total_seconds() >= 0 -def test_fetch_managed_request_headers_stripped(fake_pyfetch): +async def test_fetch_managed_request_headers_stripped(): request = httpx.Request( "POST", "http://h:8080/v1/objects", @@ -162,79 +176,76 @@ def test_fetch_managed_request_headers_stripped(fake_pyfetch): }, content=b"{}", ) - _handle(request) - sent = fake_pyfetch.calls[0]["headers"] - assert sent["authorization"] == "Bearer k" - assert sent["content-type"] == "application/json" - for managed in ("host", "connection", "accept-encoding", "content-length", "transfer-encoding"): - assert managed not in sent - - -def test_get_without_body_omits_body_kwarg(fake_pyfetch): + with fake_pyfetch() as fetch: + await _handle(request) + sent = fetch.calls[0]["headers"] + assert sent["authorization"] == "Bearer k" + assert sent["content-type"] == "application/json" + for managed in ( + "host", + "connection", + "accept-encoding", + "content-length", + "transfer-encoding", + ): + assert managed not in sent, managed + + +async def test_get_without_body_omits_body_kwarg(): # fetch rejects GET/HEAD requests that carry a body, so the kwarg must be absent - _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) - assert "body" not in fake_pyfetch.calls[0] + with fake_pyfetch() as fetch: + await _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) + assert "body" not in fetch.calls[0] -def test_post_body_passed(fake_pyfetch): - _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) - assert fake_pyfetch.calls[0]["body"] == b'{"query": "x"}' +async def test_post_body_passed(): + with fake_pyfetch() as fetch: + await _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) + assert fetch.calls[0]["body"] == b'{"query": "x"}' -def test_delete_with_body_passed(fake_pyfetch): +async def test_delete_with_body_passed(): # the REST batch-delete path sends DELETE with a JSON body - _handle(httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}')) - assert fake_pyfetch.calls[0]["body"] == b'{"match": {}}' + with fake_pyfetch() as fetch: + await _handle( + httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}') + ) + assert fetch.calls[0]["body"] == b'{"match": {}}' -def test_query_string_preserved_in_url(fake_pyfetch): - _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) - assert fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" +async def test_query_string_preserved_in_url(): + with fake_pyfetch() as fetch: + await _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) + assert fetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" -def test_content_encoding_stripped_from_response(fake_pyfetch): +async def test_content_encoding_stripped_from_response(): # fetch hands back ALREADY-decompressed bytes; if the original content-encoding # header were passed through, httpx.Response would gunzip a second time and raise # DecodingError. content-length is stale for the same reason. - fake_pyfetch.response = FakeFetchResponse( - status=200, - headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, - body=b'{"version": "1.30.0"}', - ) - response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.json() == {"version": "1.30.0"} - assert "content-encoding" not in response.headers - assert response.headers["x-other"] == "kept" + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse( + status=200, + headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, + body=b'{"version": "1.30.0"}', + ) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.json() == {"version": "1.30.0"} + assert "content-encoding" not in response.headers + assert response.headers["x-other"] == "kept" -def test_unreadable_response_headers_tolerated(fake_pyfetch): +async def test_unreadable_response_headers_tolerated(): class BadHeaders: def keys(self): raise TypeError("header shape varies across Pyodide versions") - fake_pyfetch.response = FakeFetchResponse(status=200, body=b"ok") - fake_pyfetch.response.headers = BadHeaders() - response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.status_code == 200 - assert response.content == b"ok" - - -class _AbortSignalRecorder: - def __init__(self): - self.timeouts: List[int] = [] - - def timeout(self, ms: int): - self.timeouts.append(ms) - return f"signal-{ms}" - - -@pytest.fixture -def fake_abort_signal(monkeypatch) -> _AbortSignalRecorder: - recorder = _AbortSignalRecorder() - js_mod = types.ModuleType("js") - js_mod.AbortSignal = recorder # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "js", js_mod) - return recorder + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=200, body=b"ok") + fetch.response.headers = BadHeaders() + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.status_code == 200 + assert response.content == b"ok" def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request: @@ -243,58 +254,68 @@ def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request return request -def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_signal): +async def test_read_timeout_maps_to_abort_signal_ms(): # mirrors what weaviate's AsyncClient puts in extensions: connect/read/write/pool - _handle(_request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0})) - assert fake_abort_signal.timeouts == [30000] - assert fake_pyfetch.calls[0]["signal"] == "signal-30000" + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle( + _request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0}) + ) + assert signals.timeouts == [30000] + assert fetch.calls[0]["signal"] == "signal-30000" -def test_read_none_means_no_deadline_even_with_pool_and_connect_set( - fake_pyfetch, fake_abort_signal -): +async def test_read_none_means_no_deadline_even_with_pool_and_connect_set(): # what the base client hands over for a non-finite request timeout: read=None with the # session pool timeout still set; falling back to pool/connect would abort a long # insert after 5 s - _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5})) - _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) - assert fake_abort_signal.timeouts == [] - assert all("signal" not in c for c in fake_pyfetch.calls) + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle( + _request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5}) + ) + await _handle( + _request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0}) + ) + assert signals.timeouts == [] + assert all("signal" not in c for c in fetch.calls) -def test_read_timeout_alone_sets_the_deadline(fake_pyfetch, fake_abort_signal): - _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) - assert fake_abort_signal.timeouts == [7000] - assert fake_pyfetch.calls[0]["signal"] == "signal-7000" +async def test_read_timeout_alone_sets_the_deadline(): + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) + assert signals.timeouts == [7000] + assert fetch.calls[0]["signal"] == "signal-7000" -def test_no_timeout_extension_sends_no_signal(fake_pyfetch, fake_abort_signal): - _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert fake_abort_signal.timeouts == [] - assert "signal" not in fake_pyfetch.calls[0] +async def test_no_timeout_extension_sends_no_signal(): + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert signals.timeouts == [] + assert "signal" not in fetch.calls[0] -def test_missing_js_module_degrades_to_no_signal(fake_pyfetch): - # off-browser (no js module) the AbortSignal import fails; the request must still go out - assert "js" not in sys.modules - response = _handle( - _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) - ) - assert response.status_code == 200 - assert "signal" not in fake_pyfetch.calls[0] +async def test_missing_js_module_degrades_to_no_signal(): + # without the js bridge the AbortSignal import fails; the request must still go out + with fake_pyfetch() as fetch, missing_js_module(): + response = await _handle( + _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) + ) + assert response.status_code == 200 + assert "signal" not in fetch.calls[0] -def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): +async def test_zero_timeout_means_no_deadline(): # an explicit read=0 must not fall through to the 5s connect timeout, nor become an # immediate AbortSignal.timeout(0) - _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) - assert fake_abort_signal.timeouts == [] - assert "signal" not in fake_pyfetch.calls[0] + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle( + _request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None}) + ) + assert signals.timeouts == [] + assert "signal" not in fetch.calls[0] -@pytest.mark.parametrize( - "timeout,expected_ms", - [ +def test_abort_signal_ms_bounds(): + cases = [ (None, None), (0, None), (-1, None), @@ -305,26 +326,29 @@ def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): (1e8, _MAX_ABORT_SIGNAL_MS), (1e10, _MAX_ABORT_SIGNAL_MS), (1e308, _MAX_ABORT_SIGNAL_MS), # finite, but *1000 overflows: capped, not an error - ], -) -def test_abort_signal_ms_bounds(timeout, expected_ms): - assert _abort_signal_ms(timeout) == expected_ms + ] + for timeout, expected_ms in cases: + assert _abort_signal_ms(timeout) == expected_ms, timeout -def test_infinite_timeout_sends_no_signal(fake_pyfetch, fake_abort_signal): +async def test_infinite_timeout_sends_no_signal(): # an inf read deadline reaching the transport: no signal, not an OverflowError - _handle( - _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) - ) - assert fake_abort_signal.timeouts == [] - assert "signal" not in fake_pyfetch.calls[0] + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle( + _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) + ) + assert signals.timeouts == [] + assert "signal" not in fetch.calls[0] -def test_huge_timeout_is_capped_to_int32_ms(fake_pyfetch, fake_abort_signal): +async def test_huge_timeout_is_capped_to_int32_ms(): # setTimeout delays above 2^31-1 ms overflow and fire at once, aborting the request - _handle(_request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None})) - assert fake_abort_signal.timeouts == [_MAX_ABORT_SIGNAL_MS] - assert fake_pyfetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle( + _request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None}) + ) + assert signals.timeouts == [_MAX_ABORT_SIGNAL_MS] + assert fetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" class RaisingPyfetch: @@ -335,271 +359,123 @@ async def __call__(self, url: str, **kwargs: Any): raise self.exc -def _install_raising_pyfetch(monkeypatch, exc: BaseException) -> None: - pyodide_mod = types.ModuleType("pyodide") - http_mod = types.ModuleType("pyodide.http") - http_mod.pyfetch = RaisingPyfetch(exc) # type: ignore[attr-defined] - pyodide_mod.http = http_mod # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "pyodide", pyodide_mod) - monkeypatch.setitem(sys.modules, "pyodide.http", http_mod) +@contextlib.contextmanager +def raising_pyfetch(exc: BaseException): + with patched(_httpx_fetch, "pyfetch", RaisingPyfetch(exc)): + yield -def test_fetch_failure_maps_to_httpx_connect_error(monkeypatch): +async def test_fetch_failure_maps_to_httpx_connect_error(): # pyodide surfaces JS fetch rejections as OSError; the base client can only classify # httpx exceptions (WeaviateConnectionError etc.), so the shim must translate - _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) - with pytest.raises(httpx.ConnectError, match="Failed to fetch") as excinfo: - _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert isinstance(excinfo.value.__cause__, OSError) + with raising_pyfetch(OSError("TypeError: Failed to fetch")): + with raises(httpx.ConnectError, contains="Failed to fetch") as excinfo: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert isinstance(excinfo.value.__cause__, OSError) -def test_fetch_abort_with_deadline_maps_to_read_timeout(monkeypatch, fake_abort_signal): +async def test_fetch_abort_with_deadline_maps_to_read_timeout(): # AbortSignal.timeout firing surfaces as an OSError subclass mentioning the abort; # with a deadline set this must classify as a timeout, not a connection error - _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) - with pytest.raises(httpx.ReadTimeout, match="signal timed out"): - _handle(_request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None})) + with raising_pyfetch(OSError("AbortError: signal timed out")), fake_abort_signal(): + with raises(httpx.ReadTimeout, contains="signal timed out"): + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) -def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error( - monkeypatch, fake_abort_signal -): +async def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error(): # nearly every weaviate request sets a read deadline; a plain network failure on # such a request must remain a connection error, not become a timeout - _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) - with pytest.raises(httpx.ConnectError, match="Failed to fetch"): - _handle(_request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None})) + with raising_pyfetch(OSError("TypeError: Failed to fetch")), fake_abort_signal(): + with raises(httpx.ConnectError, contains="Failed to fetch"): + await _handle( + _request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None}) + ) -def test_fetch_abort_without_deadline_stays_connect_error(monkeypatch): - # the same message without a deadline set (no js module -> no signal) is not OUR +async def test_fetch_abort_without_deadline_stays_connect_error(): + # the same message without a deadline set (no js bridge -> no signal) is not OUR # timeout, so it must stay a connection error - _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) - assert "js" not in sys.modules - with pytest.raises(httpx.ConnectError): - _handle(_request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None})) + with raising_pyfetch(OSError("AbortError: signal timed out")), missing_js_module(): + with raises(httpx.ConnectError): + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) -def test_empty_oserror_str_keeps_repr_detail(monkeypatch): - _install_raising_pyfetch(monkeypatch, OSError()) - with pytest.raises(httpx.ConnectError) as excinfo: - _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert "OSError" in str(excinfo.value) +async def test_empty_oserror_str_keeps_repr_detail(): + with raising_pyfetch(OSError()): + with raises(httpx.ConnectError) as excinfo: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert "OSError" in str(excinfo.value) -def test_crlf_in_header_value_rejected(fake_pyfetch): +async def test_crlf_in_header_value_rejected(): # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at # send time; this transport bypasses h11 and must keep that defence request = httpx.Request( "GET", "http://h:8080/v1/meta", headers={"x-key": "val\r\nx-injected: evil"} ) - with pytest.raises(httpx.LocalProtocolError): - _handle(request) - assert fake_pyfetch.calls == [] + with fake_pyfetch() as fetch: + with raises(httpx.LocalProtocolError): + await _handle(request) + assert fetch.calls == [] # --------------------------------------------------------------------------- -# Install semantics: these patch httpx.AsyncHTTPTransport globally, so each -# scenario runs in a fresh subprocess (same pattern as test_shim_install.py). +# install semantics, against this interpreter's real installation # --------------------------------------------------------------------------- -_FAKE_PYODIDE_PRELUDE = """ -import sys, types - -class _FakeResponse: - status = 200 - headers = {"content-type": "application/json"} - async def bytes(self): - return b'{"ok": true}' - -CALLS = [] -async def pyfetch(url, **kwargs): - CALLS.append((url, kwargs)) - return _FakeResponse() - -_pyodide = types.ModuleType("pyodide") -_http = types.ModuleType("pyodide.http") -_http.pyfetch = pyfetch -_pyodide.http = _http -sys.modules["pyodide"] = _pyodide -sys.modules["pyodide.http"] = _http -""" - - -def _run(body: str, prelude: str = "") -> subprocess.CompletedProcess: - script = f"import sys\nsys.path.insert(0, {_SRC!r})\n" + prelude + textwrap.dedent(body) - return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) +def test_bootstrap_installed_fetch_transport(): + # This interpreter imported weaviate_client_web at the top of this file, so the real + # bootstrap ran — even though Pyodide's bundled httpx carries its own jsfetch + # transport, the package's transport must be the active one. + assert weaviate_client_web.is_fetch_transport_installed() + assert ( + getattr(httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False) + is True + ) -def test_force_install_routes_async_client_through_pyfetch(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import asyncio, httpx - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - - install_fetch_transport(force=True) - assert is_fetch_transport_installed() - async def main(): - async with httpx.AsyncClient() as client: - return await client.get("http://h:8080/v1/meta") - - resp = asyncio.run(main()) - assert resp.status_code == 200, resp.status_code - assert resp.json() == {"ok": True} - assert CALLS and CALLS[0][0] == "http://h:8080/v1/meta" - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_install_without_force_is_noop_off_emscripten(): - result = _run( - """ - import sys - assert sys.platform != "emscripten" - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - install_fetch_transport() - assert not is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is before - print("OK") - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_force_install_is_idempotent(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - from weaviate_client_web import install_fetch_transport - install_fetch_transport(force=True) - patched = httpx.AsyncHTTPTransport.handle_async_request - install_fetch_transport(force=True) - assert httpx.AsyncHTTPTransport.handle_async_request is patched - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout +def test_install_fetch_transport_is_idempotent(): + patched_method = httpx.AsyncHTTPTransport.handle_async_request + weaviate_client_web.install_fetch_transport() + assert httpx.AsyncHTTPTransport.handle_async_request is patched_method def test_sync_transport_left_untouched(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - sync_before = httpx.HTTPTransport.handle_request - from weaviate_client_web import install_fetch_transport - install_fetch_transport(force=True) - assert httpx.HTTPTransport.handle_request is sync_before - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + assert not getattr(httpx.HTTPTransport.handle_request, "__weaviate_fetch_shim__", False) def test_uninstall_restores_original_transport(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import ( - install_fetch_transport, - is_fetch_transport_installed, - uninstall_fetch_transport, - ) - uninstall_fetch_transport() # no-op when not installed - install_fetch_transport(force=True) - assert is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is not before - uninstall_fetch_transport() - assert not is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is before - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_patched_method_carries_sentinel(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - from weaviate_client_web import install_fetch_transport + patched_method = httpx.AsyncHTTPTransport.handle_async_request + try: + weaviate_client_web.uninstall_fetch_transport() + assert not weaviate_client_web.is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is not patched_method assert not getattr( httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False ) - install_fetch_transport(force=True) - assert getattr( - httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False - ) is True - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_force_install_without_pyodide_fails_fast(): - # without a pyodide module the install must raise immediately, not let every later - # request die with a lazy ModuleNotFoundError - result = _run( - """ - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - try: - install_fetch_transport(force=True) - except ModuleNotFoundError: - assert not is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is before - print("OK") - else: - raise AssertionError("expected install to fail fast without pyodide") - """ + weaviate_client_web.uninstall_fetch_transport() # no-op when not installed + finally: + weaviate_client_web.install_fetch_transport() + assert weaviate_client_web.is_fetch_transport_installed() + assert ( + getattr(httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False) + is True ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_emscripten_installs_even_when_platform_httpx_has_jsfetch(): - # Pyodide's bundled httpx ships a jsfetch transport that crashes on body-less - # responses (HEAD / 204); the shim must take over regardless of the httpx build - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import importlib.machinery, sys, types - - sys.platform = "emscripten" - fake = types.ModuleType("httpx._transports.jsfetch") - fake.__spec__ = importlib.machinery.ModuleSpec( - "httpx._transports.jsfetch", loader=None + + +async def test_installed_transport_routes_async_client_through_pyfetch(): + # the globally installed transport (no custom transport argument) must reach pyfetch + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"ok": true}' ) - sys.modules["httpx._transports.jsfetch"] = fake - - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - install_fetch_transport() # no force: the platform alone must trigger it - assert is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is not before - assert getattr( - httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False - ) is True - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + async with httpx.AsyncClient() as client: + response = await client.get("http://h:8080/v1/meta") + assert response.status_code == 200 + assert response.json() == {"ok": True} + assert fetch.calls and fetch.calls[0]["url"] == "http://h:8080/v1/meta" diff --git a/packages/web/tests/test_shim_install.py b/packages/web/tests/test_shim_install.py index cdcbed56d..d5dede428 100644 --- a/packages/web/tests/test_shim_install.py +++ b/packages/web/tests/test_shim_install.py @@ -1,123 +1,83 @@ -"""Shim/import tests. +"""grpc shim tests, against this interpreter's real installation. -Installing the shim replaces ``sys.modules['grpc']`` process-wide, so each scenario runs -in a fresh subprocess to avoid clobbering the real ``grpc`` used by the rest of the suite. +Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``: importing ``weaviate_client_web`` +bootstrapped the shim, so ``sys.modules['grpc']`` here IS the shim. Bootstrap scenarios +that need a clean import state (a bare ``import weaviate``, a missing or broken +companion) live in ``units.mjs``, one fresh interpreter each. """ -import pathlib -import subprocess -import sys -import textwrap +import struct -_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") +from harness import raises +import weaviate_client_web +from weaviate_client_web import GrpcWebChannel, set_sender +from weaviate_client_web._sender import pyfetch_sender +from weaviate_client_web._shim import FAKE_GRPC_VERSION -def _run(body: str) -> subprocess.CompletedProcess: - script = f"import sys\nsys.path.insert(0, {_SRC!r})\n" + textwrap.dedent(body) - return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + +def _frame(payload: bytes, flag: int = 0x00) -> bytes: + return struct.pack(">BI", flag, len(payload)) + payload def test_import_weaviate_under_shim(): - result = _run( - """ - import weaviate_client_web - assert weaviate_client_web.install(force=True) is True - assert weaviate_client_web.is_installed() - - import grpc - assert getattr(grpc, "__weaviate_client_web_shim__", False) is True - assert grpc.__version__ == "1.72.1" - assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False - from grpc.aio._typing import ChannelArgumentType # noqa: F401 - - import weaviate # must not raise even though grpcio is shimmed - from weaviate.proto.v1 import weaviate_pb2_grpc - from weaviate_client_web import GrpcWebChannel + import grpc - ch = GrpcWebChannel("localhost:50051", secure=False) - stub = weaviate_pb2_grpc.WeaviateStub(ch) - assert stub.Search is not None - assert stub.BatchObjects is not None - assert stub.BatchDelete is not None - assert isinstance(ch, grpc.aio.Channel) - print("OK") - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + assert weaviate_client_web.is_installed() + assert getattr(grpc, "__weaviate_client_web_shim__", False) is True + assert grpc.__version__ == FAKE_GRPC_VERSION + assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False # type: ignore[attr-defined] + from grpc.aio._typing import ChannelArgumentType # noqa: F401 + + import weaviate # noqa: F401 # must not raise even though grpcio is shimmed + from weaviate.proto.v1 import weaviate_pb2_grpc + + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + assert stub.Search is not None + assert stub.BatchObjects is not None + assert stub.BatchDelete is not None + assert isinstance(ch, grpc.aio.Channel) def test_sync_channel_factory_raises_async_only(): - result = _run( - """ - import weaviate_client_web - weaviate_client_web.install(force=True) - import grpc - try: - grpc.insecure_channel("localhost:50051") - except RuntimeError as exc: - assert "async" in str(exc).lower() - print("OK") - else: - raise AssertionError("expected sync channel factory to raise") - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_real_proto_unary_round_trip_under_shim(): - result = _run( - """ - import asyncio - import struct - import weaviate_client_web - weaviate_client_web.install(force=True) - - import weaviate # noqa: F401 - from weaviate.proto.v1 import tenants_pb2, weaviate_pb2_grpc - - reply = tenants_pb2.TenantsGetReply() - payload = reply.SerializeToString() - - def frame(p, flag=0x00): - return struct.pack(">BI", flag, len(p)) + p - - body = frame(payload) + frame(b"grpc-status:0\\r\\n", 0x80) - - async def sender(url, headers, body_in, timeout): - assert headers["authorization"] == "Bearer k" - assert url.endswith("/weaviate.v1.Weaviate/TenantsGet") - return 200, {}, body - - weaviate_client_web.set_sender(sender) - from weaviate_client_web import GrpcWebChannel - ch = GrpcWebChannel("localhost:50051", secure=False) - stub = weaviate_pb2_grpc.WeaviateStub(ch) + import grpc - async def main(): - res = await stub.TenantsGet( - tenants_pb2.TenantsGetRequest(), - metadata=[("authorization", "Bearer k")], - timeout=5, - ) - assert isinstance(res, tenants_pb2.TenantsGetReply) - print("OK") + with raises(RuntimeError) as excinfo: + grpc.insecure_channel("localhost:50051") + assert "async" in str(excinfo.value).lower() - asyncio.run(main()) - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + +async def test_real_proto_unary_round_trip_under_shim(): + from weaviate.proto.v1 import tenants_pb2, weaviate_pb2_grpc + + reply = tenants_pb2.TenantsGetReply() + payload = reply.SerializeToString() + body = _frame(payload) + _frame(b"grpc-status:0\r\n", 0x80) + + async def sender(url, headers, body_in, timeout): + assert headers["authorization"] == "Bearer k" + assert url.endswith("/weaviate.v1.Weaviate/TenantsGet") + return 200, {}, body + + set_sender(sender) + try: + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + res = await stub.TenantsGet( + tenants_pb2.TenantsGetRequest(), + metadata=[("authorization", "Bearer k")], + timeout=5, + ) + assert isinstance(res, tenants_pb2.TenantsGetReply) + finally: + set_sender(pyfetch_sender) def test_fake_grpc_version_matches_base_fallback(): - # In-process on purpose: nothing here installs the shim, we only compare the two - # copies of the pinned version. The shim advertises FAKE_GRPC_VERSION as - # grpc.__version__ and the base package falls back to _GRPCIO_FALLBACK_VERSION - # under Emscripten — the vendored stubs' version gates see both, so they must - # never drift apart. + # The shim advertises FAKE_GRPC_VERSION as grpc.__version__ and the base package + # falls back to _GRPCIO_FALLBACK_VERSION under Emscripten — the vendored stubs' + # version gates see both, so they must never drift apart. from weaviate.proto.v1 import _GRPCIO_FALLBACK_VERSION - from weaviate_client_web._shim import FAKE_GRPC_VERSION assert FAKE_GRPC_VERSION == _GRPCIO_FALLBACK_VERSION diff --git a/packages/web/tests/test_single_import.py b/packages/web/tests/test_single_import.py deleted file mode 100644 index 562f31e0a..000000000 --- a/packages/web/tests/test_single_import.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Tests for the single-import hook in the base client (``weaviate/__init__.py``). - -The hook fires on ``sys.platform == "emscripten"`` and (via the companion's bootstrap) -replaces ``sys.modules['grpc']`` process-wide, so each scenario runs in a fresh -subprocess with the platform faked before ``import weaviate`` — the same pattern as -test_shim_install.py / test_httpx_fetch.py's install tests. -""" - -import pathlib -import subprocess -import sys -import textwrap - -_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") -_REPO_ROOT = str(pathlib.Path(__file__).resolve().parents[3]) - -# CPython derives the _sysconfigdata module name from sys.platform on first use, so a -# faked platform breaks any later sysconfig lookup (pydantic imports zoneinfo, which -# calls sysconfig.get_config_var). Prime the cache before faking. -_PRIME_SYSCONFIG = """ -import sysconfig - -sysconfig.get_config_vars() -""" - -# The companion's bootstrap installs the fetch transport under Emscripten and fails fast -# if pyodide.http cannot be imported, so a faked platform needs a stand-in module. -_FAKE_PYODIDE = """ -import types - -_pyodide = types.ModuleType("pyodide") -_http = types.ModuleType("pyodide.http") -_http.pyfetch = None -_pyodide.http = _http -sys.modules["pyodide"] = _pyodide -sys.modules["pyodide.http"] = _http -""" - - -def _run( - body: str, *, prelude: str = "", path_entry: str = _SRC, no_site: bool = False -) -> subprocess.CompletedProcess: - # -I -S: skip site-packages entirely (plain -I still processes the venv's .pth - # files), so nothing pip-installed is importable — only stdlib plus `path_entry`. - interp = [sys.executable, "-I", "-S"] if no_site else [sys.executable] - script = f"import sys\nsys.path.insert(0, {path_entry!r})\n" + prelude + textwrap.dedent(body) - return subprocess.run([*interp, "-c", script], capture_output=True, text=True) - - -def test_bare_import_weaviate_installs_shim_under_emscripten(): - result = _run( - prelude=_PRIME_SYSCONFIG + _FAKE_PYODIDE, - body=""" - sys.platform = "emscripten" - - import weaviate # the ONLY weaviate-side import: must bootstrap the companion - - assert "weaviate_client_web" in sys.modules, "hook did not import the companion" - import weaviate_client_web - assert weaviate_client_web.is_installed() - assert weaviate_client_web.is_fetch_transport_installed() - import grpc - assert getattr(grpc, "__weaviate_client_web_shim__", False) is True - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_bare_import_without_companion_raises_clear_import_error(): - # No site-packages, so neither weaviate_client_web nor grpcio is importable; the repo - # root goes on sys.path so the weaviate package itself is still found. - result = _run( - """ - sys.platform = "emscripten" - try: - import weaviate - except ImportError as e: - assert "weaviate-client-web" in str(e), str(e) - assert "WebAssembly/Pyodide" in str(e), str(e) - print("OK") - else: - raise AssertionError("expected ImportError without the companion") - """, - path_entry=_REPO_ROOT, - no_site=True, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_bare_import_with_grpc_present_falls_through_silently(): - # Companion blocked but a real grpc IS importable (grpcio in the dev env): the hook - # must fall through and leave the normal import path untouched. - result = _run( - prelude=_PRIME_SYSCONFIG, - body=""" - sys.platform = "emscripten" - sys.modules["weaviate_client_web"] = None # makes its import raise ImportError - - import weaviate - import grpc - - assert not getattr(grpc, "__weaviate_client_web_shim__", False) - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_bare_import_with_broken_companion_surfaces_its_own_error(tmp_path): - # An INSTALLED companion whose import fails (here: a missing dependency of its own) - # must raise that error, not the "install weaviate-client-web" hint — the hint would - # send the user to reinstall a package that is already there. - fake_pkg = tmp_path / "weaviate_client_web" - fake_pkg.mkdir() - (fake_pkg / "__init__.py").write_text( - "raise ModuleNotFoundError(\"No module named 'anyio'\", name='anyio')\n" - ) - result = _run( - prelude=_PRIME_SYSCONFIG, - body=""" - sys.platform = "emscripten" - try: - import weaviate - except ImportError as e: - assert e.name == "anyio", (e.name, str(e)) - assert "anyio" in str(e), str(e) - assert "weaviate-client-web" not in str(e), str(e) - print("OK") - else: - raise AssertionError("expected the companion's own ImportError to surface") - """, - path_entry=str(tmp_path), - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index e1a43492b..9bb118b97 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -1,8 +1,8 @@ -"""In-process tests for the grpc-web channel/multicallable. +"""grpc-web channel / multicallable tests. Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``. -These exercise the transport classes directly (they import their grpc base classes from -``weaviate_client_web._shim``, not from ``sys.modules['grpc']``), so no shim install is -needed and the real ``grpc`` in the dev environment is left untouched. +These exercise the transport classes directly through fake senders — no network, no +running Weaviate. Tests are ``async def`` awaited on Pyodide's event loop by +``runner.py`` (``asyncio.run()`` cannot be used inside Pyodide). """ import asyncio @@ -10,15 +10,17 @@ import sys from typing import Dict, List, Optional, Tuple -import pytest +from harness import patched, raises -from weaviate_client_web._channel import ( - GrpcWebChannel, - _body_excerpt, - _encode_timeout, - set_sender, +from weaviate_client_web import GrpcWebChannel, set_sender +from weaviate_client_web._channel import _body_excerpt, _encode_timeout +from weaviate_client_web._sender import pyfetch_sender +from weaviate_client_web._shim import ( + AioChannel, + AioRpcError, + StatusCode, + _aio_insecure_channel, ) -from weaviate_client_web._shim import AioChannel, AioRpcError, StatusCode def _frame(payload: bytes, flag: int = 0x00) -> bytes: @@ -52,7 +54,7 @@ def test_grpcwebchannel_is_grpc_aio_channel(): assert isinstance(_channel(FakeSender()), AioChannel) -def test_unary_success_round_trip(): +async def test_unary_success_round_trip(): sender = FakeSender(body=_ok_response(b"reply-bytes")) channel = _channel(sender) mc = channel.unary_unary( @@ -62,7 +64,7 @@ def test_unary_success_round_trip(): _registered_method=True, ) - result = asyncio.run(mc(b"request-bytes", metadata=[("authorization", "Bearer k")], timeout=5)) + result = await mc(b"request-bytes", metadata=[("authorization", "Bearer k")], timeout=5) assert result == b"reply-bytes" url, headers, body, timeout = sender.calls[0] @@ -74,50 +76,50 @@ def test_unary_success_round_trip(): assert timeout == 5 -def test_secure_channel_uses_https(): +async def test_secure_channel_uses_https(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender, secure=True) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q")) + await mc(b"q") assert sender.calls[0][0].startswith("https://example.com:443/") -def test_health_call_without_metadata(): +async def test_health_call_without_metadata(): sender = FakeSender(body=_ok_response(b"pong")) channel = _channel(sender) mc = channel.unary_unary("/grpc.health.v1.Health/Check", lambda x: x, lambda b: b) # mirrors the health check in connect/v4.py — request + timeout, no metadata - assert asyncio.run(mc(b"ping", timeout=2)) == b"pong" + assert await mc(b"ping", timeout=2) == b"pong" -def test_error_trailer_raises_aiorpcerror(): +async def test_error_trailer_raises_aiorpcerror(): body = _frame(b"grpc-status:7\r\ngrpc-message:nope\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.PERMISSION_DENIED assert excinfo.value.code().name == "PERMISSION_DENIED" assert excinfo.value.details() == "nope" -def test_percent_encoded_grpc_message_decoded(): +async def test_percent_encoded_grpc_message_decoded(): body = _frame(b"grpc-status:5\r\ngrpc-message:not%20found\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.details() == "not found" -def test_trailers_only_status_in_http_headers(): +async def test_trailers_only_status_in_http_headers(): channel = _channel( FakeSender(status=200, headers={"grpc-status": "16", "grpc-message": "auth"}, body=b"") ) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAUTHENTICATED @@ -149,19 +151,19 @@ def test_trailers_only_status_in_http_headers(): ) -def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/Check"): +async def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/Check"): """Run one request against a canned HTTP response and return the AioRpcError.""" channel = _channel(FakeSender(status=status, headers=headers or {}, body=body)) mc = channel.unary_unary(path, lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") return excinfo.value -def test_weaviate_404_json_names_both_candidate_causes(): +async def test_weaviate_404_json_names_both_candidate_causes(): # A 404 means EITHER the server predates the native /v1/grpc-web endpoint OR the # configured path prefix is wrong. The channel cannot tell which, so it must say both. - err = _details_of(404, WEAVIATE_404_JSON, {"content-type": "application/json"}) + err = await _details_of(404, WEAVIATE_404_JSON, {"content-type": "application/json"}) details = err.details() assert err.code() is StatusCode.UNIMPLEMENTED @@ -174,44 +176,46 @@ def test_weaviate_404_json_names_both_candidate_causes(): assert "malformed grpc-web response" not in details -def test_nginx_502_maps_to_unavailable_so_the_client_retries(): +async def test_nginx_502_maps_to_unavailable_so_the_client_retries(): # weaviate/retry.py retries UNAVAILABLE and nothing else; a gateway error arriving # as INTERNAL is silently un-retried, which is the regression this pins. - err = _details_of(502, NGINX_502_HTML) + err = await _details_of(502, NGINX_502_HTML) assert err.code() is StatusCode.UNAVAILABLE assert err.details().startswith("HTTP 502 ") assert "502 Bad Gateway" in err.details() -@pytest.mark.parametrize("status", [503, 504]) -def test_gateway_errors_are_unavailable(status): - err = _details_of(status, b"upstream down") - assert err.code() is StatusCode.UNAVAILABLE +async def test_gateway_errors_are_unavailable(): + for status in (503, 504): + err = await _details_of(status, b"upstream down") + assert err.code() is StatusCode.UNAVAILABLE, status -def test_nginx_404_html_is_reported_as_an_http_404(): - err = _details_of(404, NGINX_404_HTML) +async def test_nginx_404_html_is_reported_as_an_http_404(): + err = await _details_of(404, NGINX_404_HTML) assert err.code() is StatusCode.UNIMPLEMENTED assert err.details().startswith("HTTP 404 ") assert "404 Not Found" in err.details() assert "malformed grpc-web response" not in err.details() -def test_405_names_the_wrong_prefix(): +async def test_405_names_the_wrong_prefix(): # a 405 can only come from an existing HTTP route (measured live: a prefix pointing # at /v1/objects answers "method POST is not allowed"), so the prefix is wrong - err = _details_of(405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}') + err = await _details_of( + 405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}' + ) assert err.details().startswith("HTTP 405 ") assert "path prefix" in err.details() assert "/v1/grpc-web" in err.details() assert "method POST is not allowed" in err.details() -def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): +async def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): # a valid frame header whose payload was cut short: the endpoint IS grpc-web, so the # SPA / path-prefix hint would send the user the wrong way body = _ok_response(b"reply-bytes")[:-6] - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.INTERNAL assert "truncated" in err.details() assert "cut short" in err.details() @@ -219,43 +223,43 @@ def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): assert "path prefix" not in err.details() -def test_spa_html_body_is_not_reported_as_truncated(): +async def test_spa_html_body_is_not_reported_as_truncated(): # text bodies decode to an unknown flag byte and a garbage length; they must read as # "not grpc-web framing", never as a truncated grpc-web body - err = _details_of(200, SPA_INDEX_HTML) + err = await _details_of(200, SPA_INDEX_HTML) assert "truncated" not in err.details() assert "not grpc-web framing" in err.details() -def test_message_frame_after_trailer_is_internal(): +async def test_message_frame_after_trailer_is_internal(): body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.INTERNAL assert "malformed grpc-web response" in err.details() assert "after the trailer" in err.details() assert "single-page-app" not in err.details() -def test_multiple_message_frames_in_unary_response_is_internal(): +async def test_multiple_message_frames_in_unary_response_is_internal(): # a unary RPC has exactly one message; silently taking the first would hide a proxy # or server that streams several body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:0\r\n", 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.INTERNAL assert "2 message frames" in err.details() -def test_error_status_wins_over_multiple_message_frames(): +async def test_error_status_wins_over_multiple_message_frames(): body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:5\r\ngrpc-message:gone\r\n", 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.NOT_FOUND assert err.details() == "gone" -def test_spa_fallback_html_200_is_distinguishable_from_a_404(): +async def test_spa_fallback_html_200_is_distinguishable_from_a_404(): # An HTTP 200 serving index.html is the other half of a wrong path prefix: the app's # catch-all route answers instead of Weaviate. It must not read as malformed framing. - err = _details_of(200, SPA_INDEX_HTML) + err = await _details_of(200, SPA_INDEX_HTML) details = err.details() assert details.startswith("HTTP 200 ") @@ -263,158 +267,157 @@ def test_spa_fallback_html_200_is_distinguishable_from_a_404(): assert "single-page-app" in details # names the actual cause assert "malformed grpc-web response" not in details # distinguishable from the 404 case, not the same generic message - assert details != _details_of(404, NGINX_404_HTML).details() + assert details != (await _details_of(404, NGINX_404_HTML)).details() -def test_401_json_body_maps_to_unauthenticated(): - err = _details_of(401, b'{"error":[{"message":"anonymous access not enabled"}]}') +async def test_401_json_body_maps_to_unauthenticated(): + err = await _details_of(401, b'{"error":[{"message":"anonymous access not enabled"}]}') assert err.code() is StatusCode.UNAUTHENTICATED assert err.details().startswith("HTTP 401 ") assert "anonymous access not enabled" in err.details() -def test_403_error_body_reaches_details(): +async def test_403_error_body_reaches_details(): # regression: the response body is the most actionable part of the error and must # survive into details() rather than being parsed as frames and discarded - err = _details_of(403, b'{"code":403,"message":"forbidden: rbac denied"}') + err = await _details_of(403, b'{"code":403,"message":"forbidden: rbac denied"}') assert err.code() is StatusCode.PERMISSION_DENIED assert "forbidden: rbac denied" in err.details() -def test_error_body_excerpt_is_capped(): - err = _details_of(500, b"E" * 5000) +async def test_error_body_excerpt_is_capped(): + err = await _details_of(500, b"E" * 5000) details = err.details() assert "EEEE" in details assert details.endswith("...") assert len(details) < 600 # the 5000-byte body is excerpted, not pasted in -def test_binary_error_body_does_not_break_the_error(): +async def test_binary_error_body_does_not_break_the_error(): # a proxy answering with a binary payload must not raise UnicodeDecodeError while # the error message is being built - err = _details_of(502, b"\xff\xfe\x00\x01\x02") + err = await _details_of(502, b"\xff\xfe\x00\x01\x02") assert err.code() is StatusCode.UNAVAILABLE assert err.details().startswith("HTTP 502 ") -def test_non_200_with_valid_grpc_web_trailers_still_uses_grpc_status(): +async def test_non_200_with_valid_grpc_web_trailers_still_uses_grpc_status(): # guard on the fix's shape: the HTTP status must not shadow a real grpc-status that # a proxy shipped alongside a non-200 - err = _details_of(500, _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80)) + err = await _details_of(500, _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80)) assert err.code() is StatusCode.PERMISSION_DENIED assert err.details() == "denied" -def test_non_ascii_grpc_message_preserves_the_status(): +async def test_non_ascii_grpc_message_preserves_the_status(): # a trailer carrying raw UTF-8 (an un-percent-encoded proxy, or an error quoting a # collection name) must not degrade to INTERNAL and lose grpc-status body = _frame("grpc-status:5\r\ngrpc-message:collection Café not found\r\n".encode(), 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.NOT_FOUND assert "Caf" in err.details() -def test_invalid_utf8_grpc_message_preserves_the_status(): +async def test_invalid_utf8_grpc_message_preserves_the_status(): # latin-1 bytes are not valid UTF-8; the status must still survive body = _frame(b"grpc-status:9\r\ngrpc-message:tenant caf\xe9 is COLD\r\n", 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.FAILED_PRECONDITION assert "tenant caf" in err.details() -def test_binary_metadata_base64_encoded(): +async def test_binary_metadata_base64_encoded(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q", metadata=[("trace-bin", b"\x00\x01\x02")])) + await mc(b"q", metadata=[("trace-bin", b"\x00\x01\x02")]) assert sender.calls[0][1]["trace-bin"] == "AAEC" def test_stream_stream_raises_clear_error(): channel = _channel(FakeSender()) mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) - with pytest.raises(RuntimeError) as excinfo: + with raises(RuntimeError, contains="not supported over grpc-web"): mc(request_iterator=iter([]), timeout=5, metadata=None) - assert "not supported over grpc-web" in str(excinfo.value) -def test_timeout_maps_to_deadline_exceeded(): +async def test_timeout_maps_to_deadline_exceeded(): async def slow_sender(url, headers, body, timeout): await asyncio.sleep(0.5) return 200, {}, _ok_response(b"x") channel = GrpcWebChannel("h:1", secure=False, sender=slow_sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q", timeout=0.01)) + with raises(AioRpcError) as excinfo: + await mc(b"q", timeout=0.01) assert excinfo.value.code() is StatusCode.DEADLINE_EXCEEDED -def test_transport_exception_maps_to_unavailable(): +async def test_transport_exception_maps_to_unavailable(): async def boom(url, headers, body, timeout): raise ConnectionError("connection refused") channel = GrpcWebChannel("h:1", secure=False, sender=boom) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAVAILABLE assert "ConnectionError: connection refused" in str(excinfo.value.details()) -def test_transport_exception_with_empty_str_keeps_type(): +async def test_transport_exception_with_empty_str_keeps_type(): # httpx transport errors commonly stringify to '' — the detail must still name them async def boom(url, headers, body, timeout): raise ConnectionError() channel = GrpcWebChannel("h:1", secure=False, sender=boom) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert "ConnectionError" in str(excinfo.value.details()) -def test_empty_ok_response_hints_at_cors_expose_headers(): +async def test_empty_ok_response_hints_at_cors_expose_headers(): # HTTP 200, empty body, no grpc-status anywhere: the shape of a trailers-only error # whose grpc-status/grpc-message headers were stripped by CORS channel = _channel(FakeSender(status=200, headers={}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "Access-Control-Expose-Headers" in str(excinfo.value.details()) -def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): +async def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): # when grpc-status WAS visible (status 0, no frames), it is a malformed response, # not a CORS problem — the hint must not appear channel = _channel(FakeSender(status=200, headers={"grpc-status": "0"}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "Access-Control-Expose-Headers" not in str(excinfo.value.details()) -def test_message_frame_without_grpc_status_is_internal_not_success(): +async def test_message_frame_without_grpc_status_is_internal_not_success(): # HTTP 200 with a valid message frame but no grpc-status anywhere (e.g. a proxy # dropped the trailer frame) must be an error, never a fabricated success channel = _channel(FakeSender(status=200, headers={}, body=_frame(b"reply-bytes"))) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "missing grpc-status" in str(excinfo.value.details()) -def test_message_frame_with_grpc_status_header_still_succeeds(): +async def test_message_frame_with_grpc_status_header_still_succeeds(): # trailers-only-in-headers responses (grpc-status as an HTTP header, no trailer # frame) remain valid per the grpc-web contract channel = _channel( FakeSender(status=200, headers={"grpc-status": "0"}, body=_frame(b"reply-bytes")) ) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - assert asyncio.run(mc(b"q")) == b"reply-bytes" + assert await mc(b"q") == b"reply-bytes" def test_stream_stream_error_recommends_insert_many_only(): @@ -422,37 +425,37 @@ def test_stream_stream_error_recommends_insert_many_only(): # only one supported under WASM), so the error must not recommend them channel = _channel(FakeSender()) mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) - with pytest.raises(RuntimeError) as excinfo: + with raises(RuntimeError) as excinfo: mc(request_iterator=iter([]), timeout=5, metadata=None) assert "insert_many" in str(excinfo.value) for sync_only in ("dynamic", "fixed_size", "rate_limit"): assert sync_only not in str(excinfo.value) -def test_malformed_frame_maps_to_internal(): +async def test_malformed_frame_maps_to_internal(): # A 3-byte body cannot contain even a 5-byte frame header -> framing ValueError. channel = _channel(FakeSender(body=b"\x00\x00\x00")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL -def test_malformed_grpc_status_maps_to_internal(): +async def test_malformed_grpc_status_maps_to_internal(): body = _frame(b"grpc-status:notanint\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL -def test_grpc_timeout_header_rounds_up(): +async def test_grpc_timeout_header_rounds_up(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) # 123.4ms must round UP to 124ms (never advertise a shorter deadline than requested). - asyncio.run(mc(b"q", timeout=0.1234)) + await mc(b"q", timeout=0.1234) assert sender.calls[0][1]["grpc-timeout"] == "124m" @@ -462,9 +465,8 @@ def test_body_excerpt_empty_and_non_printable(): assert _body_excerpt(b"ok\x00\x01") == "ok" -@pytest.mark.parametrize( - "seconds,expected", - [ +def test_encode_timeout_stays_within_eight_digits(): + cases = [ (None, None), (float("inf"), None), (float("nan"), None), @@ -479,123 +481,118 @@ def test_body_excerpt_empty_and_non_printable(): (1e10, None), # would need hours, which transcoders reject above 8H: no deadline (1e15, None), (1e308, None), # finite, but *1000 overflows to infinity: must not raise - ], -) -def test_encode_timeout_stays_within_eight_digits(seconds, expected): - encoded = _encode_timeout(seconds) - assert encoded == expected - if encoded is not None: - assert len(encoded) <= 9 # 8 digits + unit - assert not encoded.endswith("H") + ] + for seconds, expected in cases: + encoded = _encode_timeout(seconds) + assert encoded == expected, (seconds, encoded, expected) + if encoded is not None: + assert len(encoded) <= 9, encoded # 8 digits + unit + assert not encoded.endswith("H"), encoded -def test_infinite_timeout_sends_no_deadline(): +async def test_infinite_timeout_sends_no_deadline(): # Timeout(query=inf): neither a grpc-timeout header nor a client-side wait sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - assert asyncio.run(mc(b"q", timeout=float("inf"))) == b"x" + assert await mc(b"q", timeout=float("inf")) == b"x" _, headers, _, timeout = sender.calls[0] assert "grpc-timeout" not in headers assert timeout is None -def test_huge_timeout_uses_minutes_then_no_deadline(): +async def test_huge_timeout_uses_minutes_then_no_deadline(): # 1e8 s in milliseconds is 12 digits; the server rejects more than 8 ("timeout is # too long", HTTP 400) and transcoders reject hour values above 8H, so past the # minute range the request carries no deadline at all sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q", timeout=1e8)) - asyncio.run(mc(b"q", timeout=1e9)) - asyncio.run(mc(b"q", timeout=1e10)) + await mc(b"q", timeout=1e8) + await mc(b"q", timeout=1e9) + await mc(b"q", timeout=1e10) assert sender.calls[0][1]["grpc-timeout"] == "1666667M" assert sender.calls[1][1]["grpc-timeout"] == "16666667M" assert "grpc-timeout" not in sender.calls[2][1] assert sender.calls[2][3] is None # no client-side wait either -@pytest.mark.parametrize("bad", ["val\r\nx-injected: evil", "val\nx", "v\0"]) -def test_crlf_in_metadata_rejected(bad): +async def test_crlf_in_metadata_rejected(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(ValueError, match="Illegal character"): - asyncio.run(mc(b"q", metadata=[("x-key", bad)])) - with pytest.raises(ValueError, match="Illegal character"): - asyncio.run(mc(b"q", metadata=[("x-key\r\n", "v")])) + for bad in ("val\r\nx-injected: evil", "val\nx", "v\0"): + with raises(ValueError, contains="Illegal character"): + await mc(b"q", metadata=[("x-key", bad)]) + with raises(ValueError, contains="Illegal character"): + await mc(b"q", metadata=[("x-key\r\n", "v")]) assert sender.calls == [] -def _unavailable_details(monkeypatch, path_prefix, platform): +async def _unavailable_details(path_prefix, platform): async def boom(url, headers, body, timeout): raise ConnectionError("Failed to fetch") - monkeypatch.setattr(sys, "platform", platform) - channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) - mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with patched(sys, "platform", platform): + channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAVAILABLE return excinfo.value.details() -def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix(monkeypatch): +async def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix(): # the connect helpers always set the prefix under Emscripten, so a prefix-less channel # here means hand-built ConnectionParams; the error must say what to do instead - details = _unavailable_details(monkeypatch, path_prefix="", platform="emscripten") + details = await _unavailable_details(path_prefix="", platform="emscripten") assert "grpc_path_prefix='/v1/grpc-web'" in details assert "1.38.3" in details assert "connect helpers" in details -def test_unavailable_with_path_prefix_has_no_prefix_hint(monkeypatch): - details = _unavailable_details(monkeypatch, path_prefix="/v1/grpc-web", platform="emscripten") +async def test_unavailable_with_path_prefix_has_no_prefix_hint(): + details = await _unavailable_details(path_prefix="/v1/grpc-web", platform="emscripten") assert "no grpc_path_prefix" not in details -def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(monkeypatch): - # on CPython an empty prefix against a transcoder is the normal configuration - details = _unavailable_details(monkeypatch, path_prefix="", platform="linux") +async def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(): + # off Emscripten an empty prefix against a transcoder is the normal configuration + details = await _unavailable_details(path_prefix="", platform="linux") assert "no grpc_path_prefix" not in details -def test_close_is_awaitable_noop(): +async def test_close_is_awaitable_noop(): channel = _channel(FakeSender()) - assert asyncio.run(channel.close()) is None + assert await channel.close() is None -def test_path_prefix_prepended_to_url(): +async def test_path_prefix_prepended_to_url(): sender = FakeSender(body=_ok_response(b"r")) channel = GrpcWebChannel( "example.com:8090", secure=False, sender=sender, path_prefix="/grpc-web" ) mc = channel.unary_unary("/weaviate.v1.Weaviate/Search", lambda x: x, lambda b: b) - asyncio.run(mc(b"q")) + await mc(b"q") assert sender.calls[0][0] == "http://example.com:8090/grpc-web/weaviate.v1.Weaviate/Search" -@pytest.mark.parametrize( - "raw,expected_url", - [ +async def test_path_prefix_normalized_in_url(): + cases = [ ("grpc-web", "http://h:1/grpc-web/svc/M"), ("/grpc-web/", "http://h:1/grpc-web/svc/M"), ("/a/b", "http://h:1/a/b/svc/M"), ("", "http://h:1/svc/M"), - ], -) -def test_path_prefix_normalized_in_url(raw, expected_url): - sender = FakeSender(body=_ok_response(b"r")) - channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) - mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q")) - assert sender.calls[0][0] == expected_url + ] + for raw, expected_url in cases: + sender = FakeSender(body=_ok_response(b"r")) + channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + await mc(b"q") + assert sender.calls[0][0] == expected_url, raw def test_shim_factory_extracts_path_prefix_option(): - from weaviate_client_web._shim import _aio_insecure_channel - with_prefix = _aio_insecure_channel( target="h:1", options=[("grpc.max_send_message_length", 1), ("grpc-web.path_prefix", "/grpc-web")], @@ -608,15 +605,13 @@ def test_shim_factory_extracts_path_prefix_option(): assert without_prefix._path_prefix == "" -def test_set_sender_overrides_default(): +async def test_set_sender_overrides_default(): sender = FakeSender(body=_ok_response(b"y")) set_sender(sender) try: channel = GrpcWebChannel("h:1", secure=False) # no explicit sender mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - assert asyncio.run(mc(b"q")) == b"y" + assert await mc(b"q") == b"y" finally: - # restore the real default so other tests/processes are unaffected - from weaviate_client_web._sender import pyfetch_sender - + # restore the real default so other tests are unaffected set_sender(pyfetch_sender) diff --git a/setup.cfg b/setup.cfg index 7343116a5..4ebf52c4b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,12 @@ python_requires = >=3.10 [options.extras_require] agents = weaviate-agents >=1.0.0, <2.0.0 +grpc-web = + # The Pyodide/WASM grpc-web companion, built from packages/web in this repo and + # versioned in lockstep (see TODO(lockstep) in packages/web/pyproject.toml). The + # marker makes the extra a no-op on CPython: the companion imports pyodide at + # module scope and only makes sense under Emscripten. + weaviate-client-web; sys_platform == "emscripten" [options.package_data] # If any package or subpackage contains *.txt, *.rst or *.md files, include them: diff --git a/weaviate/__init__.py b/weaviate/__init__.py index fbd0fdabf..ef40e7a96 100644 --- a/weaviate/__init__.py +++ b/weaviate/__init__.py @@ -19,8 +19,9 @@ "weaviate requires the weaviate-client-web package under " "WebAssembly/Pyodide: there is no grpcio wheel for Emscripten, and " "weaviate-client-web provides the grpc-web (fetch) transport in its " - "place. Install it (e.g. micropip.install('weaviate-client-web')) and " - "import weaviate again." + "place. Install it via the extra (e.g. " + "micropip.install('weaviate-client[grpc-web]')) and import weaviate " + "again." ) from exc import os diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index 0f7d40bd1..82c8b3851 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -165,10 +165,11 @@ def _check_grpc_web_usable(self, is_async: bool) -> None: raise WeaviateInvalidInputError( "grpc_path_prefix enables grpc-web, which requires the " "'weaviate-client-web' package (it installs a grpc shim before " - "'import weaviate'); it is not active in this environment. Under Pyodide a " - "plain `import weaviate` activates it; on CPython call " - "weaviate_client_web.install(force=True) and set_sender(make_httpx_sender()) " - "before importing weaviate (intended for integration testing)." + "'import weaviate'); it is not active in this environment. grpc-web is " + "only available under WebAssembly/Pyodide, where a plain `import " + "weaviate` activates it (install the companion with " + "micropip.install('weaviate-client[grpc-web]')); on CPython use native " + "gRPC instead." ) def _grpc_channel( From f7912ed3f263e694e003fe13a2deffae481c9cc3 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:28:56 +0200 Subject: [PATCH 05/12] build(grpc-web): version weaviate-client-web in lockstep via setuptools_scm The companion's version now derives from the repository's git tags (setuptools_scm with its root at the repo root) instead of a hardcoded 0.0.1.dev0, so every build carries the same version as weaviate-client; the pyodide-e2e job asserts the two built wheels match. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- .github/workflows/main.yaml | 8 ++++++++ packages/web/pyproject.toml | 11 +++++++---- setup.cfg | 8 ++++---- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 44e52d6d0..8a1f1d3f8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -134,6 +134,14 @@ jobs: pip install build python -m build --wheel --outdir dist . python -m build --wheel --outdir dist packages/web + - name: Assert both wheels carry the same version + # weaviate-client-web is versioned in lockstep with weaviate-client: both derive + # from the same git tag via setuptools_scm. + run: | + base=$(basename dist/weaviate_client-*.whl); base=${base#weaviate_client-}; base=${base%%-*} + web=$(basename dist/weaviate_client_web-*.whl); web=${web#weaviate_client_web-}; web=${web%%-*} + echo "weaviate-client=$base weaviate-client-web=$web" + test "$base" = "$web" - name: Run the unit suite inside Pyodide under Node # The grpc-web package imports pyodide at module scope, so its unit tests only # run here. Needs no running Weaviate. diff --git a/packages/web/pyproject.toml b/packages/web/pyproject.toml index 95792c315..1ad43f705 100644 --- a/packages/web/pyproject.toml +++ b/packages/web/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=65", "wheel"] +requires = ["setuptools>=65", "setuptools_scm[toml] >6.2", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -10,9 +10,9 @@ requires-python = ">=3.10" license = { text = "BSD-3-Clause" } authors = [{ name = "Weaviate", email = "hello@weaviate.io" }] keywords = ["weaviate", "grpc-web", "pyodide", "wasm", "emscripten"] -# Version is kept in lockstep with weaviate-client. TODO(lockstep): derive from the same -# git tag via setuptools_scm and assert the built versions match in CI before publishing. -version = "0.0.1.dev0" +# Version comes from the repository's git tags via setuptools_scm (root below), so every +# build carries the same version as weaviate-client; CI asserts the built wheels match. +dynamic = ["version"] # Deliberately depends on weaviate-client WITHOUT grpcio (grpcio is excluded under # Emscripten by the `sys_platform != "emscripten"` marker in the base package's deps). dependencies = [ @@ -25,6 +25,9 @@ dependencies = [ Source = "https://github.com/weaviate/weaviate-python-client" Tracker = "https://github.com/weaviate/weaviate-python-client/issues" +[tool.setuptools_scm] +root = "../.." + [tool.setuptools.packages.find] where = ["src"] diff --git a/setup.cfg b/setup.cfg index 4ebf52c4b..26b397613 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,10 +49,10 @@ python_requires = >=3.10 agents = weaviate-agents >=1.0.0, <2.0.0 grpc-web = - # The Pyodide/WASM grpc-web companion, built from packages/web in this repo and - # versioned in lockstep (see TODO(lockstep) in packages/web/pyproject.toml). The - # marker makes the extra a no-op on CPython: the companion imports pyodide at - # module scope and only makes sense under Emscripten. + # The Pyodide/WASM grpc-web companion, built from packages/web in this repo. Both + # packages derive their version from the same git tag (setuptools_scm; CI asserts + # the built wheels match). The marker makes the extra a no-op on CPython: the + # companion imports pyodide at module scope and only makes sense under Emscripten. weaviate-client-web; sys_platform == "emscripten" [options.package_data] From e9cd331fb600d8499313824d848cf378c99b8308 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:17:30 +0200 Subject: [PATCH 06/12] build(grpc-web): pin weaviate-client to the lockstep version at build time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit packages/web/setup.py injects weaviate-client== into the companion's requirements when the wheel is built, so a mismatched pair can never resolve at install time — the two packages share private contracts (error-string markers, exception constants). Consequence: every tag must publish both packages. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- packages/web/pyproject.toml | 11 +++-------- packages/web/setup.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 packages/web/setup.py diff --git a/packages/web/pyproject.toml b/packages/web/pyproject.toml index 1ad43f705..0f4872e18 100644 --- a/packages/web/pyproject.toml +++ b/packages/web/pyproject.toml @@ -12,14 +12,9 @@ authors = [{ name = "Weaviate", email = "hello@weaviate.io" }] keywords = ["weaviate", "grpc-web", "pyodide", "wasm", "emscripten"] # Version comes from the repository's git tags via setuptools_scm (root below), so every # build carries the same version as weaviate-client; CI asserts the built wheels match. -dynamic = ["version"] -# Deliberately depends on weaviate-client WITHOUT grpcio (grpcio is excluded under -# Emscripten by the `sys_platform != "emscripten"` marker in the base package's deps). -dependencies = [ - "weaviate-client", - # Pyodide's bundled httpx build omits anyio, but authlib imports it directly. - 'anyio ; sys_platform == "emscripten"', -] +# Dependencies are computed in setup.py: the weaviate-client requirement is pinned to +# that same version at build time, so mismatched pairs cannot resolve at install time. +dynamic = ["version", "dependencies"] [project.urls] Source = "https://github.com/weaviate/weaviate-python-client" diff --git a/packages/web/setup.py b/packages/web/setup.py new file mode 100644 index 000000000..e99c3f653 --- /dev/null +++ b/packages/web/setup.py @@ -0,0 +1,25 @@ +"""Injects the lockstep ``weaviate-client==`` pin at build time. + +Both packages derive their version from the repository's git tags (setuptools_scm), so +the version is only known when the wheel is built and a static ``dependencies`` list +cannot express the pin. The pin makes mismatched pairs unresolvable at install time: +the two packages share private contracts (the ``HTTP `` error-string markers, +the exception constants ``_channel`` imports), so a companion must only ever install +next to the base client it was built with. + +Consequence for releasing: every tag must publish BOTH packages — a base-only release +would leave the extra pointing at a companion whose pin no longer resolves. +""" + +from setuptools import setup +from setuptools_scm import get_version + +version = get_version(root="../..", relative_to=__file__) + +setup( + install_requires=[ + f"weaviate-client=={version}", + # Pyodide's bundled httpx build omits anyio, but authlib imports it directly. + 'anyio ; sys_platform == "emscripten"', + ] +) From c3ff78089cc02ff34c486baa9c8711b895b4afb5 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:17:37 +0200 Subject: [PATCH 07/12] test(grpc-web): run the unit suite with pytest inside Pyodide via JSPI pytest executes the packages/web/tests suite inside Pyodide under Node with --experimental-wasm-jspi: async tests run through run_until_complete, which stack-switches when the runner enters via callPromising(). This replaces the hand-rolled runner/harness with standard pytest collection, fixtures and parametrize, and makes empty or partial collection fail the run (pytest exit codes reach JS as a return value, never as an exception across the bridge). The base client's import-hook branches (missing companion, broken companion, grpc-present fall-through) are covered by subprocess tests in test/test_wasm_compat.py on CPython; the bootstrap scenario in units.mjs keeps the one path that needs real Pyodide, micropip and the wheels. Also strengthens assertions the port had weakened (identity restore on uninstall, the literal grpc version pin, install() returning True) and fixes two stale README claims about direct installs and off-Emscripten imports. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- .github/workflows/main.yaml | 6 +- ci/pyodide-e2e/units.mjs | 140 ++++---- packages/web/README.md | 21 +- packages/web/tests/conftest.py | 14 +- packages/web/tests/harness.py | 53 --- packages/web/tests/runner.py | 42 --- packages/web/tests/test_framing.py | 22 +- packages/web/tests/test_httpx_fetch.py | 421 +++++++++++------------- packages/web/tests/test_shim_install.py | 17 +- packages/web/tests/test_transport.py | 127 +++---- test/test_wasm_compat.py | 105 ++++++ 11 files changed, 473 insertions(+), 495 deletions(-) delete mode 100644 packages/web/tests/harness.py delete mode 100644 packages/web/tests/runner.py diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8a1f1d3f8..da9ea3302 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -144,10 +144,12 @@ jobs: test "$base" = "$web" - name: Run the unit suite inside Pyodide under Node # The grpc-web package imports pyodide at module scope, so its unit tests only - # run here. Needs no running Weaviate. + # run here. Needs no running Weaviate. The JSPI flag lets pytest's synchronous + # runner await async tests via stack switching; without it the run fails + # loudly at startup (never a false green). run: | npm install --prefix ci/pyodide-e2e - node ci/pyodide-e2e/units.mjs dist + node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist - name: start weaviate run: | source ./ci/compose.sh diff --git a/ci/pyodide-e2e/units.mjs b/ci/pyodide-e2e/units.mjs index 61ba72c2c..f333080a7 100644 --- a/ci/pyodide-e2e/units.mjs +++ b/ci/pyodide-e2e/units.mjs @@ -1,17 +1,22 @@ -// Runs the weaviate_client_web unit suite (packages/web/tests) inside Pyodide (WASM) -// under Node, plus bootstrap scenarios that each need a fresh interpreter. No running -// Weaviate is needed — everything is driven through fake senders / a fake pyfetch. +// Runs the weaviate_client_web unit suite (packages/web/tests) with pytest inside +// Pyodide (WASM) under Node, plus a bootstrap scenario in a fresh interpreter. No +// running Weaviate is needed — everything is driven through fake senders / a fake +// pyfetch. // -// Usage: node units.mjs +// Usage: node --experimental-wasm-jspi units.mjs // must contain exactly the two locally-built pure wheels: // weaviate_client-*.whl and weaviate_client_web-*.whl (same layout as run.mjs). // +// The JSPI flag is required: pytest's runner is synchronous, so async tests execute +// through run_until_complete, which needs stack switching (enableRunUntilComplete + +// a callPromising() entrypoint). Without the flag the run fails loudly at startup — +// it can never produce a false green. +// // The package imports pyodide at module scope, so this harness is the only place its -// unit tests can run; scenarios needing a clean import state (bootstrap and -// install-hint semantics) get one loadPyodide() each. Deliberately untested: the -// fall-through in weaviate/__init__.py when a real grpc module exists — no grpcio -// wheel exists for Emscripten, so that branch is CPython-only defence and cannot be -// exercised here. +// unit tests can run. The base client's hook logic (missing companion, broken +// companion, grpc-present fall-through) is covered by subprocess tests in +// test/test_wasm_compat.py on CPython; the bootstrap scenario below covers the one +// path that needs real Pyodide, micropip and the wheels. import { readdirSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -39,40 +44,25 @@ if ( process.exit(2); } -// Fresh interpreter with micropip ready and the wheels dir mounted; `only` restricts -// which of the two wheels get installed (the install-hint scenarios need the base -// client without its companion). -async function freshPyodide(only = prefixes) { - const pyodide = await loadPyodide({}); +// Fresh interpreter with micropip ready and the wheels dir mounted. +async function freshPyodide() { + const pyodide = await loadPyodide({ enableRunUntilComplete: true }); await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); pyodide.FS.mkdirTree("/wheels"); pyodide.mountNodeFS("/wheels", wheelsDir); for (const wheel of wheels) { - if (only.some((p) => wheel.startsWith(p))) { - await micropip.install(`emfs:/wheels/${wheel}`); - } + await micropip.install(`emfs:/wheels/${wheel}`); } return pyodide; } -async function scenario(name, pyodide, code) { - try { - pyodide.runPython(code); - console.log(`OK scenario: ${name}`); - } catch (err) { - console.error(`FAIL scenario: ${name}`); - console.error(err); - process.exit(1); - } -} +// --- bootstrap scenario: needs a clean import state, so its own interpreter -------- -// --- fresh-interpreter bootstrap scenarios ----------------------------------------- - -await scenario( - "bare 'import weaviate' bootstraps the companion", - await freshPyodide(), - ` +{ + const pyodide = await freshPyodide(); + try { + pyodide.runPython(` import sys assert "weaviate_client_web" not in sys.modules import weaviate # the ONLY weaviate-side import: must bootstrap the companion @@ -86,69 +76,55 @@ assert getattr(grpc, "__weaviate_client_web_shim__", False) is True assert getattr( httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False ) is True -`, -); - -await scenario( - "missing companion raises the install hint", - await freshPyodide(["weaviate_client-"]), - ` -try: - import weaviate -except ImportError as e: - assert "weaviate-client-web" in str(e), str(e) - assert "weaviate-client[grpc-web]" in str(e), str(e) - assert "WebAssembly/Pyodide" in str(e), str(e) -else: - raise AssertionError("expected ImportError without the companion") -`, -); - -await scenario( - "a broken companion surfaces its own error, not the install hint", - await freshPyodide(["weaviate_client-"]), - ` -# An INSTALLED companion whose import fails (here: a missing dependency of its own) -# must raise that error — the "install weaviate-client[grpc-web]" hint would send the -# user to reinstall a package that is already there. -import pathlib, sys -pkg = pathlib.Path("/broken/weaviate_client_web") -pkg.mkdir(parents=True) -(pkg / "__init__.py").write_text( - "raise ModuleNotFoundError(\\"No module named 'anyio'\\", name='anyio')\\n" -) -sys.path.insert(0, "/broken") -try: - import weaviate -except ImportError as e: - assert e.name == "anyio", (e.name, str(e)) - assert "anyio" in str(e), str(e) - assert "grpc-web" not in str(e), str(e) -else: - raise AssertionError("expected the companion's own ImportError to surface") -`, -); +`); + console.log("OK scenario: bare 'import weaviate' bootstraps the companion"); + } catch (err) { + console.error("FAIL scenario: bare 'import weaviate' bootstraps the companion"); + console.error(err); + process.exit(1); + } +} -// --- the main unit suite ----------------------------------------------------------- +// --- the pytest suite -------------------------------------------------------------- const testsDir = resolve(here, "../../packages/web/tests"); const pyodide = await freshPyodide(); console.log( `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, ); +const micropip = pyodide.pyimport("micropip"); +await micropip.install(["pytest==9.0.2", "pytest-asyncio==0.25.3"]); pyodide.FS.mkdirTree("/units"); pyodide.mountNodeFS("/units", testsDir); -try { - await pyodide.runPythonAsync(` + +// pytest.main is synchronous; entering through callPromising() lets the async tests +// stack-switch (run_until_complete) instead of failing with "Cannot stack switch". +const runPytest = pyodide.runPython(` import sys -sys.path.insert(0, "/units") -import runner -await runner.main() +sys.dont_write_bytecode = True # /units is the host checkout: no __pycache__ in it + +import pytest + +def _run(): + return int(pytest.main([ + "-v", + "-p", "no:cacheprovider", # no .pytest_cache in the host checkout either + "-o", "asyncio_mode=auto", + "-o", "asyncio_default_fixture_loop_scope=function", + "/units", + ])) + +_run `); +let exitCode; +try { + exitCode = await runPytest.callPromising(); } catch (err) { console.error(err); process.exit(1); } +// Any nonzero pytest exit code fails the run — including 5, "no tests collected". +console.log(`pytest exit code: ${exitCode}`); // The interpreters loaded above keep live handles on the Node event loop, so the // process does not exit on its own. -process.exit(0); +process.exit(exitCode === 0 ? 0 : 1); diff --git a/packages/web/README.md b/packages/web/README.md index c17e7f8df..3c4fd46c1 100644 --- a/packages/web/README.md +++ b/packages/web/README.md @@ -23,7 +23,10 @@ await micropip.install("weaviate-client[grpc-web]") The extra carries a `sys_platform == "emscripten"` marker, so the same requirement is a no-op on CPython — one requirements list works everywhere. Installing the companion -directly (`micropip.install("weaviate-client-web")`) works too. +directly (`micropip.install("weaviate-client-web")`) works too: it pins +`weaviate-client` to its own exact version, so a mismatched pair can never resolve. +Both forms require the first `weaviate-client` release that ships the extra and the +Emscripten marker; against older releases the resolver fails on `grpcio`. This package is defined for its environment: it imports `pyodide` at module scope and is therefore only importable under Emscripten/Pyodide. On CPython the base client never @@ -124,7 +127,7 @@ Pass `headers={...}` / `auth_credentials=...` as usual for API keys, OIDC or WCD Importing the companion explicitly first also works and remains the explicit form: ```python -import weaviate_client_web # installs the grpc shim under Emscripten (no-op elsewhere) +import weaviate_client_web # installs the grpc shim (only importable under Emscripten) import weaviate ``` @@ -187,11 +190,13 @@ Pyodide. From the repository root: python -m build --wheel --outdir dist . python -m build --wheel --outdir dist packages/web npm install --prefix ci/pyodide-e2e -node ci/pyodide-e2e/units.mjs dist # unit suite + bootstrap scenarios, no Weaviate needed -node ci/pyodide-e2e/run.mjs dist # e2e suite, needs a running Weaviate (see ci/) +node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist # pytest unit suite, no Weaviate needed +node ci/pyodide-e2e/run.mjs dist # e2e suite, needs a running Weaviate (see ci/) ``` -`units.mjs` runs the unit suite in `packages/web/tests/` (driven entirely through fake -senders and a fake `pyfetch`) plus fresh-interpreter bootstrap scenarios; `run.mjs` -runs the e2e suite against a live Weaviate. A `conftest.py` keeps pytest from -collecting the test modules on CPython, where they cannot import. +`units.mjs` runs pytest over `packages/web/tests/` inside Pyodide — async tests execute +on Pyodide's event loop via JSPI stack switching, hence the Node flag — plus a +fresh-interpreter bootstrap scenario; everything is driven through fake senders and a +fake `pyfetch`. `run.mjs` runs the e2e suite against a live Weaviate. On CPython the +`conftest.py` keeps pytest from collecting these modules (they cannot import there); +the base client's import-hook branches are covered by `test/test_wasm_compat.py`. diff --git a/packages/web/tests/conftest.py b/packages/web/tests/conftest.py index 9bc903df3..a6491b8dc 100644 --- a/packages/web/tests/conftest.py +++ b/packages/web/tests/conftest.py @@ -1,8 +1,14 @@ -"""Keep pytest away from this directory. +"""Restrict this suite to Pyodide. The package under test imports ``pyodide`` at module scope, so the test modules here -are only importable under Emscripten/Pyodide. They are run by -``ci/pyodide-e2e/units.mjs`` (via ``runner.py``), not by pytest. +are only importable under Emscripten/Pyodide. There, pytest runs them via +``ci/pyodide-e2e/units.mjs`` (async tests execute on Pyodide's event loop through JSPI +stack switching). On CPython, keep pytest from collecting them: + + node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist """ -collect_ignore_glob = ["*.py"] +import sys + +if sys.platform != "emscripten": + collect_ignore_glob = ["*.py"] diff --git a/packages/web/tests/harness.py b/packages/web/tests/harness.py deleted file mode 100644 index c5c8c0ad2..000000000 --- a/packages/web/tests/harness.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Shared helpers for the in-Pyodide test modules (pytest is not available here).""" - -import contextlib -import sys -from typing import Any, Optional - - -class _Caught: - value: Any - - -@contextlib.contextmanager -def raises(exc_type, contains: Optional[str] = None): - """``pytest.raises`` stand-in; ``contains`` is a plain substring, not a regex.""" - caught = _Caught() - try: - yield caught - except exc_type as e: - caught.value = e - if contains is not None and contains not in str(e): - raise AssertionError(f"{e!r} does not contain {contains!r}") from e - else: - raise AssertionError(f"expected {exc_type.__name__} to be raised") - - -@contextlib.contextmanager -def patched(obj, name: str, value): - """``monkeypatch.setattr`` stand-in with restore-on-exit.""" - sentinel = object() - old = getattr(obj, name, sentinel) - setattr(obj, name, value) - try: - yield - finally: - if old is sentinel: - delattr(obj, name) - else: - setattr(obj, name, old) - - -@contextlib.contextmanager -def sys_module(name: str, module): - """Set ``sys.modules[name]`` (``None`` makes ``import name`` raise) and restore.""" - present = name in sys.modules - old = sys.modules.get(name) - sys.modules[name] = module - try: - yield - finally: - if present: - sys.modules[name] = old # type: ignore[assignment] - else: - del sys.modules[name] diff --git a/packages/web/tests/runner.py b/packages/web/tests/runner.py deleted file mode 100644 index e9a6ae1fc..000000000 --- a/packages/web/tests/runner.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Collects and runs the test modules in this directory on Pyodide's event loop. - -Invoked by ``ci/pyodide-e2e/units.mjs``, which mounts this directory into the -interpreter and awaits :func:`main` (``asyncio.run()`` cannot be used inside Pyodide). -Test functions are ``test_*`` callables — plain or ``async`` — collected per module, -so names are qualified and cannot shadow across files. -""" - -import inspect -import traceback -from typing import List - -import test_framing -import test_httpx_fetch -import test_shim_install -import test_transport - -_MODULES = (test_framing, test_transport, test_httpx_fetch, test_shim_install) - - -async def main() -> None: - tests = [] - for module in _MODULES: - for name in sorted(vars(module)): - fn = getattr(module, name) - if name.startswith("test_") and callable(fn): - tests.append((f"{module.__name__}.{name}", fn)) - failures: List[str] = [] - for name, fn in tests: - try: - result = fn() - if inspect.iscoroutine(result): - await result - except Exception: - failures.append(name) - traceback.print_exc() - print(f"FAIL {name}", flush=True) - else: - print(f"OK {name}", flush=True) - print(f"{len(tests) - len(failures)}/{len(tests)} unit tests passed", flush=True) - if failures: - raise SystemExit(f"{len(failures)} unit test(s) failed: {', '.join(failures)}") diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py index 16aeebe69..8054dbf94 100644 --- a/packages/web/tests/test_framing.py +++ b/packages/web/tests/test_framing.py @@ -1,8 +1,8 @@ -"""grpc-web framing tests. Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``.""" +"""grpc-web framing tests. Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``.""" import struct -from harness import raises +import pytest from weaviate_client_web._framing import ( FrameError, @@ -44,7 +44,7 @@ def test_split_response_multiple_messages(): def test_split_response_message_after_trailer_raises(): body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") - with raises(FrameError, contains="after the trailer"): + with pytest.raises(FrameError, match="after the trailer"): split_response(body) @@ -97,18 +97,18 @@ def test_parse_trailers_keeps_status_when_a_key_is_not_ascii(): def test_truncated_frame_raises(): framed = encode_message(b"hello")[:-2] - with raises(TruncatedFrameError): + with pytest.raises(TruncatedFrameError): list(iter_frames(framed)) - with raises(TruncatedFrameError): + with pytest.raises(TruncatedFrameError): list(iter_frames(b"\x00\x00\x00")) # shorter than one frame header -def test_unknown_frame_flag_raises(): +@pytest.mark.parametrize("first_byte", [b"{", b"<", b"\x02", b"\x40", b"\xff"]) +def test_unknown_frame_flag_raises(first_byte): # a JSON / HTML body, or a flag bit this transport does not know - for first_byte in (b"{", b"<", b"\x02", b"\x40", b"\xff"): - body = first_byte + b"\x00\x00\x00\x01x" - with raises(UnknownFrameFlagError, contains="unknown grpc-web frame flag"): - list(iter_frames(body)) + body = first_byte + b"\x00\x00\x00\x01x" + with pytest.raises(UnknownFrameFlagError, match="unknown grpc-web frame flag"): + list(iter_frames(body)) def test_frame_errors_are_value_errors(): @@ -118,5 +118,5 @@ def test_frame_errors_are_value_errors(): def test_compressed_message_frame_rejected(): body = _frame(b"x", 0x01) - with raises(FrameError, contains="compressed"): + with pytest.raises(FrameError, match="compressed"): split_response(body) diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py index c12a29d5f..c1833d74c 100644 --- a/packages/web/tests/test_httpx_fetch.py +++ b/packages/web/tests/test_httpx_fetch.py @@ -1,20 +1,20 @@ """Tests for the fetch-based httpx transport (``_httpx_fetch.py``). -Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``. ``pyfetch`` is bound at import -time in ``_httpx_fetch``, so fakes patch that module attribute; ``from js import -AbortSignal`` is resolved per request, so a fake js module in ``sys.modules`` +Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``. ``pyfetch`` is bound at +import time in ``_httpx_fetch``, so fakes patch that module attribute; ``from js +import AbortSignal`` is resolved per request, so a fake js module in ``sys.modules`` intercepts it even under real Pyodide. The install-semantics tests at the bottom run against this interpreter's real installation: importing ``weaviate_client_web`` bootstrapped the transport globally. """ -import contextlib +import sys import types from typing import Any, Dict, List, Optional import httpx -from harness import patched, raises, sys_module +import pytest import weaviate_client_web import weaviate_client_web._httpx_fetch as _httpx_fetch @@ -44,11 +44,11 @@ async def __call__(self, url: str, **kwargs: Any) -> FakeFetchResponse: return self.response -@contextlib.contextmanager -def fake_pyfetch(): +@pytest.fixture +def fake_pyfetch(monkeypatch) -> FakePyfetch: fetch = FakePyfetch() - with patched(_httpx_fetch, "pyfetch", fetch): - yield fetch + monkeypatch.setattr(_httpx_fetch, "pyfetch", fetch) + return fetch class _AbortSignalRecorder: @@ -60,21 +60,20 @@ def timeout(self, ms: int): return f"signal-{ms}" -@contextlib.contextmanager -def fake_abort_signal(): +@pytest.fixture +def fake_abort_signal(monkeypatch) -> _AbortSignalRecorder: recorder = _AbortSignalRecorder() js_mod = types.ModuleType("js") js_mod.AbortSignal = recorder # type: ignore[attr-defined] - with sys_module("js", js_mod): - yield recorder + monkeypatch.setitem(sys.modules, "js", js_mod) + return recorder -@contextlib.contextmanager -def missing_js_module(): - # sys.modules[name] = None makes ``import js`` raise ImportError — the closest - # in-process stand-in for an environment without the js bridge. - with sys_module("js", None): - yield +@pytest.fixture +def missing_js(monkeypatch): + # sys.modules["js"] = None makes ``import js`` raise ImportError — the closest + # in-process stand-in for an environment without the js bridge + monkeypatch.setitem(sys.modules, "js", None) async def _handle(request: httpx.Request) -> httpx.Response: @@ -97,72 +96,66 @@ async def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: return await client.request(method, url, **kwargs) -async def test_basic_get_round_trip(): - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse( - status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' - ) - response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) +async def test_basic_get_round_trip(fake_pyfetch): + fake_pyfetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' + ) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.status_code == 200 - assert response.json() == {"version": "1.30.0"} - assert response.headers["content-type"] == "application/json" - call = fetch.calls[0] - assert call["url"] == "http://h:8080/v1/meta" - assert call["method"] == "GET" + assert response.status_code == 200 + assert response.json() == {"version": "1.30.0"} + assert response.headers["content-type"] == "application/json" + call = fake_pyfetch.calls[0] + assert call["url"] == "http://h:8080/v1/meta" + assert call["method"] == "GET" -async def test_response_has_request_attached_for_raise_for_status(): - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=404, body=b"") - response = await _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) - with raises(httpx.HTTPStatusError): - response.raise_for_status() +async def test_response_has_request_attached_for_raise_for_status(fake_pyfetch): + fake_pyfetch.response = FakeFetchResponse(status=404, body=b"") + response = await _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) + with pytest.raises(httpx.HTTPStatusError): + response.raise_for_status() -async def test_head_response_without_body_yields_empty_content(): +@pytest.mark.parametrize("status", [204, 404]) +async def test_head_response_without_body_yields_empty_content(fake_pyfetch, status): # data.exists() / tenants.exists() are HEAD requests answered 204/404 with a null # body; the transport must hand httpx an empty response, not fail on the missing body - for status in (204, 404): - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=status, body=None) - response = await _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) - assert response.status_code == status - assert response.content == b"" - assert "body" not in fetch.calls[0] + fake_pyfetch.response = FakeFetchResponse(status=status, body=None) + response = await _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == status + assert response.content == b"" + assert "body" not in fake_pyfetch.calls[0] -async def test_delete_204_without_body_yields_empty_content(): +async def test_delete_204_without_body_yields_empty_content(fake_pyfetch): # data.delete_by_id() / reference_delete() are answered 204 with a null body - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=204, body=None) - response = await _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) - assert response.status_code == 204 - assert response.content == b"" + fake_pyfetch.response = FakeFetchResponse(status=204, body=None) + response = await _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == 204 + assert response.content == b"" -async def test_body_less_response_through_async_client(): +async def test_body_less_response_through_async_client(fake_pyfetch): # the full httpx.AsyncClient path (stream wrapping + read) on a body-less response - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=204, body=None) - response = await _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") - assert response.status_code == 204 - assert response.content == b"" + fake_pyfetch.response = FakeFetchResponse(status=204, body=None) + response = await _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") + assert response.status_code == 204 + assert response.content == b"" -async def test_response_through_async_client_exposes_elapsed_and_content(): +async def test_response_through_async_client_exposes_elapsed_and_content(fake_pyfetch): # the batch-references path reads ``res.elapsed``, which httpx only sets after it # has read/closed a stream-backed response; a pre-loaded body never gets one payload = b'[{"result": {"status": "SUCCESS"}}]' - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=200, body=payload) - response = await _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") - assert response.content == payload - assert response.json() == [{"result": {"status": "SUCCESS"}}] - assert response.elapsed.total_seconds() >= 0 + fake_pyfetch.response = FakeFetchResponse(status=200, body=payload) + response = await _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") + assert response.content == payload + assert response.json() == [{"result": {"status": "SUCCESS"}}] + assert response.elapsed.total_seconds() >= 0 -async def test_fetch_managed_request_headers_stripped(): +async def test_fetch_managed_request_headers_stripped(fake_pyfetch): request = httpx.Request( "POST", "http://h:8080/v1/objects", @@ -176,76 +169,63 @@ async def test_fetch_managed_request_headers_stripped(): }, content=b"{}", ) - with fake_pyfetch() as fetch: - await _handle(request) - sent = fetch.calls[0]["headers"] - assert sent["authorization"] == "Bearer k" - assert sent["content-type"] == "application/json" - for managed in ( - "host", - "connection", - "accept-encoding", - "content-length", - "transfer-encoding", - ): - assert managed not in sent, managed - - -async def test_get_without_body_omits_body_kwarg(): + await _handle(request) + sent = fake_pyfetch.calls[0]["headers"] + assert sent["authorization"] == "Bearer k" + assert sent["content-type"] == "application/json" + for managed in ("host", "connection", "accept-encoding", "content-length", "transfer-encoding"): + assert managed not in sent + + +async def test_get_without_body_omits_body_kwarg(fake_pyfetch): # fetch rejects GET/HEAD requests that carry a body, so the kwarg must be absent - with fake_pyfetch() as fetch: - await _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) - assert "body" not in fetch.calls[0] + await _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) + assert "body" not in fake_pyfetch.calls[0] -async def test_post_body_passed(): - with fake_pyfetch() as fetch: - await _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) - assert fetch.calls[0]["body"] == b'{"query": "x"}' +async def test_post_body_passed(fake_pyfetch): + await _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) + assert fake_pyfetch.calls[0]["body"] == b'{"query": "x"}' -async def test_delete_with_body_passed(): +async def test_delete_with_body_passed(fake_pyfetch): # the REST batch-delete path sends DELETE with a JSON body - with fake_pyfetch() as fetch: - await _handle( - httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}') - ) - assert fetch.calls[0]["body"] == b'{"match": {}}' + await _handle( + httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}') + ) + assert fake_pyfetch.calls[0]["body"] == b'{"match": {}}' -async def test_query_string_preserved_in_url(): - with fake_pyfetch() as fetch: - await _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) - assert fetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" +async def test_query_string_preserved_in_url(fake_pyfetch): + await _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) + assert fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" -async def test_content_encoding_stripped_from_response(): +async def test_content_encoding_stripped_from_response(fake_pyfetch): # fetch hands back ALREADY-decompressed bytes; if the original content-encoding # header were passed through, httpx.Response would gunzip a second time and raise # DecodingError. content-length is stale for the same reason. - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse( - status=200, - headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, - body=b'{"version": "1.30.0"}', - ) - response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.json() == {"version": "1.30.0"} - assert "content-encoding" not in response.headers - assert response.headers["x-other"] == "kept" + fake_pyfetch.response = FakeFetchResponse( + status=200, + headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, + body=b'{"version": "1.30.0"}', + ) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.json() == {"version": "1.30.0"} + assert "content-encoding" not in response.headers + assert response.headers["x-other"] == "kept" -async def test_unreadable_response_headers_tolerated(): +async def test_unreadable_response_headers_tolerated(fake_pyfetch): class BadHeaders: def keys(self): raise TypeError("header shape varies across Pyodide versions") - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=200, body=b"ok") - fetch.response.headers = BadHeaders() - response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.status_code == 200 - assert response.content == b"ok" + fake_pyfetch.response = FakeFetchResponse(status=200, body=b"ok") + fake_pyfetch.response.headers = BadHeaders() + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.status_code == 200 + assert response.content == b"ok" def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request: @@ -254,68 +234,57 @@ def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request return request -async def test_read_timeout_maps_to_abort_signal_ms(): +async def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_signal): # mirrors what weaviate's AsyncClient puts in extensions: connect/read/write/pool - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle( - _request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0}) - ) - assert signals.timeouts == [30000] - assert fetch.calls[0]["signal"] == "signal-30000" + await _handle(_request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0})) + assert fake_abort_signal.timeouts == [30000] + assert fake_pyfetch.calls[0]["signal"] == "signal-30000" -async def test_read_none_means_no_deadline_even_with_pool_and_connect_set(): +async def test_read_none_means_no_deadline_even_with_pool_and_connect_set( + fake_pyfetch, fake_abort_signal +): # what the base client hands over for a non-finite request timeout: read=None with the # session pool timeout still set; falling back to pool/connect would abort a long # insert after 5 s - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle( - _request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5}) - ) - await _handle( - _request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0}) - ) - assert signals.timeouts == [] - assert all("signal" not in c for c in fetch.calls) + await _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5})) + await _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) + assert fake_abort_signal.timeouts == [] + assert all("signal" not in c for c in fake_pyfetch.calls) -async def test_read_timeout_alone_sets_the_deadline(): - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) - assert signals.timeouts == [7000] - assert fetch.calls[0]["signal"] == "signal-7000" +async def test_read_timeout_alone_sets_the_deadline(fake_pyfetch, fake_abort_signal): + await _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) + assert fake_abort_signal.timeouts == [7000] + assert fake_pyfetch.calls[0]["signal"] == "signal-7000" -async def test_no_timeout_extension_sends_no_signal(): - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert signals.timeouts == [] - assert "signal" not in fetch.calls[0] +async def test_no_timeout_extension_sends_no_signal(fake_pyfetch, fake_abort_signal): + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] -async def test_missing_js_module_degrades_to_no_signal(): +async def test_missing_js_module_degrades_to_no_signal(fake_pyfetch, missing_js): # without the js bridge the AbortSignal import fails; the request must still go out - with fake_pyfetch() as fetch, missing_js_module(): - response = await _handle( - _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) - ) - assert response.status_code == 200 - assert "signal" not in fetch.calls[0] + response = await _handle( + _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) + ) + assert response.status_code == 200 + assert "signal" not in fake_pyfetch.calls[0] -async def test_zero_timeout_means_no_deadline(): +async def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): # an explicit read=0 must not fall through to the 5s connect timeout, nor become an # immediate AbortSignal.timeout(0) - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle( - _request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None}) - ) - assert signals.timeouts == [] - assert "signal" not in fetch.calls[0] + await _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] -def test_abort_signal_ms_bounds(): - cases = [ +@pytest.mark.parametrize( + "timeout,expected_ms", + [ (None, None), (0, None), (-1, None), @@ -326,29 +295,28 @@ def test_abort_signal_ms_bounds(): (1e8, _MAX_ABORT_SIGNAL_MS), (1e10, _MAX_ABORT_SIGNAL_MS), (1e308, _MAX_ABORT_SIGNAL_MS), # finite, but *1000 overflows: capped, not an error - ] - for timeout, expected_ms in cases: - assert _abort_signal_ms(timeout) == expected_ms, timeout + ], +) +def test_abort_signal_ms_bounds(timeout, expected_ms): + assert _abort_signal_ms(timeout) == expected_ms -async def test_infinite_timeout_sends_no_signal(): +async def test_infinite_timeout_sends_no_signal(fake_pyfetch, fake_abort_signal): # an inf read deadline reaching the transport: no signal, not an OverflowError - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle( - _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) - ) - assert signals.timeouts == [] - assert "signal" not in fetch.calls[0] + await _handle( + _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) + ) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] -async def test_huge_timeout_is_capped_to_int32_ms(): +async def test_huge_timeout_is_capped_to_int32_ms(fake_pyfetch, fake_abort_signal): # setTimeout delays above 2^31-1 ms overflow and fire at once, aborting the request - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle( - _request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None}) - ) - assert signals.timeouts == [_MAX_ABORT_SIGNAL_MS] - assert fetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" + await _handle( + _request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None}) + ) + assert fake_abort_signal.timeouts == [_MAX_ABORT_SIGNAL_MS] + assert fake_pyfetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" class RaisingPyfetch: @@ -359,68 +327,67 @@ async def __call__(self, url: str, **kwargs: Any): raise self.exc -@contextlib.contextmanager -def raising_pyfetch(exc: BaseException): - with patched(_httpx_fetch, "pyfetch", RaisingPyfetch(exc)): - yield +def _install_raising_pyfetch(monkeypatch, exc: BaseException) -> None: + monkeypatch.setattr(_httpx_fetch, "pyfetch", RaisingPyfetch(exc)) -async def test_fetch_failure_maps_to_httpx_connect_error(): +async def test_fetch_failure_maps_to_httpx_connect_error(monkeypatch): # pyodide surfaces JS fetch rejections as OSError; the base client can only classify # httpx exceptions (WeaviateConnectionError etc.), so the shim must translate - with raising_pyfetch(OSError("TypeError: Failed to fetch")): - with raises(httpx.ConnectError, contains="Failed to fetch") as excinfo: - await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert isinstance(excinfo.value.__cause__, OSError) + _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) + with pytest.raises(httpx.ConnectError, match="Failed to fetch") as excinfo: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert isinstance(excinfo.value.__cause__, OSError) -async def test_fetch_abort_with_deadline_maps_to_read_timeout(): +async def test_fetch_abort_with_deadline_maps_to_read_timeout(monkeypatch, fake_abort_signal): # AbortSignal.timeout firing surfaces as an OSError subclass mentioning the abort; # with a deadline set this must classify as a timeout, not a connection error - with raising_pyfetch(OSError("AbortError: signal timed out")), fake_abort_signal(): - with raises(httpx.ReadTimeout, contains="signal timed out"): - await _handle( - _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) - ) + _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) + with pytest.raises(httpx.ReadTimeout, match="signal timed out"): + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) -async def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error(): +async def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error( + monkeypatch, fake_abort_signal +): # nearly every weaviate request sets a read deadline; a plain network failure on # such a request must remain a connection error, not become a timeout - with raising_pyfetch(OSError("TypeError: Failed to fetch")), fake_abort_signal(): - with raises(httpx.ConnectError, contains="Failed to fetch"): - await _handle( - _request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None}) - ) + _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) + with pytest.raises(httpx.ConnectError, match="Failed to fetch"): + await _handle( + _request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None}) + ) -async def test_fetch_abort_without_deadline_stays_connect_error(): +async def test_fetch_abort_without_deadline_stays_connect_error(monkeypatch, missing_js): # the same message without a deadline set (no js bridge -> no signal) is not OUR # timeout, so it must stay a connection error - with raising_pyfetch(OSError("AbortError: signal timed out")), missing_js_module(): - with raises(httpx.ConnectError): - await _handle( - _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) - ) + _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) + with pytest.raises(httpx.ConnectError): + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) -async def test_empty_oserror_str_keeps_repr_detail(): - with raising_pyfetch(OSError()): - with raises(httpx.ConnectError) as excinfo: - await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert "OSError" in str(excinfo.value) +async def test_empty_oserror_str_keeps_repr_detail(monkeypatch): + _install_raising_pyfetch(monkeypatch, OSError()) + with pytest.raises(httpx.ConnectError) as excinfo: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert "OSError" in str(excinfo.value) -async def test_crlf_in_header_value_rejected(): +async def test_crlf_in_header_value_rejected(fake_pyfetch): # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at # send time; this transport bypasses h11 and must keep that defence request = httpx.Request( "GET", "http://h:8080/v1/meta", headers={"x-key": "val\r\nx-injected: evil"} ) - with fake_pyfetch() as fetch: - with raises(httpx.LocalProtocolError): - await _handle(request) - assert fetch.calls == [] + with pytest.raises(httpx.LocalProtocolError): + await _handle(request) + assert fake_pyfetch.calls == [] # --------------------------------------------------------------------------- @@ -450,15 +417,18 @@ def test_sync_transport_left_untouched(): def test_uninstall_restores_original_transport(): + # the module keeps the pre-patch method while installed, so the restore can be + # checked by identity even though the install happened at import time + original = _httpx_fetch._original_handle_async_request + assert original is not None patched_method = httpx.AsyncHTTPTransport.handle_async_request try: weaviate_client_web.uninstall_fetch_transport() assert not weaviate_client_web.is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is original assert httpx.AsyncHTTPTransport.handle_async_request is not patched_method - assert not getattr( - httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False - ) weaviate_client_web.uninstall_fetch_transport() # no-op when not installed + assert httpx.AsyncHTTPTransport.handle_async_request is original finally: weaviate_client_web.install_fetch_transport() assert weaviate_client_web.is_fetch_transport_installed() @@ -468,14 +438,13 @@ def test_uninstall_restores_original_transport(): ) -async def test_installed_transport_routes_async_client_through_pyfetch(): +async def test_installed_transport_routes_async_client_through_pyfetch(fake_pyfetch): # the globally installed transport (no custom transport argument) must reach pyfetch - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse( - status=200, headers={"content-type": "application/json"}, body=b'{"ok": true}' - ) - async with httpx.AsyncClient() as client: - response = await client.get("http://h:8080/v1/meta") - assert response.status_code == 200 - assert response.json() == {"ok": True} - assert fetch.calls and fetch.calls[0]["url"] == "http://h:8080/v1/meta" + fake_pyfetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"ok": true}' + ) + async with httpx.AsyncClient() as client: + response = await client.get("http://h:8080/v1/meta") + assert response.status_code == 200 + assert response.json() == {"ok": True} + assert fake_pyfetch.calls and fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/meta" diff --git a/packages/web/tests/test_shim_install.py b/packages/web/tests/test_shim_install.py index d5dede428..0efab5973 100644 --- a/packages/web/tests/test_shim_install.py +++ b/packages/web/tests/test_shim_install.py @@ -1,14 +1,15 @@ """grpc shim tests, against this interpreter's real installation. -Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``: importing ``weaviate_client_web`` -bootstrapped the shim, so ``sys.modules['grpc']`` here IS the shim. Bootstrap scenarios -that need a clean import state (a bare ``import weaviate``, a missing or broken -companion) live in ``units.mjs``, one fresh interpreter each. +Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``: importing +``weaviate_client_web`` bootstrapped the shim, so ``sys.modules['grpc']`` here IS the +shim. Bootstrap scenarios that need a clean import state live in ``units.mjs`` (one +fresh interpreter) and, for the base client's hook logic, in +``test/test_wasm_compat.py`` on CPython. """ import struct -from harness import raises +import pytest import weaviate_client_web from weaviate_client_web import GrpcWebChannel, set_sender @@ -24,8 +25,9 @@ def test_import_weaviate_under_shim(): import grpc assert weaviate_client_web.is_installed() + assert weaviate_client_web.install() is True # idempotent, reports the shim in place assert getattr(grpc, "__weaviate_client_web_shim__", False) is True - assert grpc.__version__ == FAKE_GRPC_VERSION + assert grpc.__version__ == "1.72.1" assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False # type: ignore[attr-defined] from grpc.aio._typing import ChannelArgumentType # noqa: F401 @@ -43,9 +45,8 @@ def test_import_weaviate_under_shim(): def test_sync_channel_factory_raises_async_only(): import grpc - with raises(RuntimeError) as excinfo: + with pytest.raises(RuntimeError, match="async"): grpc.insecure_channel("localhost:50051") - assert "async" in str(excinfo.value).lower() async def test_real_proto_unary_round_trip_under_shim(): diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index 9bb118b97..b063767ee 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -1,8 +1,9 @@ -"""grpc-web channel / multicallable tests. Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``. +"""grpc-web channel / multicallable tests. -These exercise the transport classes directly through fake senders — no network, no -running Weaviate. Tests are ``async def`` awaited on Pyodide's event loop by -``runner.py`` (``asyncio.run()`` cannot be used inside Pyodide). +Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs`` — async tests execute on +Pyodide's event loop through JSPI stack switching (pytest-asyncio in auto mode). These +exercise the transport classes directly through fake senders — no network, no running +Weaviate. """ import asyncio @@ -10,7 +11,7 @@ import sys from typing import Dict, List, Optional, Tuple -from harness import patched, raises +import pytest from weaviate_client_web import GrpcWebChannel, set_sender from weaviate_client_web._channel import _body_excerpt, _encode_timeout @@ -97,7 +98,7 @@ async def test_error_trailer_raises_aiorpcerror(): channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.PERMISSION_DENIED assert excinfo.value.code().name == "PERMISSION_DENIED" @@ -108,7 +109,7 @@ async def test_percent_encoded_grpc_message_decoded(): body = _frame(b"grpc-status:5\r\ngrpc-message:not%20found\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.details() == "not found" @@ -118,7 +119,7 @@ async def test_trailers_only_status_in_http_headers(): FakeSender(status=200, headers={"grpc-status": "16", "grpc-message": "auth"}, body=b"") ) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.UNAUTHENTICATED @@ -155,7 +156,7 @@ async def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/C """Run one request against a canned HTTP response and return the AioRpcError.""" channel = _channel(FakeSender(status=status, headers=headers or {}, body=body)) mc = channel.unary_unary(path, lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") return excinfo.value @@ -185,10 +186,10 @@ async def test_nginx_502_maps_to_unavailable_so_the_client_retries(): assert "502 Bad Gateway" in err.details() -async def test_gateway_errors_are_unavailable(): - for status in (503, 504): - err = await _details_of(status, b"upstream down") - assert err.code() is StatusCode.UNAVAILABLE, status +@pytest.mark.parametrize("status", [503, 504]) +async def test_gateway_errors_are_unavailable(status): + err = await _details_of(status, b"upstream down") + assert err.code() is StatusCode.UNAVAILABLE async def test_nginx_404_html_is_reported_as_an_http_404(): @@ -337,7 +338,7 @@ async def test_binary_metadata_base64_encoded(): def test_stream_stream_raises_clear_error(): channel = _channel(FakeSender()) mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) - with raises(RuntimeError, contains="not supported over grpc-web"): + with pytest.raises(RuntimeError, match="not supported over grpc-web"): mc(request_iterator=iter([]), timeout=5, metadata=None) @@ -348,7 +349,7 @@ async def slow_sender(url, headers, body, timeout): channel = GrpcWebChannel("h:1", secure=False, sender=slow_sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q", timeout=0.01) assert excinfo.value.code() is StatusCode.DEADLINE_EXCEEDED @@ -359,7 +360,7 @@ async def boom(url, headers, body, timeout): channel = GrpcWebChannel("h:1", secure=False, sender=boom) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.UNAVAILABLE assert "ConnectionError: connection refused" in str(excinfo.value.details()) @@ -372,7 +373,7 @@ async def boom(url, headers, body, timeout): channel = GrpcWebChannel("h:1", secure=False, sender=boom) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert "ConnectionError" in str(excinfo.value.details()) @@ -382,7 +383,7 @@ async def test_empty_ok_response_hints_at_cors_expose_headers(): # whose grpc-status/grpc-message headers were stripped by CORS channel = _channel(FakeSender(status=200, headers={}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "Access-Control-Expose-Headers" in str(excinfo.value.details()) @@ -393,7 +394,7 @@ async def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): # not a CORS problem — the hint must not appear channel = _channel(FakeSender(status=200, headers={"grpc-status": "0"}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "Access-Control-Expose-Headers" not in str(excinfo.value.details()) @@ -404,7 +405,7 @@ async def test_message_frame_without_grpc_status_is_internal_not_success(): # dropped the trailer frame) must be an error, never a fabricated success channel = _channel(FakeSender(status=200, headers={}, body=_frame(b"reply-bytes"))) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "missing grpc-status" in str(excinfo.value.details()) @@ -425,7 +426,7 @@ def test_stream_stream_error_recommends_insert_many_only(): # only one supported under WASM), so the error must not recommend them channel = _channel(FakeSender()) mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) - with raises(RuntimeError) as excinfo: + with pytest.raises(RuntimeError) as excinfo: mc(request_iterator=iter([]), timeout=5, metadata=None) assert "insert_many" in str(excinfo.value) for sync_only in ("dynamic", "fixed_size", "rate_limit"): @@ -436,7 +437,7 @@ async def test_malformed_frame_maps_to_internal(): # A 3-byte body cannot contain even a 5-byte frame header -> framing ValueError. channel = _channel(FakeSender(body=b"\x00\x00\x00")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL @@ -445,7 +446,7 @@ async def test_malformed_grpc_status_maps_to_internal(): body = _frame(b"grpc-status:notanint\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL @@ -465,8 +466,9 @@ def test_body_excerpt_empty_and_non_printable(): assert _body_excerpt(b"ok\x00\x01") == "ok" -def test_encode_timeout_stays_within_eight_digits(): - cases = [ +@pytest.mark.parametrize( + "seconds,expected", + [ (None, None), (float("inf"), None), (float("nan"), None), @@ -481,13 +483,14 @@ def test_encode_timeout_stays_within_eight_digits(): (1e10, None), # would need hours, which transcoders reject above 8H: no deadline (1e15, None), (1e308, None), # finite, but *1000 overflows to infinity: must not raise - ] - for seconds, expected in cases: - encoded = _encode_timeout(seconds) - assert encoded == expected, (seconds, encoded, expected) - if encoded is not None: - assert len(encoded) <= 9, encoded # 8 digits + unit - assert not encoded.endswith("H"), encoded + ], +) +def test_encode_timeout_stays_within_eight_digits(seconds, expected): + encoded = _encode_timeout(seconds) + assert encoded == expected + if encoded is not None: + assert len(encoded) <= 9 # 8 digits + unit + assert not encoded.endswith("H") async def test_infinite_timeout_sends_no_deadline(): @@ -517,48 +520,52 @@ async def test_huge_timeout_uses_minutes_then_no_deadline(): assert sender.calls[2][3] is None # no client-side wait either -async def test_crlf_in_metadata_rejected(): +@pytest.mark.parametrize("bad", ["val\r\nx-injected: evil", "val\nx", "v\0"]) +async def test_crlf_in_metadata_rejected(bad): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - for bad in ("val\r\nx-injected: evil", "val\nx", "v\0"): - with raises(ValueError, contains="Illegal character"): - await mc(b"q", metadata=[("x-key", bad)]) - with raises(ValueError, contains="Illegal character"): - await mc(b"q", metadata=[("x-key\r\n", "v")]) + with pytest.raises(ValueError, match="Illegal character"): + await mc(b"q", metadata=[("x-key", bad)]) + with pytest.raises(ValueError, match="Illegal character"): + await mc(b"q", metadata=[("x-key\r\n", "v")]) assert sender.calls == [] -async def _unavailable_details(path_prefix, platform): +async def _unavailable_details(monkeypatch, path_prefix, platform): async def boom(url, headers, body, timeout): raise ConnectionError("Failed to fetch") - with patched(sys, "platform", platform): - channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) - mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: - await mc(b"q") + monkeypatch.setattr(sys, "platform", platform) + channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAVAILABLE return excinfo.value.details() -async def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix(): +async def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix( + monkeypatch, +): # the connect helpers always set the prefix under Emscripten, so a prefix-less channel # here means hand-built ConnectionParams; the error must say what to do instead - details = await _unavailable_details(path_prefix="", platform="emscripten") + details = await _unavailable_details(monkeypatch, path_prefix="", platform="emscripten") assert "grpc_path_prefix='/v1/grpc-web'" in details assert "1.38.3" in details assert "connect helpers" in details -async def test_unavailable_with_path_prefix_has_no_prefix_hint(): - details = await _unavailable_details(path_prefix="/v1/grpc-web", platform="emscripten") +async def test_unavailable_with_path_prefix_has_no_prefix_hint(monkeypatch): + details = await _unavailable_details( + monkeypatch, path_prefix="/v1/grpc-web", platform="emscripten" + ) assert "no grpc_path_prefix" not in details -async def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(): +async def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(monkeypatch): # off Emscripten an empty prefix against a transcoder is the normal configuration - details = await _unavailable_details(path_prefix="", platform="linux") + details = await _unavailable_details(monkeypatch, path_prefix="", platform="linux") assert "no grpc_path_prefix" not in details @@ -577,19 +584,21 @@ async def test_path_prefix_prepended_to_url(): assert sender.calls[0][0] == "http://example.com:8090/grpc-web/weaviate.v1.Weaviate/Search" -async def test_path_prefix_normalized_in_url(): - cases = [ +@pytest.mark.parametrize( + "raw,expected_url", + [ ("grpc-web", "http://h:1/grpc-web/svc/M"), ("/grpc-web/", "http://h:1/grpc-web/svc/M"), ("/a/b", "http://h:1/a/b/svc/M"), ("", "http://h:1/svc/M"), - ] - for raw, expected_url in cases: - sender = FakeSender(body=_ok_response(b"r")) - channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) - mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - await mc(b"q") - assert sender.calls[0][0] == expected_url, raw + ], +) +async def test_path_prefix_normalized_in_url(raw, expected_url): + sender = FakeSender(body=_ok_response(b"r")) + channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + await mc(b"q") + assert sender.calls[0][0] == expected_url def test_shim_factory_extracts_path_prefix_option(): diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py index 5398a825a..ab2da4bb7 100644 --- a/test/test_wasm_compat.py +++ b/test/test_wasm_compat.py @@ -6,7 +6,10 @@ """ import asyncio +import pathlib +import subprocess import sys +import textwrap import grpc import pytest @@ -288,3 +291,105 @@ def test_an_explicit_local_grpc_port_is_warned_about_but_the_default_is_not(emsc with pytest.warns(UserWarning, match="Con006"): client = weaviate.use_async_with_local(port=8080, grpc_port=8081) _assert_grpc_rides_rest(client) + + +# --- the single-import hook (weaviate/__init__.py) ------------------------------------ +# +# The hook fires on sys.platform == "emscripten" and (via the companion's bootstrap) +# replaces sys.modules['grpc'] process-wide, so each scenario runs in a fresh subprocess +# with the platform faked before `import weaviate`. The success path — a bare import +# that bootstraps the real companion — needs real Pyodide and runs in +# ci/pyodide-e2e/units.mjs; the hook's other branches are plain CPython logic and are +# pinned here. + +_REPO_ROOT = str(pathlib.Path(__file__).resolve().parents[1]) + +# CPython derives the _sysconfigdata module name from sys.platform on first use, so a +# faked platform breaks any later sysconfig lookup (pydantic imports zoneinfo, which +# calls sysconfig.get_config_var). Prime the cache before faking. +_PRIME_SYSCONFIG = """ +import sysconfig + +sysconfig.get_config_vars() +""" + + +def _run_hook_scenario( + body: str, *, prelude: str = "", path_entry: str = _REPO_ROOT, no_site: bool = False +) -> subprocess.CompletedProcess: + # -I -S: skip site-packages entirely (plain -I still processes the venv's .pth + # files), so nothing pip-installed is importable — only stdlib plus `path_entry`. + interp = [sys.executable, "-I", "-S"] if no_site else [sys.executable] + script = f"import sys\nsys.path.insert(0, {path_entry!r})\n" + prelude + textwrap.dedent(body) + return subprocess.run([*interp, "-c", script], capture_output=True, text=True) + + +def test_bare_import_without_companion_raises_clear_import_error() -> None: + # No site-packages, so neither weaviate_client_web nor grpcio is importable; the repo + # root goes on sys.path so the weaviate package itself is still found. + result = _run_hook_scenario( + """ + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert "weaviate-client-web" in str(e), str(e) + assert "weaviate-client[grpc-web]" in str(e), str(e) + assert "WebAssembly/Pyodide" in str(e), str(e) + print("OK") + else: + raise AssertionError("expected ImportError without the companion") + """, + no_site=True, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_grpc_present_falls_through_silently() -> None: + # Companion blocked but a real grpc IS importable (grpcio in the dev env): the hook + # must fall through and leave the normal import path untouched. + result = _run_hook_scenario( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + sys.modules["weaviate_client_web"] = None # makes its import raise ImportError + + import weaviate + import grpc + + assert not getattr(grpc, "__weaviate_client_web_shim__", False) + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_broken_companion_surfaces_its_own_error(tmp_path) -> None: + # An INSTALLED companion whose import fails (here: a missing dependency of its own) + # must raise that error, not the install hint — the hint would send the user to + # reinstall a package that is already there. + fake_pkg = tmp_path / "weaviate_client_web" + fake_pkg.mkdir() + (fake_pkg / "__init__.py").write_text( + "raise ModuleNotFoundError(\"No module named 'anyio'\", name='anyio')\n" + ) + result = _run_hook_scenario( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert e.name == "anyio", (e.name, str(e)) + assert "anyio" in str(e), str(e) + assert "grpc-web" not in str(e), str(e) + print("OK") + else: + raise AssertionError("expected the companion's own ImportError to surface") + """, + path_entry=str(tmp_path), + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout From c98ca479bef1bb11335f207e9eb1e26434f6796a Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:04:34 +0200 Subject: [PATCH 08/12] ci(grpc-web): publish the companion wheel alongside the base client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release job builds the weaviate-client-web wheel into dist and asserts both packages carry the same version on the artifacts actually uploaded — the companion's weaviate-client== pin makes a base-only release leave the [grpc-web] extra unresolvable. The companion is wheel-only: its setup.py resolves the lockstep version from git tags, which an unpacked sdist would not have. The wheel artifact for the GitHub release includes it too. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- .github/workflows/main.yaml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index da9ea3302..d5765e6ec 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -357,8 +357,10 @@ jobs: cache: 'pip' # caching pip dependencies - name: Install dependencies run: pip install -r requirements-test.txt -r requirements-devel.txt - - name: Build a binary wheel - run: python -m build + - name: Build binary wheels (base client + grpc-web companion) + run: | + python -m build + python -m build --wheel --outdir dist packages/web - name: Create Wheel Artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -429,9 +431,23 @@ jobs: cache: 'pip' # caching pip dependencies - name: Install dependencies run: pip install -r requirements-devel.txt - - name: Build a binary wheel - run: python -m build - - name: Publish distribution 📦 to PyPI on new tags + - name: Build distributions (base client + grpc-web companion) + # The companion is wheel-only: its setup.py resolves the lockstep version from + # the repository's git tags, which an unpacked sdist would not have — and + # micropip consumes wheels only anyway. + run: | + python -m build + python -m build --wheel --outdir dist packages/web + - name: Assert both packages carry the same version + # weaviate-client-web pins weaviate-client== at build time, so + # a base-only release would leave the [grpc-web] extra unresolvable. This gate + # runs on the artifacts that are actually uploaded. + run: | + base=$(basename dist/weaviate_client-*.whl); base=${base#weaviate_client-}; base=${base%%-*} + web=$(basename dist/weaviate_client_web-*.whl); web=${web#weaviate_client_web-}; web=${web%%-*} + echo "weaviate-client=$base weaviate-client-web=$web" + test "$base" = "$web" + - name: Publish distributions 📦 to PyPI on new tags if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: From 568d11b99874b4bf48537761e47df33dabf15037 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:04:36 +0200 Subject: [PATCH 09/12] ci(grpc-web): pin a metadata-coherent pytest / pytest-asyncio pair pytest-asyncio 0.25.3 declares pytest<9,>=8.2, so pin pytest 8.4.2 instead of the bundled 9.0.2 rather than depending on micropip tolerating the conflict. pytest-asyncio stays on 0.25.x: the asyncio.Runner-based 1.x fails under JSPI stack switching, while 0.25.x's run_until_complete-based execution works. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- ci/pyodide-e2e/units.mjs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/pyodide-e2e/units.mjs b/ci/pyodide-e2e/units.mjs index f333080a7..68721f36b 100644 --- a/ci/pyodide-e2e/units.mjs +++ b/ci/pyodide-e2e/units.mjs @@ -93,7 +93,13 @@ console.log( `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, ); const micropip = pyodide.pyimport("micropip"); -await micropip.install(["pytest==9.0.2", "pytest-asyncio==0.25.3"]); +// A metadata-coherent pair: pytest-asyncio 0.25.3 declares pytest<9,>=8.2. Its +// run_until_complete-based execution stack-switches correctly under JSPI, unlike the +// asyncio.Runner-based pytest-asyncio 1.x, whose async tests fail here. (The Pyodide +// distribution bundles pytest 9 next to pytest-asyncio 0.25.3, contradicting that +// constraint — micropip tolerates it, but there is no reason to depend on its +// leniency, so both are pinned from PyPI.) +await micropip.install(["pytest==8.4.2", "pytest-asyncio==0.25.3"]); pyodide.FS.mkdirTree("/units"); pyodide.mountNodeFS("/units", testsDir); From 1e9ad57ef967c5f4ff9b16367170988f28fba663 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:14:02 +0200 Subject: [PATCH 10/12] fix(grpc-web): address review findings on framing, headers, 405 and zero timeouts - split_response accepts exactly one uncompressed trailer frame: a second trailer could overwrite the first one's grpc-status (turning an error into a fabricated OK), and flag 0x81 (compressed trailer) was parsed as a plain trailer instead of being rejected. - Call metadata is folded before the grpc-web protocol headers are set, so additional_headers can no longer replace content-type/accept/x-grpc-web/ x-user-agent and break every RPC. - HTTP 405 maps to UNIMPLEMENTED like 404: both mean an HTTP route answered instead of the grpc-web endpoint, and the base client's wrong-path diagnosis keys on UNIMPLEMENTED. - A zero read timeout is an immediate deadline, matching native httpx and the package's own grpc-timeout encoding, instead of disabling the deadline and letting a stalled request hang. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- .../web/src/weaviate_client_web/_channel.py | 23 +++++++++++----- .../web/src/weaviate_client_web/_framing.py | 15 +++++++++-- .../src/weaviate_client_web/_httpx_fetch.py | 9 ++++--- packages/web/tests/test_framing.py | 14 ++++++++++ packages/web/tests/test_httpx_fetch.py | 13 +++++----- packages/web/tests/test_transport.py | 26 +++++++++++++++++++ 6 files changed, 82 insertions(+), 18 deletions(-) diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py index 64e6d06be..aa8d15d13 100644 --- a/packages/web/src/weaviate_client_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -195,13 +195,18 @@ async def _unary( metadata: Any, timeout: Optional[float], ) -> Any: - headers: Dict[str, str] = { - "content-type": "application/grpc-web+proto", - "accept": "application/grpc-web+proto", - "x-grpc-web": "1", - "x-user-agent": "weaviate-client-web", - } + headers: Dict[str, str] = {} _fold_metadata(headers, metadata) + # Set after folding: additional_headers reaches RPCs as call metadata, and a + # caller's Content-Type/accept must never replace the grpc-web protocol fields. + headers.update( + { + "content-type": "application/grpc-web+proto", + "accept": "application/grpc-web+proto", + "x-grpc-web": "1", + "x-user-agent": "weaviate-client-web", + } + ) grpc_timeout = _encode_timeout(timeout) if grpc_timeout is None: timeout = None # None / non-finite: no deadline, server- or client-side @@ -428,13 +433,17 @@ def _non_grpc_web_error( def _status_from_http(http_status: int) -> StatusCode: """Map an HTTP status to a gRPC status when no grpc-status is present. - Mirrors the grpc-web spec's HTTP-to-gRPC code mapping. + Mirrors the grpc-web spec's HTTP-to-gRPC code mapping, plus 405 -> UNIMPLEMENTED, + which the spec leaves unmapped: method-not-allowed means an HTTP route answered + instead of the grpc-web endpoint — the same wrong-path condition as a 404, and the + base client's diagnosis (weaviate.exceptions) keys on UNIMPLEMENTED for both. """ return { 400: StatusCode.INTERNAL, 401: StatusCode.UNAUTHENTICATED, 403: StatusCode.PERMISSION_DENIED, 404: StatusCode.UNIMPLEMENTED, + 405: StatusCode.UNIMPLEMENTED, 429: StatusCode.UNAVAILABLE, 502: StatusCode.UNAVAILABLE, 503: StatusCode.UNAVAILABLE, diff --git a/packages/web/src/weaviate_client_web/_framing.py b/packages/web/src/weaviate_client_web/_framing.py index 7e6a3dfc8..66515a244 100644 --- a/packages/web/src/weaviate_client_web/_framing.py +++ b/packages/web/src/weaviate_client_web/_framing.py @@ -82,13 +82,24 @@ def parse_trailers(raw: bytes) -> Dict[str, str]: def split_response(body: bytes) -> Tuple[List[bytes], Dict[str, str]]: - """Split a grpc-web response body into message payloads and trailers.""" + """Split a grpc-web response body into message payloads and trailers. + + Exactly one uncompressed trailer frame is accepted: a second trailer could + overwrite the first one's ``grpc-status`` (turning an error into a fabricated OK), + so it must be rejected, not merged. + """ messages: List[bytes] = [] trailers: Dict[str, str] = {} seen_trailer = False for flag, payload in iter_frames(body): if flag & _FLAG_TRAILER: - trailers.update(parse_trailers(payload)) + if flag & _FLAG_COMPRESSED: + raise FrameError( + "compressed grpc-web trailer frames are not supported by this transport" + ) + if seen_trailer: + raise FrameError("second trailer frame in a grpc-web response") + trailers = parse_trailers(payload) seen_trailer = True elif flag & _FLAG_COMPRESSED: raise FrameError( diff --git a/packages/web/src/weaviate_client_web/_httpx_fetch.py b/packages/web/src/weaviate_client_web/_httpx_fetch.py index b9d400465..c284af74a 100644 --- a/packages/web/src/weaviate_client_web/_httpx_fetch.py +++ b/packages/web/src/weaviate_client_web/_httpx_fetch.py @@ -87,10 +87,13 @@ def _pick_timeout(request: httpx.Request) -> Optional[float]: def _abort_signal_ms(timeout: Optional[float]) -> Optional[int]: """Milliseconds for ``AbortSignal.timeout``; ``None`` means no client-side deadline. - Zero, negative and non-finite timeouts all mean "no deadline". Rounded up so a - sub-millisecond timeout never becomes an immediate abort. + Absent, negative and non-finite timeouts mean "no deadline". Zero is an immediate + deadline — native httpx times a ``read=0`` request out at once, and this package's + own grpc-timeout encoding treats zero the same way, so the fetch path must not + silently turn the same configuration into an indefinite wait. Positive values are + rounded up so a sub-millisecond timeout never becomes immediate. """ - if timeout is None or not math.isfinite(timeout) or timeout <= 0: + if timeout is None or not math.isfinite(timeout) or timeout < 0: return None if timeout >= _MAX_ABORT_SIGNAL_MS / 1000: # compared before the multiplication below, which overflows to infinity for diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py index 8054dbf94..5aef2375c 100644 --- a/packages/web/tests/test_framing.py +++ b/packages/web/tests/test_framing.py @@ -120,3 +120,17 @@ def test_compressed_message_frame_rejected(): body = _frame(b"x", 0x01) with pytest.raises(FrameError, match="compressed"): split_response(body) + + +def test_second_trailer_frame_rejected(): + # a later trailer must not overwrite the first one's grpc-status — an error could + # be laundered into a fabricated OK; the spec allows exactly one trailer frame + body = _frame(b"a") + _frame(b"grpc-status:7\r\n", 0x80) + _frame(b"grpc-status:0\r\n", 0x80) + with pytest.raises(FrameError, match="second trailer frame"): + split_response(body) + + +def test_compressed_trailer_frame_rejected(): + body = _frame(b"grpc-status:0\r\n", 0x81) + with pytest.raises(FrameError, match="compressed"): + split_response(body) diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py index c1833d74c..eb0c04362 100644 --- a/packages/web/tests/test_httpx_fetch.py +++ b/packages/web/tests/test_httpx_fetch.py @@ -274,19 +274,20 @@ async def test_missing_js_module_degrades_to_no_signal(fake_pyfetch, missing_js) assert "signal" not in fake_pyfetch.calls[0] -async def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): - # an explicit read=0 must not fall through to the 5s connect timeout, nor become an - # immediate AbortSignal.timeout(0) +async def test_zero_timeout_is_an_immediate_deadline(fake_pyfetch, fake_abort_signal): + # native httpx times a read=0 request out at once; the WASM transport must not + # silently turn the same configuration into an indefinite wait (and it must not + # fall through to the 5s connect timeout either) await _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) - assert fake_abort_signal.timeouts == [] - assert "signal" not in fake_pyfetch.calls[0] + assert fake_abort_signal.timeouts == [0] + assert fake_pyfetch.calls[0]["signal"] == "signal-0" @pytest.mark.parametrize( "timeout,expected_ms", [ (None, None), - (0, None), + (0, 0), # immediate deadline, matching native httpx read=0 (-1, None), (float("inf"), None), (float("nan"), None), diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index b063767ee..b7d066c2e 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -206,6 +206,8 @@ async def test_405_names_the_wrong_prefix(): err = await _details_of( 405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}' ) + # UNIMPLEMENTED is what the base client's wrong-path diagnosis keys on for 404/405 + assert err.code() is StatusCode.UNIMPLEMENTED assert err.details().startswith("HTTP 405 ") assert "path prefix" in err.details() assert "/v1/grpc-web" in err.details() @@ -624,3 +626,27 @@ async def test_set_sender_overrides_default(): finally: # restore the real default so other tests are unaffected set_sender(pyfetch_sender) + + +async def test_metadata_cannot_replace_protocol_headers(): + # additional_headers reaches RPCs as call metadata; a caller setting Content-Type + # (reasonable for the REST side) must not break the grpc-web framing contract + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + await mc( + b"q", + metadata=[ + ("content-type", "application/json"), + ("accept", "application/json"), + ("x-grpc-web", "0"), + ("x-user-agent", "custom"), + ("x-custom", "kept"), + ], + ) + headers = sender.calls[0][1] + assert headers["content-type"] == "application/grpc-web+proto" + assert headers["accept"] == "application/grpc-web+proto" + assert headers["x-grpc-web"] == "1" + assert headers["x-user-agent"] == "weaviate-client-web" + assert headers["x-custom"] == "kept" From e4a24b0c36bd628e987e1dbc519b6c990bb844aa Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:17:35 +0200 Subject: [PATCH 11/12] refactor(grpc-web): move the grpc-web server constants into weaviate.connect GRPC_WEB_MIN_SERVER_VERSION and GRPC_WEB_SERVER_PATH_PREFIX describe the server's connectivity contract, so they now live in weaviate.connect.base next to MAX_GRPC_MESSAGE_LENGTH. WeaviateGRPCUnavailableError fetches them with a call-time import because weaviate.connect imports this module. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CCHAmyQF69W7K9CTEzCpEd --- packages/web/src/weaviate_client_web/_channel.py | 2 +- weaviate/connect/base.py | 4 ++++ weaviate/connect/helpers.py | 3 +-- weaviate/exceptions.py | 13 +++++++------ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py index aa8d15d13..efbb13b6d 100644 --- a/packages/web/src/weaviate_client_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -345,7 +345,7 @@ def _body_excerpt(body: bytes, limit: int = _BODY_EXCERPT_LIMIT) -> str: def _no_path_prefix_hint() -> str: # Lazy import: this module is imported while ``weaviate/__init__`` is still # bootstrapping the shim under Emscripten. - from weaviate.exceptions import GRPC_WEB_MIN_SERVER_VERSION, GRPC_WEB_SERVER_PATH_PREFIX + from weaviate.connect.base import GRPC_WEB_MIN_SERVER_VERSION, GRPC_WEB_SERVER_PATH_PREFIX return ( "(no grpc_path_prefix set — under WebAssembly the connect helpers route gRPC to " diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index 82c8b3851..a44d1edeb 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -19,6 +19,10 @@ JSONPayload = Union[Mapping[str, Any], Sequence[Any]] TIMEOUT_TYPE_RETURN = Tuple[NUMBER, NUMBER] MAX_GRPC_MESSAGE_LENGTH = 104858000 # 10mb, needs to be synchronized with GRPC server +# first Weaviate release that serves grpc-web on the REST port +GRPC_WEB_MIN_SERVER_VERSION = "1.38.3" +# the base path Weaviate serves grpc-web on +GRPC_WEB_SERVER_PATH_PREFIX = "/v1/grpc-web" def _grpc_web_shim_active() -> bool: diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index 80f81d711..45a67e3c7 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -16,9 +16,8 @@ ) from weaviate.client import WeaviateAsyncClient, WeaviateClient from weaviate.config import AdditionalConfig -from weaviate.connect.base import ConnectionParams, ProtocolParams +from weaviate.connect.base import GRPC_WEB_SERVER_PATH_PREFIX, ConnectionParams, ProtocolParams from weaviate.embedded import WEAVIATE_VERSION, EmbeddedOptions -from weaviate.exceptions import GRPC_WEB_SERVER_PATH_PREFIX from weaviate.util import docstring_deprecated from weaviate.validator import _validate_input, _ValidateArgument from weaviate.warnings import _Warnings diff --git a/weaviate/exceptions.py b/weaviate/exceptions.py index d046e776d..26bca2f85 100644 --- a/weaviate/exceptions.py +++ b/weaviate/exceptions.py @@ -329,12 +329,6 @@ def _grpc_status_of( return None, None -# first Weaviate release that serves grpc-web on the REST port -GRPC_WEB_MIN_SERVER_VERSION = "1.38.3" -# the base path Weaviate serves grpc-web on -GRPC_WEB_SERVER_PATH_PREFIX = "/v1/grpc-web" - - class WeaviateGRPCUnavailableError(WeaviateBaseError): """Is raised when a gRPC-backed query is made with no gRPC connection present.""" @@ -354,6 +348,13 @@ def __init__( ) if grpc_path_prefix: + # local import: weaviate.connect imports this module at import time, so a + # module-level import here would be circular + from weaviate.connect.base import ( + GRPC_WEB_MIN_SERVER_VERSION, + GRPC_WEB_SERVER_PATH_PREFIX, + ) + # grpc-web shares the REST host:port under a base path: there is no separate # gRPC port to open, and REST already worked against this endpoint, so no # firewall/wrong-port advice here From 0ce4cd4928a732086bb7612edc465d1979a8cff7 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Fri, 25 Sep 2026 09:29:22 +0200 Subject: [PATCH 12/12] docs(grpc-web): tighten comments, docstrings and the README Comment- and docstring-only pass over the grpc-web transport: one term per concept (Pyodide for the runtime, Emscripten only for sys.platform, weaviate-client-web for the package, "grpc-web path prefix"), module and harness docstrings cut to what a maintainer needs, wrong rationales fixed (the grpcio fallback, the constants _channel imports), and the README's CORS section rewritten to what core actually serves plus the Weaviate Cloud "Allow all CORS origins" setting. No code, message or metadata changes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CCHAmyQF69W7K9CTEzCpEd --- .github/workflows/main.yaml | 19 +- ci/pyodide-e2e/e2e.py | 20 +- ci/pyodide-e2e/run.mjs | 26 +-- ci/pyodide-e2e/units.mjs | 31 +--- packages/web/README.md | 171 +++++++----------- packages/web/setup.py | 14 +- .../web/src/weaviate_client_web/__init__.py | 41 +---- .../web/src/weaviate_client_web/_channel.py | 59 ++---- .../web/src/weaviate_client_web/_framing.py | 30 +-- .../src/weaviate_client_web/_httpx_fetch.py | 68 ++----- .../web/src/weaviate_client_web/_sender.py | 8 +- packages/web/src/weaviate_client_web/_shim.py | 64 ++----- packages/web/tests/conftest.py | 9 +- packages/web/tests/test_framing.py | 7 +- packages/web/tests/test_httpx_fetch.py | 33 +--- packages/web/tests/test_shim_install.py | 13 +- packages/web/tests/test_transport.py | 53 ++---- proto_test/test_proto.py | 22 +-- test/test_connection_params.py | 3 +- test/test_wasm_compat.py | 53 ++---- weaviate/__init__.py | 4 +- weaviate/connect/base.py | 30 +-- weaviate/connect/helpers.py | 42 ++--- weaviate/connect/v4.py | 6 +- weaviate/exceptions.py | 10 +- weaviate/proto/v1/__init__.py | 6 +- 26 files changed, 242 insertions(+), 600 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d5765e6ec..266224164 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -109,8 +109,7 @@ jobs: name: Run Pyodide (WASM) Unit + e2e Tests runs-on: ubuntu-latest timeout-minutes: 15 - # No Python matrix: the pinned Pyodide bundle fixes the interpreter (see - # ci/pyodide-e2e/package.json for the exact pin). + # The interpreter comes from the pyodide pin in ci/pyodide-e2e/package.json. steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -143,10 +142,8 @@ jobs: echo "weaviate-client=$base weaviate-client-web=$web" test "$base" = "$web" - name: Run the unit suite inside Pyodide under Node - # The grpc-web package imports pyodide at module scope, so its unit tests only - # run here. Needs no running Weaviate. The JSPI flag lets pytest's synchronous - # runner await async tests via stack switching; without it the run fails - # loudly at startup (never a false green). + # weaviate-client-web imports pyodide, so its unit tests run only here. + # JSPI lets pytest run async tests. run: | npm install --prefix ci/pyodide-e2e node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist @@ -432,16 +429,14 @@ jobs: - name: Install dependencies run: pip install -r requirements-devel.txt - name: Build distributions (base client + grpc-web companion) - # The companion is wheel-only: its setup.py resolves the lockstep version from - # the repository's git tags, which an unpacked sdist would not have — and - # micropip consumes wheels only anyway. + # weaviate-client-web is wheel-only: its version comes from git tags, which an + # sdist lacks. run: | python -m build python -m build --wheel --outdir dist packages/web - name: Assert both packages carry the same version - # weaviate-client-web pins weaviate-client== at build time, so - # a base-only release would leave the [grpc-web] extra unresolvable. This gate - # runs on the artifacts that are actually uploaded. + # weaviate-client-web pins weaviate-client==; a mismatch leaves the + # [grpc-web] extra unresolvable. run: | base=$(basename dist/weaviate_client-*.whl); base=${base#weaviate_client-}; base=${base%%-*} web=$(basename dist/weaviate_client_web-*.whl); web=${web#weaviate_client_web-}; web=${web%%-*} diff --git a/ci/pyodide-e2e/e2e.py b/ci/pyodide-e2e/e2e.py index b1f03ce9c..b37eacca6 100644 --- a/ci/pyodide-e2e/e2e.py +++ b/ci/pyodide-e2e/e2e.py @@ -1,12 +1,6 @@ -"""In-Pyodide e2e for the Weaviate client over core-native grpc-web. +"""e2e suite run inside Pyodide by run.mjs, which awaits main(); prints one OK line per step. -Executed by ``run.mjs`` inside Pyodide under Node: the runner runs this module's code -(imports below install the grpc shim + fetch transport) and then awaits ``main()`` on -Pyodide's event loop. Plain asserts with one ``OK`` line per step so CI logs are -diagnosable; any failure exits nonzero. - -Deliberately not covered: browser/CORS behaviour (this runs under Node, no CORS layer) -and OIDC auth flows (anonymous access only). +Not covered: browser CORS (runs under Node) and OIDC (anonymous access only). """ import os @@ -27,9 +21,7 @@ COLL = "PyodideE2E" MT_COLL = "PyodideE2ETenants" -# Weaviate core serves grpc-web natively on the REST port under this prefix (default-on -# since 1.38.3), so no proxy sits between the client and the server. Under Emscripten the -# connect helpers route gRPC there themselves — nothing here selects it. +# Weaviate's grpc-web path prefix; the connect helpers select it under Emscripten. GRPC_WEB_PREFIX = "/v1/grpc-web" @@ -42,8 +34,7 @@ async def main() -> None: assert getattr(grpc, "__weaviate_client_web_shim__", False), ( "sys.modules['grpc'] is not the shim" ) - # REST must run through the package's own fetch transport, not Pyodide's bundled - # httpx transport (which cannot read the null body of HEAD / 204 responses). + # REST must use weaviate-client-web's fetch transport, not Pyodide's bundled one. assert weaviate_client_web.is_fetch_transport_installed(), "fetch transport not installed" assert getattr( httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False @@ -142,8 +133,7 @@ async def main() -> None: assert await coll.data.delete_by_id(last) is True assert await coll.data.exists(last) is False - # deleting a missing object answers 204 or 404 depending on the server topology; - # either way it is a body-less response the transport must handle + # a repeat delete answers 204 or 404 depending on topology; both are body-less assert isinstance(await coll.data.delete_by_id(last), bool) ok("data.delete_by_id (DELETE 204; repeat -> 204/404) = True, then bool") diff --git a/ci/pyodide-e2e/run.mjs b/ci/pyodide-e2e/run.mjs index 3342c9b1a..703d0f05b 100644 --- a/ci/pyodide-e2e/run.mjs +++ b/ci/pyodide-e2e/run.mjs @@ -1,16 +1,7 @@ -// Runs the Weaviate Python client e2e suite (e2e.py) inside Pyodide (WASM) under Node. -// -// Usage: node run.mjs -// must contain exactly the two locally-built pure wheels: -// weaviate_client-*.whl and weaviate_client_web-*.whl. +// Runs e2e.py inside Pyodide under Node against a live Weaviate. +// Usage: node run.mjs (one weaviate_client-*.whl, one weaviate_client_web-*.whl) // Env: WEAVIATE_HOST (default localhost), WEAVIATE_PORT (default 8090). -// -// The pinned `pyodide` npm package fixes the interpreter (the 314.x line bundles -// CPython 3.14), so there is no Python version matrix here. micropip installs the two -// local wheels; transitive deps resolve from the Pyodide distribution -// (pydantic/pydantic_core/cryptography ship wasm builds there — pydantic_core has no -// wasm wheel on PyPI) or from PyPI as pure wheels (protobuf), and the base client's -// `grpcio; sys_platform != "emscripten"` marker correctly skips grpcio. +// The pyodide npm pin in package.json fixes the interpreter. import { readdirSync, readFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -50,9 +41,8 @@ console.log( await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); -// anyio (needed because Pyodide's httpx recipe drops it, while authlib imports it -// directly) resolves from the companion wheel's `anyio ; sys_platform == "emscripten"` -// marker — no explicit install here, so the marker stays proven. +// anyio comes from weaviate-client-web's emscripten marker; installing it here would +// hide a broken marker. pyodide.FS.mkdirTree("/wheels"); pyodide.mountNodeFS("/wheels", wheelsDir); @@ -61,8 +51,7 @@ for (const wheel of wheels) { await micropip.install(`emfs:/wheels/${wheel}`); } -// Single-import check: the FIRST weaviate-side import in this interpreter is a bare -// `import weaviate` — the base client must bootstrap the companion (and the shim) itself. +// The first import is a bare `import weaviate`: it must install the grpc shim itself. pyodide.runPython(` import sys assert "weaviate_client_web" not in sys.modules @@ -72,8 +61,7 @@ assert getattr(sys.modules.get("grpc"), "__weaviate_client_web_shim__", False), print("OK bare 'import weaviate' bootstrapped the grpc shim") `); -// Define e2e.py's globals (imports run here, installing the grpc shim), then await -// main() on Pyodide's event loop — asyncio.run() cannot be used inside Pyodide. +// asyncio.run() is unavailable in Pyodide: load e2e.py, then await main() on Pyodide's loop. pyodide.runPython(readFileSync(resolve(here, "e2e.py"), "utf8")); try { await pyodide.runPythonAsync("await main()"); diff --git a/ci/pyodide-e2e/units.mjs b/ci/pyodide-e2e/units.mjs index 68721f36b..469a3ccee 100644 --- a/ci/pyodide-e2e/units.mjs +++ b/ci/pyodide-e2e/units.mjs @@ -1,22 +1,7 @@ -// Runs the weaviate_client_web unit suite (packages/web/tests) with pytest inside -// Pyodide (WASM) under Node, plus a bootstrap scenario in a fresh interpreter. No -// running Weaviate is needed — everything is driven through fake senders / a fake -// pyfetch. -// -// Usage: node --experimental-wasm-jspi units.mjs -// must contain exactly the two locally-built pure wheels: -// weaviate_client-*.whl and weaviate_client_web-*.whl (same layout as run.mjs). -// -// The JSPI flag is required: pytest's runner is synchronous, so async tests execute -// through run_until_complete, which needs stack switching (enableRunUntilComplete + -// a callPromising() entrypoint). Without the flag the run fails loudly at startup — -// it can never produce a false green. -// -// The package imports pyodide at module scope, so this harness is the only place its -// unit tests can run. The base client's hook logic (missing companion, broken -// companion, grpc-present fall-through) is covered by subprocess tests in -// test/test_wasm_compat.py on CPython; the bootstrap scenario below covers the one -// path that needs real Pyodide, micropip and the wheels. +// Runs packages/web/tests with pytest inside Pyodide under Node, plus a fresh-interpreter +// bootstrap check. No Weaviate needed. +// Usage: node --experimental-wasm-jspi units.mjs (same wheels as run.mjs) +// JSPI is required: pytest runs async tests through run_until_complete (stack switching). import { readdirSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -93,12 +78,8 @@ console.log( `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, ); const micropip = pyodide.pyimport("micropip"); -// A metadata-coherent pair: pytest-asyncio 0.25.3 declares pytest<9,>=8.2. Its -// run_until_complete-based execution stack-switches correctly under JSPI, unlike the -// asyncio.Runner-based pytest-asyncio 1.x, whose async tests fail here. (The Pyodide -// distribution bundles pytest 9 next to pytest-asyncio 0.25.3, contradicting that -// constraint — micropip tolerates it, but there is no reason to depend on its -// leniency, so both are pinned from PyPI.) +// pytest-asyncio 1.x (asyncio.Runner) fails under JSPI; 0.25.3 needs pytest<9, so pin +// both from PyPI. await micropip.install(["pytest==8.4.2", "pytest-asyncio==0.25.3"]); pyodide.FS.mkdirTree("/units"); pyodide.mountNodeFS("/units", testsDir); diff --git a/packages/web/README.md b/packages/web/README.md index 3c4fd46c1..2331127b2 100644 --- a/packages/web/README.md +++ b/packages/web/README.md @@ -1,16 +1,11 @@ # weaviate-client-web -A grpc-web / WebAssembly (Pyodide) transport for the -[Weaviate Python client](https://github.com/weaviate/weaviate-python-client), so the -client's **async** gRPC data path can run inside a browser (marimo notebooks, Pyodide, -WASM workers) where there is no socket and no `grpcio` wheel. +Runs the async [Weaviate Python client](https://github.com/weaviate/weaviate-python-client) +under Pyodide (browser pages, marimo notebooks, Web Workers): gRPC goes over grpc-web and +REST over `fetch`. -It is built from the same repository as `weaviate-client` and reuses its generated -protobuf stubs — it does **not** fork code generation. - -Requires Weaviate ≥ 1.38.3 (the first release to serve grpc-web natively) or a grpc-web -transcoder in front of an older server. Pyodide ≥ 0.27 recommended; verified on -Pyodide 314.0.4 (CPython 3.14). +Requires Weaviate ≥ 1.38.3 (the first release serving grpc-web) or a grpc-web transcoder +in front of an older server. Tested on Pyodide 314.0.4 (CPython 3.14). ## Installation @@ -21,47 +16,26 @@ import micropip await micropip.install("weaviate-client[grpc-web]") ``` -The extra carries a `sys_platform == "emscripten"` marker, so the same requirement is a -no-op on CPython — one requirements list works everywhere. Installing the companion -directly (`micropip.install("weaviate-client-web")`) works too: it pins -`weaviate-client` to its own exact version, so a mismatched pair can never resolve. -Both forms require the first `weaviate-client` release that ships the extra and the -Emscripten marker; against older releases the resolver fails on `grpcio`. +The extra has a `sys_platform == "emscripten"` marker, so it installs nothing on CPython. +`micropip.install("weaviate-client-web")` also works; it pins `weaviate-client` to its own +version. Both need a `weaviate-client` release that ships the extra; older releases fail to +resolve `grpcio`. -This package is defined for its environment: it imports `pyodide` at module scope and is -therefore only importable under Emscripten/Pyodide. On CPython the base client never -imports it (and the extra never installs it). +The package imports `pyodide` at module scope, so it imports only under Pyodide. ## How it works -Under Pyodide there is no `grpcio` Emscripten wheel, and `import weaviate` hard-imports -`grpc` at module load. This package installs a small pure-Python `grpc` shim into -`sys.modules` **before** `import weaviate`, which: - -- satisfies every import-time `import grpc` / `from grpc(.aio) import ...` in the base - client and its generated `*_pb2_grpc` stubs; -- provides `grpc.aio.Channel` as a real base class, so the grpc-web channel - (`GrpcWebChannel`) subclasses it and the client's `isinstance(..., grpc.aio.Channel)` - assertions pass; -- satisfies the generated v6300 stub's version gate - (`grpc.__version__` / `grpc._utilities.first_version_is_lower`). - -The `GrpcWebChannel` frames unary RPCs as grpc-web (a 5-byte header + protobuf payload) -and POSTs them via `pyodide.http.pyfetch`. Call metadata (API key / OIDC bearer) is -folded into `fetch` headers. - -The target is not configurable: under Emscripten the connect helpers -(`use_async_with_local`, `use_async_with_weaviate_cloud`, `use_async_with_custom`) pin -gRPC to the **REST** endpoint — same host, port and TLS — under Weaviate's own -`/v1/grpc-web` base path, so gRPC and REST share one origin and no proxy is needed. That -is deliberate: native gRPC cannot work under WASM at all, so grpc-web on the REST -listener is not a choice that could be wrong. The TypeScript `@weaviate/web` client makes -the same call, dropping `grpcHost`/`grpcPort`/`grpcSecure` from its options entirely. - -A grpc-web transcoder on a separate endpoint (Envoy, -[connectrpc/vanguard](https://github.com/connectrpc/vanguard-go)) is therefore not -reachable through the helpers. If you need one — e.g. in front of a Weaviate older than -1.38.3 — build the connection parameters yourself: +Under Pyodide there is no `grpcio` wheel. Importing this package puts a pure-Python `grpc` +shim in `sys.modules`, and `GrpcWebChannel` sends unary RPCs as grpc-web POSTs through +`pyodide.http.pyfetch`. Call metadata (API key, OIDC bearer) becomes fetch headers. + +Under Pyodide the connect helpers send gRPC to the REST endpoint (same host, port and TLS) +under `/v1/grpc-web`. As in the TypeScript `@weaviate/web` client, native gRPC is not +available there, so this endpoint is not configurable through the helpers. + +To use a grpc-web transcoder on another endpoint (Envoy, +[vanguard](https://github.com/connectrpc/vanguard-go)), for example in front of +Weaviate < 1.38.3, build the connection parameters yourself: ```python from weaviate import WeaviateAsyncClient @@ -71,27 +45,22 @@ client = WeaviateAsyncClient( ConnectionParams.from_params( http_host="weaviate.example.com", http_port=443, http_secure=True, grpc_host="transcoder.example.com", grpc_port=443, grpc_secure=True, - # add grpc_path_prefix="/base/path" if the transcoder is not at the root + # add grpc_path_prefix="/prefix" if the transcoder is not at the root ) ) ``` -For REST (`is_ready`, collection config, `/batch/references`, …) the package patches -`httpx.AsyncHTTPTransport` with its own `pyfetch`-based transport. It does so even on -Pyodide builds whose bundled httpx has a JS-fetch transport of its own: that transport -cannot read the null body of HEAD requests and 204 responses (`data.exists`, -`data.delete_by_id`, `tenants.exists`, …) and does not enforce the client's per-request -timeouts. +REST goes through the package's own `fetch`-based httpx transport, including on Pyodide +builds whose httpx has one (that one fails on body-less HEAD/204 responses). ## Usage -With this package installed, a plain `import weaviate` is all you need — under -Emscripten the base client imports `weaviate_client_web` itself before anything else, -which installs the shim and the fetch transport (and raises a clear error if the package -is missing). Against Weaviate ≥ 1.38.3: +With this package installed, `import weaviate` imports it first, under Pyodide only. If it +is missing, `import weaviate` raises an ImportError naming the extra. Against +Weaviate ≥ 1.38.3: ```python -import weaviate # bootstraps weaviate_client_web automatically under Emscripten +import weaviate client = weaviate.use_async_with_local(port=8080) await client.connect() # runs the gRPC health check over grpc-web @@ -99,10 +68,6 @@ collection = client.collections.get("Article") await collection.query.near_text("hello", limit=3) ``` -Nothing selects grpc-web: `use_async_with_local()`, `use_async_with_weaviate_cloud()` and -`use_async_with_custom()` all route gRPC onto the REST endpoint under `/v1/grpc-web` when -they run under Emscripten, and use native gRPC everywhere else. - ```python client = weaviate.use_async_with_weaviate_cloud( cluster_url="rAnD0mD1g1t5.something.weaviate.cloud", @@ -110,6 +75,10 @@ client = weaviate.use_async_with_weaviate_cloud( ) ``` +Weaviate Cloud: browser use requires **Allow all CORS origins** in the cluster's settings +in the Weaviate Cloud console (takes a few minutes to apply). Until it applies, the client +fails at its first REST call with a `Failed to fetch` connection error. + `use_async_with_custom()` still requires `grpc_host`/`grpc_port`/`grpc_secure` — Python cannot drop required parameters on one platform the way TypeScript drops them from a type. Pass the HTTP values; anything else is overridden with them and warned about @@ -122,37 +91,28 @@ client = weaviate.use_async_with_custom( ) ``` -Pass `headers={...}` / `auth_credentials=...` as usual for API keys, OIDC or WCD. - -Importing the companion explicitly first also works and remains the explicit form: - -```python -import weaviate_client_web # installs the grpc shim (only importable under Emscripten) -import weaviate -``` +Pass `headers={...}` / `auth_credentials=...` as usual for API keys, OIDC or Weaviate Cloud. +Importing `weaviate_client_web` before `weaviate` is equivalent. ## Supported / unsupported | Feature | Kind | Status | |----------------------------------------------------------|-----------------|--------| -| Search, Aggregate, TenantsGet, BatchObjects, BatchDelete | unary gRPC | ✅ works over grpc-web | -| Health check (`/grpc.health.v1.Health/Check`) | unary gRPC | ✅ runs on `connect()` over grpc-web | -| REST (`is_ready`, config, `/batch/references`, …) | REST | ✅ via the package's own fetch transport | -| API-key auth (`Auth.api_key`) | header | ✅ | -| OIDC auth (`client_credentials` / `client_password` / `bearer_token`) | REST | ✅ token fetch + asyncio-task refresh (no threads) | -| Bulk insert: `collection.data.insert_many()` | unary gRPC | ✅ the supported bulk path under WASM | -| `batch.stream()` / `batch.experimental()` (BatchStream) | bidi streaming | ❌ not possible over grpc-web/fetch — raises immediately; use `insert_many()` | -| `batch.dynamic()` / `fixed_size()` / `rate_limit()` | sync-client API | ❌ these only exist on the sync client, which is unsupported under WASM | -| Embedded Weaviate (`use_async_with_embedded`) | subprocess | ❌ raises "not supported under WebAssembly/Pyodide" | -| Synchronous client | — | ❌ async-only under WASM | -| Weaviate Agents: `AsyncQueryAgent` `run/ask/search` | REST | ✅ via fetch | -| Weaviate Agents: `ask_stream` / `research_stream` (SSE) | REST streaming | ⚠️ degraded: the fetch transport buffers the whole response, so events arrive only when the run completes (and long runs can hit the request timeout) | -| Weaviate Agents: sync `QueryAgent`, `TransformationAgent`, `PersonalizationAgent` | REST sync | ❌ no async flavour exists | +| Search, Aggregate, TenantsGet, BatchObjects, BatchDelete | unary gRPC | Yes, over grpc-web | +| Health check (`/grpc.health.v1.Health/Check`) | unary gRPC | Yes, on `connect()` over grpc-web | +| REST (`is_ready`, config, `/batch/references`, …) | REST | Yes, via the package's own fetch transport | +| API-key auth (`Auth.api_key`) | header | Yes | +| OIDC auth (`client_credentials` / `client_password` / `bearer_token`) | REST | Untested: token refresh runs on an asyncio task (no threads) | +| Bulk insert: `collection.data.insert_many()` | unary gRPC | Yes, the bulk path under Pyodide | +| `batch.stream()` / `batch.experimental()` (BatchStream) | bidi streaming | No: grpc-web has no bidirectional streaming; raises at once, use `insert_many()` | +| `batch.dynamic()` / `fixed_size()` / `rate_limit()` | sync-client API | No: sync client only | +| Embedded Weaviate (`use_async_with_embedded`) | subprocess | No: raises "not supported under WebAssembly/Pyodide" | +| Synchronous client | — | No: async only | ## Configuration not honored in the browser `fetch` manages connections itself, so several knobs are accepted but have no effect -under WASM: +under Pyodide: - `AdditionalConfig.proxies` / `trust_env` proxy environment variables (the browser cannot proxy fetch requests per-client), @@ -166,20 +126,26 @@ under WASM: ## CORS requirements (browsers) -Weaviate ≥ 1.38.3 serves the CORS headers below for its `/v1/grpc-web` endpoint itself, -with no configuration. Its request-header list is a **closed allowlist**: custom -`headers={...}` that are not on it fail the browser's preflight. Cross-origin -deployments that go through a grpc-web transcoder or a proxy must configure CORS there: - -- allow every request header the client sends: `content-type`, `x-grpc-web`, - `x-user-agent`, `grpc-timeout`, `x-weaviate-client`, `authorization` (when auth is - used) and `x-weaviate-cluster-url` (Weaviate Cloud); -- expose the grpc-web status headers on responses: - `Access-Control-Expose-Headers: grpc-status, grpc-message` — without this, - trailers-only error responses (e.g. a bad API key) are reported as - `INTERNAL: grpc-web response contained no message frame` instead of the real error; -- note that a CORS-blocked request is indistinguishable from a network failure in the - browser (`TypeError: Failed to fetch`), and is retried as UNAVAILABLE. +Self-hosted Weaviate ≥ 1.38.3 (unchanged through 1.40.0-rc.1) answers CORS for its +`/v1/grpc-web` endpoint as follows (for Weaviate Cloud, see [Usage](#usage)): + +- allowed origins come from `CORS_ALLOW_ORIGIN` (default `*`); +- allowed request headers are `X-Grpc-Web`, `X-User-Agent`, `Grpc-Timeout`, + `Connect-Protocol-Version`, `Connect-Timeout-Ms` and `X-Weaviate-Client`, plus everything + in `CORS_ALLOW_HEADERS`, whose default already covers `Content-Type`, `Authorization`, + `X-Weaviate-Cluster-Url` and the vendor `*-Api-Key` headers; +- exposed response headers are `Grpc-Status`, `Grpc-Message` and + `Grpc-Status-Details-Bin`. + +A custom `headers={...}` entry outside that list must be added to `CORS_ALLOW_HEADERS`, +or the browser's preflight rejects the request. A proxy or grpc-web transcoder in front of +Weaviate must allow the same origins and request headers and expose the same response +headers; without `grpc-status, grpc-message` exposed, trailers-only error responses (e.g. +a bad API key) are reported as `INTERNAL: grpc-web response contained no message frame` +instead of the real error. + +In the browser a CORS-blocked request is indistinguishable from a network failure +(`TypeError: Failed to fetch`), and is retried as UNAVAILABLE. ## Testing @@ -193,10 +159,3 @@ npm install --prefix ci/pyodide-e2e node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist # pytest unit suite, no Weaviate needed node ci/pyodide-e2e/run.mjs dist # e2e suite, needs a running Weaviate (see ci/) ``` - -`units.mjs` runs pytest over `packages/web/tests/` inside Pyodide — async tests execute -on Pyodide's event loop via JSPI stack switching, hence the Node flag — plus a -fresh-interpreter bootstrap scenario; everything is driven through fake senders and a -fake `pyfetch`. `run.mjs` runs the e2e suite against a live Weaviate. On CPython the -`conftest.py` keeps pytest from collecting these modules (they cannot import there); -the base client's import-hook branches are covered by `test/test_wasm_compat.py`. diff --git a/packages/web/setup.py b/packages/web/setup.py index e99c3f653..181715c9e 100644 --- a/packages/web/setup.py +++ b/packages/web/setup.py @@ -1,14 +1,8 @@ -"""Injects the lockstep ``weaviate-client==`` pin at build time. +"""Pins weaviate-client== at build time (the version comes from git tags). -Both packages derive their version from the repository's git tags (setuptools_scm), so -the version is only known when the wheel is built and a static ``dependencies`` list -cannot express the pin. The pin makes mismatched pairs unresolvable at install time: -the two packages share private contracts (the ``HTTP `` error-string markers, -the exception constants ``_channel`` imports), so a companion must only ever install -next to the base client it was built with. - -Consequence for releasing: every tag must publish BOTH packages — a base-only release -would leave the extra pointing at a companion whose pin no longer resolves. +The packages share private contracts (the "HTTP " details that weaviate.exceptions +matches, and the grpc-web constants _channel imports from weaviate.connect.base), so every +release tag must publish both. """ from setuptools import setup diff --git a/packages/web/src/weaviate_client_web/__init__.py b/packages/web/src/weaviate_client_web/__init__.py index d573b7af3..1ef01eedf 100644 --- a/packages/web/src/weaviate_client_web/__init__.py +++ b/packages/web/src/weaviate_client_web/__init__.py @@ -1,33 +1,8 @@ -"""grpc-web / WASM transport for the Weaviate Python client. +"""grpc-web and fetch transports for the Weaviate Python client under Pyodide. -Under Pyodide/Emscripten there is no ``grpcio`` wheel. Importing this package installs a -pure-Python ``grpc`` shim into ``sys.modules`` (and forces the pure-Python protobuf -runtime) so that the subsequent ``import weaviate`` succeeds and its async gRPC data path -runs over grpc-web (``fetch``) instead of HTTP/2 sockets; REST runs through the package's -own ``fetch``-based httpx transport. - -Usage under Pyodide against Weaviate >= 1.38.3, which serves grpc-web on its REST port -under ``/v1/grpc-web`` (with this package installed, a bare ``import weaviate`` suffices — -the base client imports this package itself under Emscripten before anything else):: - - import weaviate - - client = weaviate.use_async_with_local(port=8080) - await client.connect() - -There is nothing to select. Under Emscripten ``use_async_with_local``, -``use_async_with_weaviate_cloud`` and ``use_async_with_custom`` all pin gRPC to the REST -endpoint under ``/v1/grpc-web``, because native gRPC is impossible there — the same -contract as the TypeScript ``@weaviate/web`` client. ``use_async_with_custom`` still -requires ``grpc_host``/``grpc_port``/``grpc_secure``; give it the HTTP values, or it -warns that it overrode them. - -An explicit ``import weaviate_client_web`` before ``import weaviate`` also works and -remains the explicit form. This package is defined for its environment: it imports -``pyodide`` at module scope, so it is only importable under Emscripten/Pyodide — on -CPython the base client never imports it, and the ``weaviate-client[grpc-web]`` extra -does not install it there (platform marker). Async clients only — the synchronous client -is not supported in the browser. +Importing this package (``import weaviate`` does so under Emscripten) installs a pure-Python +``grpc`` shim and a fetch-based httpx transport. It imports only under Pyodide and supports +async clients only. """ import os @@ -55,13 +30,11 @@ def _bootstrap() -> None: if sys.platform == "emscripten": - # The pure-Python protobuf runtime always works; the upb C-extension may not be - # present. Set before ``import weaviate`` (which imports protobuf) so it takes - # effect. ``setdefault`` lets a user override it explicitly. + # upb may be missing under Pyodide; set before protobuf is imported (setdefault + # keeps a user override). os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") install() - # The REST path needs fetch too: httpx/httpcore open raw sockets, which do - # not exist under WASM. + # httpcore needs sockets, so REST also goes through fetch. install_fetch_transport() diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py index efbb13b6d..e5fbbe852 100644 --- a/packages/web/src/weaviate_client_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -1,15 +1,6 @@ -"""The grpc-web channel and multicallables. - -:class:`GrpcWebChannel` implements the small slice of the ``grpc.aio`` channel interface -that ``weaviate``'s generated stub and ``ConnectionV4`` actually use — ``unary_unary``, -``stream_stream`` and ``close`` — by framing requests as grpc-web and POSTing them via a -pluggable async sender. It subclasses the shim's ``grpc.aio.Channel`` (:class:`AioChannel`) -so the ``isinstance(..., grpc.aio.Channel)`` assertions in ``connect/v4.py`` hold. - -Only unary RPCs are supported (Search, Aggregate, TenantsGet, BatchObjects, -BatchReferences, BatchDelete, and the unary health check). ``stream_stream`` (the bidi -``BatchStream`` used by opt-in server-side batching) cannot work over grpc-web/fetch and -raises a clear error. +"""grpc-web channel: the grpc.aio channel methods the client uses. + +unary_unary POSTs grpc-web through a pluggable sender; stream_stream (BatchStream) raises. """ import asyncio @@ -23,7 +14,6 @@ from ._sender import Sender, pyfetch_sender from ._shim import AioChannel, AioRpcError, StatusCode, status_from_int -# Module-level default sender; overridable for tests. _default_sender: Sender = pyfetch_sender @@ -42,13 +32,10 @@ def get_sender() -> Sender: def _encode_timeout(seconds: Optional[float]) -> Optional[str]: - """Encode a timeout as a grpc-timeout header value; ``None`` means no deadline. + """Encode seconds as a grpc-timeout value: at most 8 digits, unit m/S/M, rounded up. - ``None``, non-finite values and anything beyond 99,999,999 minutes (~190 years) carry - no deadline. Rounds up so we never advertise a shorter deadline than requested (which - would risk premature server-side cancellation), moving to a coarser unit - (m -> S -> M) to stay within 8 digits. Hours are never emitted: grpc-web transcoders - (vanguard) reject any H value above 8H with HTTP 400. + Returns None (no deadline) for None, non-finite values and anything beyond 99,999,999 + minutes. Never uses H: vanguard rejects values above 8H. """ if seconds is None or not math.isfinite(seconds): return None @@ -135,9 +122,7 @@ def __init__(self, path: str) -> None: self._path = path def __call__(self, *args: Any, **kwargs: Any) -> Any: - # NOTE: do not recommend batch.dynamic()/fixed_size()/rate_limit() here — those - # are sync-client-only APIs and do not exist on the async client, which is the - # only client supported under WASM. + # batch.dynamic()/fixed_size()/rate_limit() are sync-only, so not suggested here. raise RuntimeError( f"Bidirectional streaming RPC {self._path!r} (server-side batching / " "BatchStream) is not supported over grpc-web/fetch. Use " @@ -146,7 +131,7 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: class GrpcWebChannel(AioChannel): - """grpc-web/fetch implementation of the async grpc channel slice the client uses.""" + """grpc.aio channel that sends unary RPCs as grpc-web.""" def __init__( self, @@ -261,9 +246,8 @@ def _handle_response( deserialize: Callable[[bytes], Any], url: str = "", ) -> Any: - # A frame-parse failure must never decide the outcome by itself: real error - # responses carry non-grpc-web bodies (Weaviate's 404 JSON, an nginx page), and - # the HTTP status, URL and the server's own text must survive into the error. + # Real error responses carry non-grpc-web bodies (404 JSON, proxy HTML); keep + # status, URL and body in the error. messages: List[bytes] = [] trailers: Dict[str, str] = {} frame_error: Optional[BaseException] = None @@ -282,8 +266,8 @@ def _handle_response( message = urllib.parse.unquote(raw_message) if raw_status is None: - # No grpc-status anywhere AND either a non-200 or a body that is not - # grpc-web framing: a gRPC service did not answer this request at all. + # no grpc-status anywhere and either a non-200 or a body that is not + # grpc-web framing: a gRPC service did not answer this request if http_status != 200 or frame_error is not None: raise _frame_error_to_rpc(http_status, url, body, frame_error) if messages: @@ -377,12 +361,9 @@ def _non_grpc_web_error( body: bytes, frame_error: Optional[BaseException] = None, ) -> AioRpcError: - """Build the error for a response that is not a usable grpc-web response. + """Error for a response that is not usable grpc-web. - Details always begin with ``HTTP `` and carry the request URL plus a body - excerpt (``weaviate/connect`` matches on that prefix). The status alone rarely - separates "endpoint missing" from "proxy misconfigured"; the server's own body - text usually does. + Details start with "HTTP ", then the URL and a body excerpt. """ truncated = isinstance(frame_error, TruncatedFrameError) what = "not a grpc-web response" @@ -396,8 +377,7 @@ def _non_grpc_web_error( parts = [f"HTTP {http_status} from {url or ''}: {what}."] if http_status == 404: - # Two candidate causes, and the channel cannot tell them apart (it does not know - # the server version) — name both rather than guess. + # either cause is possible; the channel does not know the server version parts.append( "The grpc-web endpoint does not exist at that path: either this Weaviate " "server predates 1.38.3, the first release to serve grpc-web natively, or " @@ -405,7 +385,7 @@ def _non_grpc_web_error( "Weaviate's native prefix is '/v1/grpc-web'." ) elif http_status == 405: - # A 405 can only come from an existing HTTP route: the prefix points at one. + # a 405 comes only from an existing HTTP route: the prefix points at one parts.append( "An HTTP route answered instead of the grpc-web endpoint (method not " "allowed): the configured grpc-web path prefix is wrong. Weaviate's native " @@ -431,12 +411,9 @@ def _non_grpc_web_error( def _status_from_http(http_status: int) -> StatusCode: - """Map an HTTP status to a gRPC status when no grpc-status is present. + """Map an HTTP status to gRPC per the grpc-web spec. - Mirrors the grpc-web spec's HTTP-to-gRPC code mapping, plus 405 -> UNIMPLEMENTED, - which the spec leaves unmapped: method-not-allowed means an HTTP route answered - instead of the grpc-web endpoint — the same wrong-path condition as a 404, and the - base client's diagnosis (weaviate.exceptions) keys on UNIMPLEMENTED for both. + Adds 405 -> UNIMPLEMENTED: an HTTP route answered, so the path is wrong, as for 404. """ return { 400: StatusCode.INTERNAL, diff --git a/packages/web/src/weaviate_client_web/_framing.py b/packages/web/src/weaviate_client_web/_framing.py index 66515a244..8ee2e61b9 100644 --- a/packages/web/src/weaviate_client_web/_framing.py +++ b/packages/web/src/weaviate_client_web/_framing.py @@ -1,17 +1,7 @@ -r"""grpc-web binary framing (``application/grpc-web+proto``). +"""grpc-web binary framing (application/grpc-web+proto). -A grpc-web message frame is a 1-byte flag + 4-byte big-endian length + payload: - - +--------+----------------+----------------------+ - | flag | length (uint32)| payload (length bytes)| - +--------+----------------+----------------------+ - -The flag's high bit (``0x80``) marks a trailer frame whose payload is an -HTTP/1-style header block (``grpc-status: 0\\r\\ngrpc-message: ...``). The low bit -(``0x01``) marks a compressed message, which this transport neither sends nor -accepts. A unary grpc-web response body is one or more message frames followed by -exactly one trailer frame (or a "trailers-only" response carrying the status in -the HTTP headers, handled by the caller). +Each frame is a 1-byte flag, a 4-byte big-endian length and the payload. Flag 0x80 marks +the trailer frame (an HTTP/1-style header block); 0x01 (compressed) is not supported. """ import struct @@ -62,13 +52,9 @@ def iter_frames(buf: bytes) -> Iterator[Tuple[int, bytes]]: def parse_trailers(raw: bytes) -> Dict[str, str]: - """Parse a trailer frame payload into a lower-cased header dict. + """Parse a trailer payload into a lower-cased dict; accepts CRLF or LF line ends. - Decoded leniently on both sides of the colon: a proxy that does not percent-encode - ``grpc-message``, or a server error quoting a UTF-8 collection / tenant / property - name, puts raw non-ASCII bytes in the trailer, and one odd key must not discard the - ``grpc-status`` travelling with it. Lines are CRLF-terminated by spec; bare LF is - accepted. + Undecodable bytes are replaced, so an odd grpc-message never drops grpc-status. """ out: Dict[str, str] = {} for line in raw.split(b"\n"): @@ -82,11 +68,9 @@ def parse_trailers(raw: bytes) -> Dict[str, str]: def split_response(body: bytes) -> Tuple[List[bytes], Dict[str, str]]: - """Split a grpc-web response body into message payloads and trailers. + """Split a grpc-web body into message payloads and trailers. - Exactly one uncompressed trailer frame is accepted: a second trailer could - overwrite the first one's ``grpc-status`` (turning an error into a fabricated OK), - so it must be rejected, not merged. + Rejects a second trailer frame, which could overwrite an error grpc-status. """ messages: List[bytes] = [] trailers: Dict[str, str] = {} diff --git a/packages/web/src/weaviate_client_web/_httpx_fetch.py b/packages/web/src/weaviate_client_web/_httpx_fetch.py index c284af74a..dcf337f7f 100644 --- a/packages/web/src/weaviate_client_web/_httpx_fetch.py +++ b/packages/web/src/weaviate_client_web/_httpx_fetch.py @@ -1,30 +1,8 @@ -"""fetch-based httpx transport for Pyodide/Emscripten. - -The base client's REST path uses ``httpx.AsyncClient`` with explicit -``httpx.AsyncHTTPTransport`` mounts (``weaviate/connect/v4.py``). httpcore opens raw -sockets, which do not exist under WASM, so without this module every REST call -(``is_ready``, collection config, batch references, …) fails with an empty connection -error even though the grpc-web data path works. - -Installing reroutes ``AsyncHTTPTransport.handle_async_request`` through the browser's -``fetch`` via ``pyodide.http.pyfetch`` — the same install-globally-under-Emscripten -philosophy as the grpc shim in ``_shim.py``. Responses are fully buffered, which matches -how the base client consumes them (JSON bodies, no streaming). - -It installs under Emscripten even when Pyodide's bundled httpx carries its own JS-fetch -transport (``httpx/_transports/jsfetch.py``): that transport reads ``Response.body`` -unconditionally, which is ``null`` for HEAD requests and 204 responses (``data.exists``, -``data.delete_by_id``, ``tenants.exists``, …), and it does not enforce the per-request -read timeout end-to-end. - -Known divergences from native httpx (acceptable for the weaviate client's usage): -- the browser's fetch follows redirects internally, so httpx never sees a 3xx; -- multi-value response headers (e.g. Set-Cookie) are folded into one value; -- responses are fully buffered (no streaming). - -Like the rest of this package, this module imports ``pyodide`` at module scope and is -therefore only importable under Emscripten/Pyodide (or with a ``pyodide`` stand-in -pre-installed in ``sys.modules``). +"""fetch-based httpx.AsyncHTTPTransport for Pyodide, where httpcore has no sockets. + +Replaces Pyodide's bundled jsfetch transport, which fails on body-less HEAD/204 responses +and ignores read timeouts. Differs from native httpx: fetch follows redirects itself, +multi-value headers are folded, and bodies are fully buffered. """ import math @@ -48,12 +26,8 @@ "transfer-encoding", } -# Response headers describing the wire encoding of the body. fetch decompresses -# responses transparently, so the bytes handed to httpx are already plain; passing the -# original content-encoding through makes httpx run its decoders over them again and -# raise DecodingError, and the original content-length no longer matches the body. -# (Browsers usually hide content-encoding on CORS responses, which is why this never -# fired live — same-origin and Node fetch do expose it.) +# fetch has already decoded the body; passing content-encoding/length through would make +# httpx decode it again. _FETCH_DECODED_RESPONSE_HEADERS = { "content-encoding", "content-length", @@ -74,24 +48,18 @@ async def _read_request_body(request: httpx.Request) -> bytes: def _pick_timeout(request: httpx.Request) -> Optional[float]: - """Pick the request deadline from httpx's timeout extension: the ``read`` value only. + """The request deadline: httpx's ``read`` timeout only (None = no deadline). - The base client sets ``read`` per request and passes ``read=None`` for "no deadline" - while still carrying a ``pool`` value; ``connect`` is not separable under fetch and a - pool-acquire timeout has no meaning there, so neither may stand in for ``read``. + ``connect`` and ``pool`` have no meaning under fetch. """ timeouts = request.extensions.get("timeout") or {} return timeouts.get("read") def _abort_signal_ms(timeout: Optional[float]) -> Optional[int]: - """Milliseconds for ``AbortSignal.timeout``; ``None`` means no client-side deadline. + """Milliseconds for an abort timer, or None for no deadline (None, negative, non-finite). - Absent, negative and non-finite timeouts mean "no deadline". Zero is an immediate - deadline — native httpx times a ``read=0`` request out at once, and this package's - own grpc-timeout encoding treats zero the same way, so the fetch path must not - silently turn the same configuration into an indefinite wait. Positive values are - rounded up so a sub-millisecond timeout never becomes immediate. + Zero is immediate, as in httpx; positive values round up; capped at 2**31-1. """ if timeout is None or not math.isfinite(timeout) or timeout < 0: return None @@ -105,13 +73,9 @@ def _abort_signal_ms(timeout: Optional[float]) -> Optional[int]: def _map_fetch_error( e: BaseException, request: httpx.Request, deadline_set: bool ) -> httpx.TransportError: - """Translate a pyfetch failure into httpx's exception taxonomy. + """Map a pyfetch OSError (network, DNS, CORS, CSP, abort) to an httpx error. - Pyodide surfaces every JS fetch rejection (network down, DNS, CORS, CSP, an - AbortSignal firing) as OSError — or pyodide.http.AbortError, an OSError subclass — - never as an httpx exception. Without this mapping the base client cannot classify - failures (WeaviateConnectionError/WeaviateTimeoutError) and best-effort callers that - swallow httpx.RequestError break. + httpx.ReadTimeout if our deadline fired, else httpx.ConnectError. """ msg = str(e) or repr(e) if deadline_set and any(hint in msg.lower() for hint in _TIMEOUT_HINTS): @@ -120,9 +84,7 @@ def _map_fetch_error( def _validate_header(name: str, value: str) -> None: - # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at - # send time; this transport bypasses h11, so mirror that defence here rather than - # delegating it entirely to the JS runtime's fetch. + # h11 normally rejects CR/LF/NUL in headers; this transport bypasses h11. if any(c in name or c in value for c in ("\r", "\n", "\0")): raise httpx.LocalProtocolError(f"Illegal character in header {name!r}") @@ -179,7 +141,7 @@ async def _fetch_handle_async_request( ) -# sentinel so other packages (and uninstall) can recognise the patched method +# marker for detecting the patched method (tests, ci/pyodide-e2e) _fetch_handle_async_request.__weaviate_fetch_shim__ = True # type: ignore[attr-defined] diff --git a/packages/web/src/weaviate_client_web/_sender.py b/packages/web/src/weaviate_client_web/_sender.py index fd2a3488a..472ada397 100644 --- a/packages/web/src/weaviate_client_web/_sender.py +++ b/packages/web/src/weaviate_client_web/_sender.py @@ -1,12 +1,8 @@ """HTTP senders for the grpc-web transport. A *sender* is ``async def sender(url, headers, body, timeout) -> (status, headers, body)``. -The default uses ``pyodide.http.pyfetch`` (browser fetch); a sender can be injected for -testing via :func:`weaviate_client_web.set_sender`. - -Like the rest of this package, this module imports ``pyodide`` at module scope and is -therefore only importable under Emscripten/Pyodide (or with a ``pyodide`` stand-in -pre-installed in ``sys.modules``). +The default uses ``pyodide.http.pyfetch``; tests inject one with +:func:`weaviate_client_web.set_sender`. """ from typing import Awaitable, Callable, Dict, Optional, Tuple diff --git a/packages/web/src/weaviate_client_web/_shim.py b/packages/web/src/weaviate_client_web/_shim.py index 0624f5259..e8c7aa768 100644 --- a/packages/web/src/weaviate_client_web/_shim.py +++ b/packages/web/src/weaviate_client_web/_shim.py @@ -1,21 +1,7 @@ -"""A minimal pure-Python stand-in for the ``grpc`` API surface ``weaviate-client`` uses. - -It covers what ``weaviate-client`` touches at import time and on the async unary data -path. It is installed into ``sys.modules`` (as ``grpc``, ``grpc.aio``, ``grpc._utilities``, -``grpc.aio._typing``, ``grpc.experimental``) *before* ``import weaviate`` so the client -loads under Pyodide/Emscripten, where the real ``grpcio`` C-extension wheel does not -exist. The shim satisfies two contracts at once: - -1. **Import surface** — every ``import grpc`` / ``from grpc(.aio) import ...`` executed - while ``weaviate`` and its generated ``*_pb2_grpc`` stubs are imported - (``weaviate/config.py``, ``exceptions.py``, ``retry.py``, ``connect/base.py``, - ``connect/v4.py``, and the v6300 stub's ``grpc.__version__`` / - ``grpc._utilities.first_version_is_lower`` version gate). -2. **Runtime type contract** — :class:`AioChannel` becomes ``grpc.aio.Channel`` so the - real grpc-web channel (which subclasses it) passes the - ``isinstance(..., grpc.aio.Channel)`` assertions in ``connect/v4.py``; - :class:`AioRpcError` is the error the client catches and inspects via ``.code()`` / - ``.details()`` (``exceptions.py``, ``retry.py``). +"""Pure-Python stand-in for the grpc API that weaviate-client imports (Pyodide has no grpcio). + +Installed into sys.modules as grpc, grpc.aio, grpc._utilities, grpc.aio._typing and +grpc.experimental before weaviate is imported. """ import enum @@ -23,20 +9,14 @@ import types from typing import Any, Optional -# grpcio reports 1.72.1 as the version that the v6300 generated stub requires; matching -# it makes the stub's import-time version gate pass. See weaviate/proto/v1/__init__.py. +# grpc.__version__ under the shim; kept equal to weaviate.proto.v1._GRPCIO_FALLBACK_VERSION. FAKE_GRPC_VERSION = "1.72.1" _SHIM_MARKER = "__weaviate_client_web_shim__" class StatusCode(enum.Enum): - """Mirror of ``grpc.StatusCode``. - - ``value`` is the canonical ``(int, str)`` tuple, matching grpcio so ``code.value[0]`` - / ``code.value[1]`` (``exceptions.py``) and ``code.name`` (``connect/v4.py``) behave - identically. - """ + """Mirror of ``grpc.StatusCode``; ``value`` is grpcio's ``(int, str)`` tuple.""" OK = (0, "ok") CANCELLED = (1, "cancelled") @@ -72,11 +52,11 @@ class RpcError(Exception): class Call: """Stand-in for ``grpc.Call`` (imported by ``exceptions.py`` / ``retry.py``). - Only used for ``isinstance``/type-import purposes; the async-only WASM path raises + Only used for ``isinstance``/type-import purposes; the async-only grpc-web path raises :class:`AioRpcError`, never a sync ``Call``. """ - def code(self) -> StatusCode: # pragma: no cover - never instantiated under WASM + def code(self) -> StatusCode: # pragma: no cover - never instantiated raise NotImplementedError def details(self) -> str: # pragma: no cover @@ -134,19 +114,11 @@ def ssl_channel_credentials(*_args: Any, **_kwargs: Any) -> ChannelCredentials: class SyncChannel: - """Stand-in for ``grpc.Channel`` (sync). - - Never instantiated under WASM — the sync channel factory raises (the WASM transport - is async-only). - """ + """Stand-in for ``grpc.Channel``; never instantiated (the sync factories raise).""" class AioChannel: - """Become ``grpc.aio.Channel``. - - The grpc-web channel subclasses this so the ``isinstance(..., grpc.aio.Channel)`` - assertions in ``connect/v4.py`` hold. - """ + """Stand-in for ``grpc.aio.Channel``; ``GrpcWebChannel`` subclasses it.""" def first_version_is_lower(_version: str, _other: str) -> bool: @@ -205,11 +177,7 @@ def _aio_insecure_channel( def _noop(*_args: Any, **_kwargs: Any) -> None: - """Inert stand-in for imported-but-unused server-side stub-registration helpers. - - e.g. ``grpc.unary_unary_rpc_method_handler``: imported by generated ``*_pb2_grpc`` - code, never called by the client. - """ + """Stand-in for server-side registration helpers that *_pb2_grpc imports but never calls.""" return None @@ -218,19 +186,16 @@ def is_installed() -> bool: def install() -> bool: - """Install the shim into ``sys.modules`` as ``grpc`` and submodules. + """Install the shim as ``grpc`` and its submodules under Emscripten; no-op elsewhere. - On normal platforms this is a no-op — a real, working ``grpcio`` must be left in - place. Under Emscripten the bootstrap calls this automatically. Returns ``True`` - if the shim is in place afterwards. + Returns ``True`` if the shim is in place. """ if sys.platform != "emscripten": return False if is_installed(): return True - # Modules are populated via __dict__.update — dynamic module synthesis, so static - # type checkers do not flag each attribute assignment. + # __dict__ assignment keeps type checkers quiet on these synthesized modules utilities = types.ModuleType("grpc._utilities") utilities.__dict__["first_version_is_lower"] = first_version_is_lower @@ -263,7 +228,6 @@ def install() -> bool: "ssl_channel_credentials": ssl_channel_credentials, "secure_channel": _sync_channel_unsupported, "insecure_channel": _sync_channel_unsupported, - # Imported (never called) by generated *_pb2_grpc servicer/registration code. "unary_unary_rpc_method_handler": _noop, "stream_stream_rpc_method_handler": _noop, "unary_stream_rpc_method_handler": _noop, diff --git a/packages/web/tests/conftest.py b/packages/web/tests/conftest.py index a6491b8dc..6546c53ba 100644 --- a/packages/web/tests/conftest.py +++ b/packages/web/tests/conftest.py @@ -1,11 +1,6 @@ -"""Restrict this suite to Pyodide. +"""Skip collection off Pyodide: these modules import pyodide. -The package under test imports ``pyodide`` at module scope, so the test modules here -are only importable under Emscripten/Pyodide. There, pytest runs them via -``ci/pyodide-e2e/units.mjs`` (async tests execute on Pyodide's event loop through JSPI -stack switching). On CPython, keep pytest from collecting them: - - node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist +Run them with: node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist """ import sys diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py index 5aef2375c..d2f0e745d 100644 --- a/packages/web/tests/test_framing.py +++ b/packages/web/tests/test_framing.py @@ -61,9 +61,7 @@ def test_parse_trailers_lowercases_keys(): def test_parse_trailers_keeps_status_when_message_is_not_ascii(): - # A proxy that does not percent-encode grpc-message, or a server error quoting a - # UTF-8 collection/tenant name, sends raw non-ASCII bytes. Decoding must not raise: - # the grpc-status travelling with it is the part the client acts on. + # raw UTF-8 in grpc-message must not lose grpc-status parsed = parse_trailers("grpc-status:5\r\ngrpc-message:Café not found\r\n".encode("utf-8")) assert parsed["grpc-status"] == "5" assert parsed["grpc-message"] == "Café not found" @@ -123,8 +121,7 @@ def test_compressed_message_frame_rejected(): def test_second_trailer_frame_rejected(): - # a later trailer must not overwrite the first one's grpc-status — an error could - # be laundered into a fabricated OK; the spec allows exactly one trailer frame + # a second trailer could overwrite an error grpc-status; the spec allows one body = _frame(b"a") + _frame(b"grpc-status:7\r\n", 0x80) + _frame(b"grpc-status:0\r\n", 0x80) with pytest.raises(FrameError, match="second trailer frame"): split_response(body) diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py index eb0c04362..fe6adb2be 100644 --- a/packages/web/tests/test_httpx_fetch.py +++ b/packages/web/tests/test_httpx_fetch.py @@ -1,12 +1,6 @@ -"""Tests for the fetch-based httpx transport (``_httpx_fetch.py``). +"""Tests for _httpx_fetch. Fakes patch _httpx_fetch.pyfetch and sys.modules["js"]. -Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``. ``pyfetch`` is bound at -import time in ``_httpx_fetch``, so fakes patch that module attribute; ``from js -import AbortSignal`` is resolved per request, so a fake js module in ``sys.modules`` -intercepts it even under real Pyodide. - -The install-semantics tests at the bottom run against this interpreter's real -installation: importing ``weaviate_client_web`` bootstrapped the transport globally. +The install tests at the bottom use the real bootstrap. """ import sys @@ -202,9 +196,7 @@ async def test_query_string_preserved_in_url(fake_pyfetch): async def test_content_encoding_stripped_from_response(fake_pyfetch): - # fetch hands back ALREADY-decompressed bytes; if the original content-encoding - # header were passed through, httpx.Response would gunzip a second time and raise - # DecodingError. content-length is stale for the same reason. + # fetch returns decoded bytes; passing content-encoding through would decode twice fake_pyfetch.response = FakeFetchResponse( status=200, headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, @@ -244,9 +236,7 @@ async def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_sig async def test_read_none_means_no_deadline_even_with_pool_and_connect_set( fake_pyfetch, fake_abort_signal ): - # what the base client hands over for a non-finite request timeout: read=None with the - # session pool timeout still set; falling back to pool/connect would abort a long - # insert after 5 s + # a non-finite timeout arrives as read=None with pool set; pool must not become the deadline await _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5})) await _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) assert fake_abort_signal.timeouts == [] @@ -275,9 +265,7 @@ async def test_missing_js_module_degrades_to_no_signal(fake_pyfetch, missing_js) async def test_zero_timeout_is_an_immediate_deadline(fake_pyfetch, fake_abort_signal): - # native httpx times a read=0 request out at once; the WASM transport must not - # silently turn the same configuration into an indefinite wait (and it must not - # fall through to the 5s connect timeout either) + # read=0 is an immediate deadline, as in httpx; connect is ignored await _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) assert fake_abort_signal.timeouts == [0] assert fake_pyfetch.calls[0]["signal"] == "signal-0" @@ -334,7 +322,7 @@ def _install_raising_pyfetch(monkeypatch, exc: BaseException) -> None: async def test_fetch_failure_maps_to_httpx_connect_error(monkeypatch): # pyodide surfaces JS fetch rejections as OSError; the base client can only classify - # httpx exceptions (WeaviateConnectionError etc.), so the shim must translate + # httpx exceptions, so the transport maps OSError to httpx errors _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) with pytest.raises(httpx.ConnectError, match="Failed to fetch") as excinfo: await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) @@ -364,8 +352,7 @@ async def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_ async def test_fetch_abort_without_deadline_stays_connect_error(monkeypatch, missing_js): - # the same message without a deadline set (no js bridge -> no signal) is not OUR - # timeout, so it must stay a connection error + # without our deadline (no js bridge -> no signal), an abort is a connection error _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) with pytest.raises(httpx.ConnectError): await _handle( @@ -381,8 +368,6 @@ async def test_empty_oserror_str_keeps_repr_detail(monkeypatch): async def test_crlf_in_header_value_rejected(fake_pyfetch): - # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at - # send time; this transport bypasses h11 and must keep that defence request = httpx.Request( "GET", "http://h:8080/v1/meta", headers={"x-key": "val\r\nx-injected: evil"} ) @@ -397,9 +382,7 @@ async def test_crlf_in_header_value_rejected(fake_pyfetch): def test_bootstrap_installed_fetch_transport(): - # This interpreter imported weaviate_client_web at the top of this file, so the real - # bootstrap ran — even though Pyodide's bundled httpx carries its own jsfetch - # transport, the package's transport must be the active one. + # the import at the top ran the real bootstrap, which replaces Pyodide's jsfetch transport assert weaviate_client_web.is_fetch_transport_installed() assert ( getattr(httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False) diff --git a/packages/web/tests/test_shim_install.py b/packages/web/tests/test_shim_install.py index 0efab5973..710d4f164 100644 --- a/packages/web/tests/test_shim_install.py +++ b/packages/web/tests/test_shim_install.py @@ -1,11 +1,4 @@ -"""grpc shim tests, against this interpreter's real installation. - -Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``: importing -``weaviate_client_web`` bootstrapped the shim, so ``sys.modules['grpc']`` here IS the -shim. Bootstrap scenarios that need a clean import state live in ``units.mjs`` (one -fresh interpreter) and, for the base client's hook logic, in -``test/test_wasm_compat.py`` on CPython. -""" +"""grpc shim tests; sys.modules["grpc"] is the shim, installed by importing weaviate_client_web.""" import struct @@ -76,9 +69,7 @@ async def sender(url, headers, body_in, timeout): def test_fake_grpc_version_matches_base_fallback(): - # The shim advertises FAKE_GRPC_VERSION as grpc.__version__ and the base package - # falls back to _GRPCIO_FALLBACK_VERSION under Emscripten — the vendored stubs' - # version gates see both, so they must never drift apart. + # grpc.__version__ and the proto-selection fallback describe the same fake grpcio from weaviate.proto.v1 import _GRPCIO_FALLBACK_VERSION assert FAKE_GRPC_VERSION == _GRPCIO_FALLBACK_VERSION diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index b7d066c2e..a9c85bd0c 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -1,10 +1,4 @@ -"""grpc-web channel / multicallable tests. - -Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs`` — async tests execute on -Pyodide's event loop through JSPI stack switching (pytest-asyncio in auto mode). These -exercise the transport classes directly through fake senders — no network, no running -Weaviate. -""" +"""GrpcWebChannel tests through fake senders (no network, no Weaviate).""" import asyncio import struct @@ -124,11 +118,7 @@ async def test_trailers_only_status_in_http_headers(): assert excinfo.value.code() is StatusCode.UNAUTHENTICATED -# --- non-grpc-web responses ------------------------------------------------------- -# -# Every real error response carries a body, and none of them is grpc-web framing. The -# bodies below are verbatim shapes seen in the wild (an empty error body is the one -# shape no server or proxy produces). +# --- non-grpc-web responses: bodies captured from real servers and proxies --------- # Weaviate's own 404, verbatim from a 1.39.0 server asked for the wrong prefix. WEAVIATE_404_JSON = ( @@ -162,8 +152,8 @@ async def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/C async def test_weaviate_404_json_names_both_candidate_causes(): - # A 404 means EITHER the server predates the native /v1/grpc-web endpoint OR the - # configured path prefix is wrong. The channel cannot tell which, so it must say both. + # a 404 means the server predates grpc-web or the prefix is wrong; the channel + # cannot tell which err = await _details_of(404, WEAVIATE_404_JSON, {"content-type": "application/json"}) details = err.details() @@ -172,14 +162,13 @@ async def test_weaviate_404_json_names_both_candidate_causes(): assert "/grpc.health.v1.Health/Check" in details # the request path assert "1.38.3" in details # candidate 1: server too old assert "path prefix" in details # candidate 2: wrong prefix - assert "/v1/grpc-web" in details # the native prefix, spelled out + assert "/v1/grpc-web" in details # Weaviate's prefix, spelled out assert "was not found" in details # the server's own explanation assert "malformed grpc-web response" not in details async def test_nginx_502_maps_to_unavailable_so_the_client_retries(): - # weaviate/retry.py retries UNAVAILABLE and nothing else; a gateway error arriving - # as INTERNAL is silently un-retried, which is the regression this pins. + # weaviate/retry.py retries only UNAVAILABLE err = await _details_of(502, NGINX_502_HTML) assert err.code() is StatusCode.UNAVAILABLE assert err.details().startswith("HTTP 502 ") @@ -201,8 +190,7 @@ async def test_nginx_404_html_is_reported_as_an_http_404(): async def test_405_names_the_wrong_prefix(): - # a 405 can only come from an existing HTTP route (measured live: a prefix pointing - # at /v1/objects answers "method POST is not allowed"), so the prefix is wrong + # a 405 comes from an existing HTTP route (e.g. /v1/objects): the prefix is wrong err = await _details_of( 405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}' ) @@ -215,8 +203,8 @@ async def test_405_names_the_wrong_prefix(): async def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): - # a valid frame header whose payload was cut short: the endpoint IS grpc-web, so the - # SPA / path-prefix hint would send the user the wrong way + # a valid frame header with a cut-short payload: the endpoint is grpc-web, so no + # SPA / path-prefix hint body = _ok_response(b"reply-bytes")[:-6] err = await _details_of(200, body) assert err.code() is StatusCode.INTERNAL @@ -281,8 +269,7 @@ async def test_401_json_body_maps_to_unauthenticated(): async def test_403_error_body_reaches_details(): - # regression: the response body is the most actionable part of the error and must - # survive into details() rather than being parsed as frames and discarded + # the response body must reach details() err = await _details_of(403, b'{"code":403,"message":"forbidden: rbac denied"}') assert err.code() is StatusCode.PERMISSION_DENIED assert "forbidden: rbac denied" in err.details() @@ -305,16 +292,14 @@ async def test_binary_error_body_does_not_break_the_error(): async def test_non_200_with_valid_grpc_web_trailers_still_uses_grpc_status(): - # guard on the fix's shape: the HTTP status must not shadow a real grpc-status that - # a proxy shipped alongside a non-200 + # a grpc-status on a non-200 response wins over the HTTP status err = await _details_of(500, _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80)) assert err.code() is StatusCode.PERMISSION_DENIED assert err.details() == "denied" async def test_non_ascii_grpc_message_preserves_the_status(): - # a trailer carrying raw UTF-8 (an un-percent-encoded proxy, or an error quoting a - # collection name) must not degrade to INTERNAL and lose grpc-status + # raw UTF-8 grpc-message keeps its status body = _frame("grpc-status:5\r\ngrpc-message:collection Café not found\r\n".encode(), 0x80) err = await _details_of(200, body) assert err.code() is StatusCode.NOT_FOUND @@ -392,8 +377,7 @@ async def test_empty_ok_response_hints_at_cors_expose_headers(): async def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): - # when grpc-status WAS visible (status 0, no frames), it is a malformed response, - # not a CORS problem — the hint must not appear + # grpc-status was visible (status 0, no frames): malformed, not a CORS problem channel = _channel(FakeSender(status=200, headers={"grpc-status": "0"}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: @@ -403,8 +387,7 @@ async def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): async def test_message_frame_without_grpc_status_is_internal_not_success(): - # HTTP 200 with a valid message frame but no grpc-status anywhere (e.g. a proxy - # dropped the trailer frame) must be an error, never a fabricated success + # a message frame without grpc-status (dropped trailer) is INTERNAL channel = _channel(FakeSender(status=200, headers={}, body=_frame(b"reply-bytes"))) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: @@ -424,8 +407,6 @@ async def test_message_frame_with_grpc_status_header_still_succeeds(): def test_stream_stream_error_recommends_insert_many_only(): - # batch.dynamic()/fixed_size()/rate_limit() do not exist on the async client (the - # only one supported under WASM), so the error must not recommend them channel = _channel(FakeSender()) mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) with pytest.raises(RuntimeError) as excinfo: @@ -457,7 +438,7 @@ async def test_grpc_timeout_header_rounds_up(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - # 123.4ms must round UP to 124ms (never advertise a shorter deadline than requested). + # 123.4ms must round up to 124ms (never advertise a shorter deadline than requested). await mc(b"q", timeout=0.1234) assert sender.calls[0][1]["grpc-timeout"] == "124m" @@ -507,9 +488,7 @@ async def test_infinite_timeout_sends_no_deadline(): async def test_huge_timeout_uses_minutes_then_no_deadline(): - # 1e8 s in milliseconds is 12 digits; the server rejects more than 8 ("timeout is - # too long", HTTP 400) and transcoders reject hour values above 8H, so past the - # minute range the request carries no deadline at all + # beyond 8 minute digits no grpc-timeout is sent (see _encode_timeout) sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) diff --git a/proto_test/test_proto.py b/proto_test/test_proto.py index 85c173d46..e7f1a28b0 100644 --- a/proto_test/test_proto.py +++ b/proto_test/test_proto.py @@ -7,11 +7,8 @@ from packaging import version -# The CI matrix deliberately installs incompatible grpcio/protobuf pairs to exercise the -# version gate in weaviate/proto/v1/__init__.py. In those cells the package raises on -# import (covered by test_proto_import), so the get_version tests below are skipped; they -# still run in every compatible cell. This check imports nothing from weaviate, so the -# test module always loads. +# CI installs incompatible grpcio/protobuf pairs; there weaviate.proto.v1 cannot import, so +# the tests below are skipped. This check imports nothing from weaviate. def _versions_incompatible() -> bool: """Whether the installed grpcio/protobuf pair makes ``import weaviate.proto.v1`` raise.""" try: @@ -51,11 +48,9 @@ def test_proto_import(): @_skip_if_incompatible def test_grpcio_metadata_fallback_under_emscripten(monkeypatch): - """Fall back for grpcio when its metadata is absent; protobuf still surfaces. + """Under Emscripten a missing grpcio falls back to _GRPCIO_FALLBACK_VERSION. - Under Pyodide/Emscripten grpcio is excluded via an environment marker, so its - distribution metadata is missing and ``get_version`` must fall back to a working - proto variant; a genuinely missing protobuf is still surfaced, not masked. + A missing protobuf still raises. """ mod = importlib.import_module("weaviate.proto.v1") @@ -86,14 +81,7 @@ def raises(pkg: str) -> str: @_skip_if_incompatible def test_grpcio_fallback_version_passes_every_vendored_stub_gate(): - """The Emscripten fallback version must satisfy every vendored stub's version gate. - - Under Pyodide ``get_version("grpcio")`` returns ``_GRPCIO_FALLBACK_VERSION`` and the - shim reports it as ``grpc.__version__``, so every vendored ``*_pb2_grpc.py`` whose - import-time gate (``first_version_is_lower``) rejects it would break at import. If - the protos are regenerated with a newer grpcio-tools, this fails until the fallback - (and the grpc-web shim's ``FAKE_GRPC_VERSION``) is bumped to match. - """ + """_GRPCIO_FALLBACK_VERSION is at least every vendored stub's GRPC_GENERATED_VERSION.""" try: from grpc._utilities import first_version_is_lower except ImportError: diff --git a/test/test_connection_params.py b/test/test_connection_params.py index 1a4df4c88..0be4b87cf 100644 --- a/test/test_connection_params.py +++ b/test/test_connection_params.py @@ -179,8 +179,7 @@ def test_sync_client_construction_rejects_grpc_web_prefix() -> None: ], ) def test_helper_params_off_emscripten_are_unchanged(call, expected) -> None: - # the no-regression pin: off Emscripten the helpers build exactly the params they - # always did; grpc_path_prefix is new and stays None (native gRPC) + # off Emscripten the helpers use native gRPC: grpc_path_prefix stays None import weaviate assert sys.platform != "emscripten" diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py index ab2da4bb7..bcb4b9e8e 100644 --- a/test/test_wasm_compat.py +++ b/test/test_wasm_compat.py @@ -1,9 +1,4 @@ -"""Unit tests for WASM/Pyodide-compatibility behaviour that runs on CPython too. - -Under Emscripten there are no subprocesses, no threads and no sockets — these tests pin -the guards, the grpc-web auto-routing and the grpc-web diagnostics added for that -environment without needing a browser. -""" +"""CPython tests for the Emscripten guards, grpc-web routing and grpc-web diagnostics.""" import asyncio import pathlib @@ -29,24 +24,20 @@ def test_embedded_raises_explicit_error_under_emscripten(monkeypatch) -> None: - # without the guard, the Emscripten socket emulation makes the port probe - # "succeed" and embedded misreports that Weaviate is already listening monkeypatch.setattr(sys, "platform", "emscripten") with pytest.raises(WeaviateStartUpError, match="WebAssembly/Pyodide"): _EmbeddedBase.check_supported_platform() def test_sync_client_construction_raises_async_only_under_emscripten(monkeypatch) -> None: - # without the guard the sync client constructs fine and the first REST call fails - # with an opaque ConnectError; the clear async-only error must win, at construction + # fails at construction, not with a ConnectError on the first REST call monkeypatch.setattr(sys, "platform", "emscripten") with pytest.raises(WeaviateStartUpError, match="async client"): WeaviateClient(connection_params=ConnectionParams.from_url("http://localhost:8080", 50051)) def test_batch_stream_fails_fast_when_grpc_web_shim_active(monkeypatch) -> None: - # over grpc-web the BatchStream RPC would die inside the background tasks (silent - # drop / endless flush); _start must raise before any task is created + # _start must raise before any background task exists monkeypatch.setattr(grpc, "__weaviate_client_web_shim__", True, raising=False) batch = object.__new__(_BatchBaseAsync) # the guard runs before any attribute access with pytest.raises(WeaviateBatchStreamError, match="insert_many"): @@ -74,8 +65,6 @@ def _ping_exception(conn: _ConnectionBase, error: Exception) -> None: def test_grpc_web_404_names_the_two_real_causes_and_drops_firewall_advice() -> None: - # over grpc-web there is no separate gRPC port and no firewall: REST just succeeded - # against this very host:port. A 404 means the path was not routed. conn = _connection(prefix="/grpc-web") error = AioRpcError( grpc.StatusCode.UNIMPLEMENTED, @@ -141,7 +130,7 @@ def test_native_grpc_message_keeps_its_advice_and_gains_the_real_status() -> Non # unchanged guidance for native gRPC ... assert "The gRPC traffic at the specified port is blocked by a firewall." in msg assert "Please check that the server address and port (localhost:50051) are correct." in msg - # ... plus the error that was previously discarded + # ... plus the call's status and details assert "UNAVAILABLE" in msg assert "failed to connect" in msg @@ -154,18 +143,14 @@ def test_non_grpc_ping_error_is_still_reported() -> None: assert "blocked by a firewall" in str(excinfo.value) -# --- grpc-web auto-routing under Emscripten ------------------------------------------- -# -# Native gRPC is impossible under WASM (no sockets, no grpcio wheel), so the async connect -# helpers pin gRPC to the REST endpoint under Weaviate's own grpc-web base path — the same -# contract as the TypeScript @weaviate/web client's webify(). Nothing selects it. +# --- grpc-web routing under Emscripten ------------------------------------------------ GRPC_WEB_PREFIX = "/v1/grpc-web" @pytest.fixture def emscripten(monkeypatch): - """Fake Emscripten, with the grpc-web shim marked active. + """Fake Emscripten, with the grpc shim marked active. Under real Pyodide ``import weaviate`` installs the shim itself; here only the routing decision is under test, not the environment check that guards it. @@ -199,7 +184,7 @@ def test_use_async_with_local_routes_grpc_to_rest_under_emscripten(emscripten) - def test_use_async_with_weaviate_cloud_routes_grpc_to_the_cluster_host(emscripten) -> None: - # WCD serves grpc-web on the cluster's own REST endpoint, not on grpc- + # Weaviate Cloud serves grpc-web on the cluster's own REST endpoint, not on grpc- import weaviate client = weaviate.use_async_with_weaviate_cloud("abc.something.weaviate.cloud", None) @@ -227,8 +212,7 @@ def test_use_async_with_custom_routes_grpc_to_rest_under_emscripten(emscripten) def test_matching_grpc_arguments_are_not_warned_about(emscripten, recwarn) -> None: - # the documented WASM shape: gRPC arguments equal to the HTTP ones. Nothing is - # discarded, so warning here would just train users to ignore the warning. + # gRPC arguments equal to the HTTP ones: nothing is replaced, so no warning import weaviate weaviate.use_async_with_custom( @@ -245,9 +229,7 @@ def test_matching_grpc_arguments_are_not_warned_about(emscripten, recwarn) -> No def test_overridden_grpc_arguments_are_warned_about(emscripten) -> None: - # Python cannot drop required parameters the way TypeScript drops them from a type, - # so a WASM caller must pass something. Overriding keeps the client usable, but it - # must never look like the endpoint they gave was honoured. + # a replaced caller endpoint must warn import weaviate with pytest.warns(UserWarning, match="Con006") as record: @@ -295,12 +277,8 @@ def test_an_explicit_local_grpc_port_is_warned_about_but_the_default_is_not(emsc # --- the single-import hook (weaviate/__init__.py) ------------------------------------ # -# The hook fires on sys.platform == "emscripten" and (via the companion's bootstrap) -# replaces sys.modules['grpc'] process-wide, so each scenario runs in a fresh subprocess -# with the platform faked before `import weaviate`. The success path — a bare import -# that bootstraps the real companion — needs real Pyodide and runs in -# ci/pyodide-e2e/units.mjs; the hook's other branches are plain CPython logic and are -# pinned here. +# The import hook replaces sys.modules["grpc"], so each scenario runs in a fresh subprocess. +# The success path runs in ci/pyodide-e2e/units.mjs. _REPO_ROOT = str(pathlib.Path(__file__).resolve().parents[1]) @@ -347,8 +325,8 @@ def test_bare_import_without_companion_raises_clear_import_error() -> None: def test_bare_import_with_grpc_present_falls_through_silently() -> None: - # Companion blocked but a real grpc IS importable (grpcio in the dev env): the hook - # must fall through and leave the normal import path untouched. + # weaviate_client_web blocked but a real grpc importable (grpcio in the dev env): the + # hook falls through and leaves the normal import path untouched result = _run_hook_scenario( prelude=_PRIME_SYSCONFIG, body=""" @@ -367,9 +345,8 @@ def test_bare_import_with_grpc_present_falls_through_silently() -> None: def test_bare_import_with_broken_companion_surfaces_its_own_error(tmp_path) -> None: - # An INSTALLED companion whose import fails (here: a missing dependency of its own) - # must raise that error, not the install hint — the hint would send the user to - # reinstall a package that is already there. + # an installed weaviate-client-web that fails to import surfaces its own error, not the + # install hint fake_pkg = tmp_path / "weaviate_client_web" fake_pkg.mkdir() (fake_pkg / "__init__.py").write_text( diff --git a/weaviate/__init__.py b/weaviate/__init__.py index ef40e7a96..751663920 100644 --- a/weaviate/__init__.py +++ b/weaviate/__init__.py @@ -2,8 +2,8 @@ import sys -# Must run before every other import: under Pyodide there is no grpcio, so importing -# weaviate-client-web first installs the pure-Python grpc replacement the imports below use. +# Must run first: under Emscripten weaviate_client_web installs the grpc shim the imports +# below need. if sys.platform == "emscripten": try: import weaviate_client_web # noqa: F401 diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index a44d1edeb..5ac4689d8 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -19,19 +19,14 @@ JSONPayload = Union[Mapping[str, Any], Sequence[Any]] TIMEOUT_TYPE_RETURN = Tuple[NUMBER, NUMBER] MAX_GRPC_MESSAGE_LENGTH = 104858000 # 10mb, needs to be synchronized with GRPC server -# first Weaviate release that serves grpc-web on the REST port +# first Weaviate release serving grpc-web on the REST endpoint GRPC_WEB_MIN_SERVER_VERSION = "1.38.3" -# the base path Weaviate serves grpc-web on +# Weaviate's grpc-web path prefix GRPC_WEB_SERVER_PATH_PREFIX = "/v1/grpc-web" def _grpc_web_shim_active() -> bool: - """Whether the 'weaviate-client-web' package has replaced the grpc module. - - That replacement (used under WASM/Pyodide, where grpcio is not available) sends unary - RPCs over grpc-web/fetch and cannot do bidirectional streaming. The marker attribute - is the agreed contract between the two packages; check it only through this helper. - """ + """Whether weaviate-client-web's grpc shim is installed (unary grpc-web RPCs, no streaming).""" return getattr(grpc, "__weaviate_client_web_shim__", False) is True @@ -62,9 +57,8 @@ def is_gcp(self) -> bool: class ConnectionParams(BaseModel): http: ProtocolParams grpc: ProtocolParams - # Optional base path of a grpc-web endpoint on the REST host:port (e.g. "/grpc-web"). - # None/"" means native gRPC. When set, gRPC may share the REST host:port and the - # prefix is passed on to the grpc-web transport. + # grpc-web path prefix (e.g. "/v1/grpc-web"); None/"" = native gRPC. When set, gRPC may + # share the REST host:port. grpc_path_prefix: Optional[str] = None @classmethod @@ -128,8 +122,6 @@ def is_gcp_on_wcd(self) -> bool: @model_validator(mode="after") def _check_port_collision(self: T) -> T: same_endpoint = self.http.host == self.grpc.host and self.http.port == self.grpc.port - # with a grpc-web prefix gRPC may share the REST host:port; without one (native - # gRPC) the same host:port is a conflict if same_endpoint and self._grpc_web_path_prefix == "": raise ValueError("http.port and grpc.port must be different if using the same host") return self @@ -144,19 +136,18 @@ def _grpc_target(self) -> str: @property def _grpc_web_path_prefix(self) -> str: - """The normalized grpc-web base path; "" means native gRPC. + """Normalized grpc-web path prefix; "" means native gRPC. - A set prefix comes back with one leading slash and no trailing slash - (e.g. "grpc-web/" -> "/grpc-web"); empty/None -> "". + One leading slash, no trailing slash ("grpc-web/" -> "/grpc-web"); empty or + None -> "". """ cleaned = (self.grpc_path_prefix or "").strip("/") return f"/{cleaned}" if cleaned else "" def _check_grpc_web_usable(self, is_async: bool) -> None: - """Fail early on a grpc-web prefix this client cannot use; does nothing for native gRPC. + """Raise if a grpc-web prefix is set but unusable (sync client, or no grpc shim). - A native grpcio channel would silently ignore the ``grpc-web.path_prefix`` option - and use native gRPC, so the replacement grpc module must be in place. + grpcio would ignore the prefix option. """ if self._grpc_web_path_prefix == "": return @@ -199,7 +190,6 @@ def _grpc_channel( if grpc_config is not None and grpc_config.channel_options is not None: options.extend(grpc_config.channel_options) - # only grpc-web adds an option; native gRPC channel options are unchanged if (prefix := self._grpc_web_path_prefix) != "": options.append(("grpc-web.path_prefix", prefix)) diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index 45a67e3c7..c35e23b25 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -35,16 +35,10 @@ def _grpc_endpoint_str(params: ProtocolParams) -> str: def _webify( http: ProtocolParams, grpc: ProtocolParams, *, grpc_chosen_by_caller: bool ) -> ConnectionParams: - """Build connection params; under WebAssembly, route gRPC over grpc-web. + """Build ConnectionParams; under Emscripten, route gRPC to the REST endpoint over grpc-web. - Under Emscripten there is no grpcio and no sockets, so native gRPC cannot work; the - only option is grpc-web on the REST endpoint. gRPC is therefore pointed at the HTTP - host/port under Weaviate's own grpc-web base path, the same thing the TypeScript - ``@weaviate/web`` client does in its ``webify()``. On every other platform ``grpc`` - is used exactly as given. - - ``grpc_chosen_by_caller`` says whether ``grpc`` came from the caller rather than from - a default of the helper. If a caller's endpoint is replaced, a warning says so. + grpc_chosen_by_caller: whether ``grpc`` is caller input rather than a helper default; a + replaced caller endpoint raises warning Con006. """ if sys.platform != "emscripten": # grpc_path_prefix=None is passed explicitly so the constructor call (and pydantic's @@ -422,9 +416,8 @@ def use_async_with_weaviate_cloud( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. - Under WebAssembly/Pyodide gRPC goes over grpc-web on the cluster's own REST endpoint - (443/TLS) rather than the separate ``grpc-`` host, because native gRPC cannot work - there. Nothing to configure: the cluster serves grpc-web itself. + Under Pyodide, gRPC runs over grpc-web on the cluster's REST endpoint (443) instead of + the ``grpc-`` host. Args: cluster_url: The WCD cluster URL or hostname to connect to. Usually in the form: rAnD0mD1g1t5.something.weaviate.cloud @@ -490,15 +483,13 @@ def use_async_with_local( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. - Under WebAssembly/Pyodide gRPC goes over grpc-web on the REST endpoint, because native - gRPC cannot work there. ``grpc_port`` is then replaced by ``port``; if you passed a - ``grpc_port`` of your own it is ignored and a ``UserWarning`` says so. + Under Pyodide, gRPC runs over grpc-web on the REST endpoint; a non-default + ``grpc_port`` is ignored with a warning. Args: host: The host to use for the underlying REST and GraphQL API calls. port: The port to use for the underlying REST and GraphQL API calls. - grpc_port: The port to use for the underlying gRPC API. Ignored under - WebAssembly/Pyodide, where gRPC shares the REST ``port`` over grpc-web. + grpc_port: The port to use for the underlying gRPC API. Ignored under Pyodide. headers: Additional headers to include in the requests, e.g. API keys for Cloud vectorization. additional_config: This includes many additional, rarely used config options. use wvc.init.AdditionalConfig() to configure. skip_init_checks: Whether to skip the initialization checks when connecting to Weaviate. @@ -647,23 +638,18 @@ def use_async_with_custom( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. - Under WebAssembly/Pyodide gRPC goes over grpc-web on the REST endpoint, because native - gRPC cannot work there (no sockets, no ``grpcio`` wheel). ``grpc_host``, ``grpc_port`` - and ``grpc_secure`` are then replaced by ``http_host``, ``http_port`` and - ``http_secure``; if what you passed differed, it is ignored and a ``UserWarning`` - names both endpoints. This mirrors the TypeScript ``@weaviate/web`` client, which - removes those three options from its API altogether. + Under Pyodide, gRPC runs over grpc-web on the REST endpoint: ``grpc_host``, + ``grpc_port`` and ``grpc_secure`` are replaced by the HTTP values, with a warning if + they differ. Args: http_host: The host to use for the underlying REST and GraphQL API calls. http_port: The port to use for the underlying REST and GraphQL API calls. http_secure: Whether to use https for the underlying REST and GraphQL API calls. - grpc_host: The host to use for the underlying gRPC API. Ignored under - WebAssembly/Pyodide, where gRPC shares the REST endpoint over grpc-web. - grpc_port: The port to use for the underlying gRPC API. Ignored under - WebAssembly/Pyodide, where gRPC shares the REST endpoint over grpc-web. + grpc_host: The host to use for the underlying gRPC API. Ignored under Pyodide. + grpc_port: The port to use for the underlying gRPC API. Ignored under Pyodide. grpc_secure: Whether to use a secure channel for the underlying gRPC API. Ignored - under WebAssembly/Pyodide, where gRPC shares the REST endpoint over grpc-web. + under Pyodide. headers: Additional headers to include in the requests, e.g. API keys for Cloud vectorization. additional_config: This includes many additional, rarely used config options. use wvc.init.AdditionalConfig() to configure. auth_credentials: The credentials to use for authentication with your Weaviate instance. This can be an API key, in which case pass a string or use `weaviate.classes.init.Auth.api_key()`, diff --git a/weaviate/connect/v4.py b/weaviate/connect/v4.py index e94cbc003..1cba0ec50 100644 --- a/weaviate/connect/v4.py +++ b/weaviate/connect/v4.py @@ -358,9 +358,7 @@ def __handle_ping_response(self, res: health_weaviate_pb2.WeaviateHealthCheckRes return None def __handle_ping_exception(self, e: Exception) -> None: - # pass the error on: its code()/details() say what actually went wrong, and the - # generic advice does not apply to grpc-web (no separate gRPC port, no firewall; - # REST just worked against this same endpoint) + # pass the error on so the message can report its status and details raise WeaviateGRPCUnavailableError( f"v{self.server_version}", self._connection_params._grpc_address, @@ -369,7 +367,7 @@ def __handle_ping_exception(self, e: Exception) -> None: ) from e def __grpc_web_prefix(self) -> Optional[str]: - """The configured grpc-web base path, or None when this is native gRPC.""" + """The configured grpc-web path prefix, or None for native gRPC.""" return self._connection_params._grpc_web_path_prefix or None @property diff --git a/weaviate/exceptions.py b/weaviate/exceptions.py index 26bca2f85..f83e05224 100644 --- a/weaviate/exceptions.py +++ b/weaviate/exceptions.py @@ -355,13 +355,11 @@ def __init__( GRPC_WEB_SERVER_PATH_PREFIX, ) - # grpc-web shares the REST host:port under a base path: there is no separate - # gRPC port to open, and REST already worked against this endpoint, so no - # firewall/wrong-port advice here + # no firewall/wrong-port advice: REST already worked, and grpc-web normally + # shares its endpoint address = f"{grpc_address[0]}:{grpc_address[1]}" - # the grpc-web channel reports an unrouted path as UNIMPLEMENTED with the - # HTTP status in the details (see weaviate_client_web._channel); a genuine - # UNIMPLEMENTED from a routed endpoint must not get the wrong-path diagnosis + # weaviate_client_web reports an unrouted path as UNIMPLEMENTED with "HTTP 404/405" + # in details; a routed endpoint's own UNIMPLEMENTED is not a wrong path if code is StatusCode.UNIMPLEMENTED and any( marker in (details or "") for marker in ("HTTP 404", "HTTP 405") ): diff --git a/weaviate/proto/v1/__init__.py b/weaviate/proto/v1/__init__.py index be18e3402..92297b8b9 100644 --- a/weaviate/proto/v1/__init__.py +++ b/weaviate/proto/v1/__init__.py @@ -16,10 +16,8 @@ from weaviate.exceptions import WeaviateProtobufIncompatibility -# grpcio version to assume under Pyodide/Emscripten, where grpcio is not installed (see the -# sys_platform marker in setup.cfg) and the grpc module comes from weaviate-client-web. -# Limited to grpcio AND Emscripten: a broken grpcio install elsewhere still raises -# PackageNotFoundError, and a missing protobuf is never hidden. +# grpcio version assumed under Emscripten, where weaviate-client-web provides grpc; must be +# >= 1.72.0, or the protobuf >= 6.30 check below raises. _GRPCIO_FALLBACK_VERSION = "1.72.1" def get_version(pkg: str) -> version.Version: