From 5385441c7996ebfac59a35f0c4c2a098245dff71 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 23:04:00 +0000 Subject: [PATCH 1/4] Stream update progress as it arrives, and count the final reboot as success A Vector board writes its update progress as JSON objects back to back with no newline between them, closes without a length, and reboots without closing the connection once the update is written. apply_update split on newlines and read in 4096 byte buffers, so the progress callback fired once at the very end with everything glued into one "log" line, and the reboot then surfaced as DeviceUnreachableError: a successful update reported as a failure. - Decode records with a raw JSON decoder, so back to back objects, newline separated objects and plain text lines all work. - HTTP streams hand back what has arrived (urllib3 read1) instead of waiting for a full buffer; raw urllib3 errors map to the same typed transport errors as before. - A connection lost after the board has said "Update complete" returns the records normally; lost any earlier, it still raises. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XmBYzaorDTZcj96eHCBC35 --- docs/machine.md | 2 + tests/test_http.py | 79 +++++++++++++++++++++++++++ tests/test_machine.py | 92 +++++++++++++++++++++++++++++-- warpedpinball/__init__.py | 2 +- warpedpinball/machine.py | 94 +++++++++++++++++++++++++------- warpedpinball/transports/http.py | 35 +++++++++++- 6 files changed, 277 insertions(+), 27 deletions(-) diff --git a/docs/machine.md b/docs/machine.md index bc04db9..50ccd98 100644 --- a/docs/machine.md +++ b/docs/machine.md @@ -122,6 +122,8 @@ m.update_player(id=3, initials="MAX", full_name="Max M.") # authenticated # Firmware updates, with streamed progress info = m.check_for_updates() # note: 10 s server-side cooldown +# progress fires as each record arrives; the board rebooting at the end is +# success, losing it earlier raises m.apply_update(progress=lambda rec: print(rec.get("percent"), rec.get("log"))) m.wait_until_reachable() # poll /api/version until the board is back diff --git a/tests/test_http.py b/tests/test_http.py index 0c93ab6..98ded0d 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -4,6 +4,7 @@ import pytest import requests +import urllib3 from conftest import FakeResponse, FakeSession from warpedpinball import auth @@ -335,3 +336,81 @@ def test_hostnames_and_public_ips_keep_proxy_behavior(): def test_caller_supplied_session_is_left_alone(): session = requests.Session() assert HttpTransport("192.168.4.7", session=session)._session.trust_env is True + + +class _RawResponse(FakeResponse): + """A response whose body is read the way urllib3 2.3+ allows.""" + + def __init__(self, reads): + super().__init__(200) + self._reads = list(reads) + self.raw = self + + def read1(self, amt): + item = self._reads.pop(0) if self._reads else b"" + if isinstance(item, Exception): + raise item + return item + + def iter_content(self, chunk_size=4096): # pragma: no cover - must not run + raise AssertionError("read1 should be used when available") + + +def test_stream_returns_what_has_arrived_rather_than_filling_a_buffer(): + session = FakeSession() + session.responses = [_RawResponse([b'{"percent": 1}', b'{"percent": 2}'])] + chunks = list(make_transport(session).stream("/api/update/apply")) + assert chunks == [b'{"percent": 1}', b'{"percent": 2}'] + + +@pytest.mark.parametrize( + "error, expected", + [ + ( + urllib3.exceptions.ProtocolError("Connection broken: reset by peer"), + DeviceUnreachableError, + ), + (ConnectionResetError(104, "reset"), DeviceUnreachableError), + ( + urllib3.exceptions.ReadTimeoutError(None, "/", "Read timed out."), + DeviceTimeoutError, + ), + ], +) +def test_stream_raw_read_errors_are_typed(error, expected): + session = FakeSession() + session.responses = [_RawResponse([b"partial", error])] + stream = make_transport(session).stream("/api/update/apply") + assert next(stream) == b"partial" + with pytest.raises(expected): + next(stream) + + +def test_stream_delivers_progress_before_the_board_closes_the_connection(): + """Against a real socket: no length, records trickle in, then a reset.""" + import socket + import threading + + server = socket.socket() + server.bind(("127.0.0.1", 0)) + server.listen(1) + port = server.getsockname()[1] + release = threading.Event() + + def serve(): + conn, _ = server.accept() + conn.recv(65536) + conn.sendall(b"HTTP/1.1 200 200\r\nContent-Type: text/html\r\n\r\n") + conn.sendall(b'{"percent": 1}') + release.wait(5) + conn.sendall(b'{"percent": 2}') + conn.close() + server.close() + + threading.Thread(target=serve, daemon=True).start() + transport = HttpTransport(f"127.0.0.1:{port}", timeout=5) + stream = transport.stream("/api/update/apply", body={"url": "u"}) + # The first record arrives while the board is still holding the second. + assert next(stream) == b'{"percent": 1}' + release.set() + assert b"".join(stream) == b'{"percent": 2}' diff --git a/tests/test_machine.py b/tests/test_machine.py index dc87386..f58a3fd 100644 --- a/tests/test_machine.py +++ b/tests/test_machine.py @@ -471,11 +471,95 @@ def test_apply_update_non_dict_check_result_raises(): machine.apply_update() -def test_iter_lines_skips_blank_lines(): - from warpedpinball.machine import _iter_lines +def test_iter_records_skips_blank_lines(): + from warpedpinball.machine import _iter_records - lines = list(_iter_lines(iter([b"a\n\n\nb\n"]))) - assert lines == ["a", "b"] # blank lines between newlines dropped + records = list(_iter_records(iter([b"a\n\n\nb\n"]))) + assert records == [{"log": "a"}, {"log": "b"}] # blank lines dropped + + +def test_iter_records_splits_back_to_back_objects_as_they_arrive(): + """The firmware writes records with nothing between them.""" + from warpedpinball.machine import _iter_records + + seen = [] + + def chunks(): + yield b'{"log": "Starting update", "percent": 0}{"perc' + # The first record is out before the rest of the stream exists. + seen.append("second chunk") + yield b'ent": 2.5}{"log": "Writing /a.py", "percent": 41}' + + records = [] + for record in _iter_records(chunks()): + records.append((record, list(seen))) + assert records == [ + ({"log": "Starting update", "percent": 0}, []), + ({"percent": 2.5}, ["second chunk"]), + ({"log": "Writing /a.py", "percent": 41}, ["second chunk"]), + ] + + +def test_iter_records_handles_a_character_split_across_chunks(): + from warpedpinball.machine import _iter_records + + raw = json.dumps({"log": "caf\u00e9"}, ensure_ascii=False).encode() + split = raw.index("\u00e9".encode()) + 1 + records = list(_iter_records(iter([raw[:split], raw[split:]]))) + assert records == [{"log": "caf\u00e9"}] + + +def test_iter_records_keeps_stray_text_and_non_object_json(): + from warpedpinball.machine import _iter_records + + assert list(_iter_records(iter([b"[1, 2]"]))) == [{"log": "[1, 2]"}] + assert list(_iter_records(iter([b'{"percent": 1}trailing']))) == [ + {"percent": 1}, + {"log": "trailing"}, + ] + assert list(_iter_records(iter([b"\xff\xfe\xfd\xfc\xfb"]))) == [ + {"log": "\ufffd" * 5} + ] + + +def _dropping_stream(chunks): + from warpedpinball.exceptions import DeviceUnreachableError + + def _gen(): + yield from chunks + raise DeviceUnreachableError("192.168.1.42") + + return _gen() + + +def test_apply_update_treats_the_reboot_after_completion_as_success(): + """A finished board reboots mid-connection; that is the update working.""" + machine, transport = make_machine( + responses={"/api/update/check": {"url": "http://u/fw.bin"}}, + ) + transport.stream = lambda *a, **k: _dropping_stream( + [ + b'{"log": "Writing files to board", "percent": 40}', + b'{"log": "Update complete, Device will now reboot", "percent": 99}', + ] + ) + records = machine.apply_update() + assert records[-1]["percent"] == 99 + + +def test_apply_update_losing_the_board_midway_raises(): + from warpedpinball.exceptions import DeviceUnreachableError + + machine, transport = make_machine( + responses={"/api/update/check": {"url": "http://u/fw.bin"}}, + ) + transport.stream = lambda *a, **k: _dropping_stream( + [b'{"log": "Writing files to board", "percent": 40}'] + ) + seen = [] + with pytest.raises(DeviceUnreachableError): + machine.apply_update(progress=seen.append) + assert seen == [{"log": "Writing files to board", "percent": 40}] def test_diff_status_non_dict_inputs_yield_status_changed(): diff --git a/warpedpinball/__init__.py b/warpedpinball/__init__.py index bde00a9..7b3ac45 100644 --- a/warpedpinball/__init__.py +++ b/warpedpinball/__init__.py @@ -33,7 +33,7 @@ from .origin import OriginAuthError from .transports.http import HttpTransport -__version__ = "0.5.0" +__version__ = "0.5.1" __all__ = [ "connect", diff --git a/warpedpinball/machine.py b/warpedpinball/machine.py index 110cb58..3969d54 100644 --- a/warpedpinball/machine.py +++ b/warpedpinball/machine.py @@ -284,8 +284,13 @@ def apply_update( """Apply a firmware update, streaming progress. When ``url`` is omitted it is taken from :meth:`check_for_updates`. - ``progress`` receives each ``{"log": ..., "percent": ...}`` line as it - streams. Returns the list of all progress records. + ``progress`` receives each ``{"log": ..., "percent": ...}`` record as + it streams. Returns the list of all progress records. + + A board that finishes an update reboots without closing the + connection, so a connection lost after the board has said the update + is complete is the update succeeding, and returns normally. Lost any + earlier, it raises. """ if url is None: info = self.check_for_updates() @@ -300,14 +305,14 @@ def apply_update( "/api/update/apply", body={"url": url}, authenticated=True ) records: List[dict] = [] - for line in _iter_lines(stream): - try: - record = json.loads(line) - except ValueError: - record = {"log": line} - records.append(record) - if progress is not None: - progress(record) + try: + for record in _iter_records(stream): + records.append(record) + if progress is not None: + progress(record) + except TransportError: + if not _update_finished(records): + raise return records # -- clock -------------------------------------------------------------------- @@ -547,19 +552,66 @@ def _call_gated( ) from None -def _iter_lines(chunks: Iterator[bytes]) -> Iterator[str]: - """Split a byte-chunk stream into decoded, non-empty lines.""" - buffer = b"" +#: The board's last words before it reboots into new firmware. +_UPDATE_COMPLETE = "update complete" + + +def _update_finished(records: List[dict]) -> bool: + """Whether the board has reported its update as written and rebooting.""" + for record in records: + if _UPDATE_COMPLETE in str(record.get("log") or "").lower(): + return True + return False + + +def _iter_records(chunks: Iterator[bytes]) -> Iterator[dict]: + """Decode a stream of JSON objects, however they are separated. + + The firmware writes each record back to back with nothing between them + (``{"percent": 2}{"percent": 3}``), so splitting on newlines would hold + every record until the stream ends. Newline-separated records still parse, + and text that is not JSON comes through as ``{"log": text}``. + """ + decoder = json.JSONDecoder() + buffer = "" + pending = b"" for chunk in chunks: - buffer += chunk - while b"\n" in buffer: - line, buffer = buffer.split(b"\n", 1) - text = line.decode("utf-8", errors="replace").strip() - if text: - yield text - tail = buffer.decode("utf-8", errors="replace").strip() + pending += chunk + try: + buffer += pending.decode("utf-8") + pending = b"" + except UnicodeDecodeError as exc: + # A multi-byte character split across chunks: keep its start. + buffer += pending[: exc.start].decode("utf-8") + pending = pending[exc.start :] + if len(pending) > 3: # not a split character, just bad bytes + buffer += pending.decode("utf-8", errors="replace") + pending = b"" + while True: + buffer = buffer.lstrip() + if not buffer: + break + if buffer[0] == "{": + try: + value, end = decoder.raw_decode(buffer) + except ValueError: + break # an object still arriving + buffer = buffer[end:] + yield value if isinstance(value, dict) else {"log": str(value)} + continue + # Plain text: one line of it, once the line is complete. + if "\n" not in buffer: + break + line, buffer = buffer.split("\n", 1) + if line.strip(): + yield {"log": line.strip()} + tail = (buffer + pending.decode("utf-8", errors="replace")).strip() if tail: - yield tail + try: + value = json.loads(tail) + except ValueError: + value = None + yield value if isinstance(value, dict) else {"log": tail} def _find_key(d: Any, *fragments: str) -> Optional[str]: diff --git a/warpedpinball/transports/http.py b/warpedpinball/transports/http.py index 3806bd3..04b0e64 100644 --- a/warpedpinball/transports/http.py +++ b/warpedpinball/transports/http.py @@ -7,6 +7,7 @@ from urllib.parse import urlsplit import requests +import urllib3 from .. import auth from ..exceptions import ( @@ -21,6 +22,30 @@ DEFAULT_TIMEOUT = 10.0 +#: Most a streamed read hands back at once. +STREAM_CHUNK = 4096 + + +def _available_chunks(resp: requests.Response) -> Iterator[bytes]: + """Yield a streamed body as it arrives, not once a fixed-size buffer fills. + + ``iter_content(4096)`` blocks until 4096 bytes are in hand. A board + streaming update progress sends a few dozen bytes at a time and closes + the connection without a length, so every record would sit in that buffer + until the end. ``read1`` returns whatever has arrived; urllib3 older than + 2.3 has no ``read1``, and byte-at-a-time reads are the responsive + fallback (these streams are small). + """ + read1 = getattr(getattr(resp, "raw", None), "read1", None) + if callable(read1): + while True: + chunk = read1(STREAM_CHUNK) + if not chunk: + return + yield chunk + else: + yield from resp.iter_content(chunk_size=1) + def _is_lan_address(host: str) -> bool: """True for private/link-local/loopback IPs (boards always live on these).""" @@ -192,11 +217,19 @@ def stream( def _iter() -> Iterator[bytes]: try: - for chunk in resp.iter_content(chunk_size=4096): + for chunk in _available_chunks(resp): if chunk: yield chunk except requests.RequestException as exc: raise self._wrap_request_exc(exc, path) from None + except (urllib3.exceptions.ReadTimeoutError, TimeoutError) as exc: + raise self._wrap_request_exc( + requests.exceptions.Timeout(exc), path + ) from None + except (urllib3.exceptions.HTTPError, OSError) as exc: + raise self._wrap_request_exc( + requests.exceptions.ConnectionError(exc), path + ) from None finally: resp.close() From 917486df3a138426a469e8f4ac0e0255779938c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 23:05:21 +0000 Subject: [PATCH 2/4] Read the whole request in the real-socket stream test Closing a socket with request bytes still unread makes the kernel send a reset, so on CI the body arrived after the headers and the clean close became a DeviceUnreachableError. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XmBYzaorDTZcj96eHCBC35 --- tests/test_http.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_http.py b/tests/test_http.py index 98ded0d..bc61c1c 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -399,7 +399,11 @@ def test_stream_delivers_progress_before_the_board_closes_the_connection(): def serve(): conn, _ = server.accept() - conn.recv(65536) + # Read the whole request, body included: closing with any of it + # unread makes the kernel send a reset instead of a clean close. + request = b"" + while not request.endswith(b'{"url": "u"}'): + request += conn.recv(65536) conn.sendall(b"HTTP/1.1 200 200\r\nContent-Type: text/html\r\n\r\n") conn.sendall(b'{"percent": 1}') release.wait(5) From f6e7c7c54da68395bc7bed2b683c247a80a84011 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 23:06:32 +0000 Subject: [PATCH 3/4] Match the request body however it is serialized Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XmBYzaorDTZcj96eHCBC35 --- tests/test_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http.py b/tests/test_http.py index bc61c1c..cc32864 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -402,7 +402,7 @@ def serve(): # Read the whole request, body included: closing with any of it # unread makes the kernel send a reset instead of a clean close. request = b"" - while not request.endswith(b'{"url": "u"}'): + while not request.endswith(b"}"): request += conn.recv(65536) conn.sendall(b"HTTP/1.1 200 200\r\nContent-Type: text/html\r\n\r\n") conn.sendall(b'{"percent": 1}') From 3ec5ed5af72bf36643be3f7e595df49b451591cb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 23:07:23 +0000 Subject: [PATCH 4/4] Drop two branches _iter_records can never take Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XmBYzaorDTZcj96eHCBC35 --- warpedpinball/machine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/warpedpinball/machine.py b/warpedpinball/machine.py index 3969d54..91fd8ff 100644 --- a/warpedpinball/machine.py +++ b/warpedpinball/machine.py @@ -597,14 +597,14 @@ def _iter_records(chunks: Iterator[bytes]) -> Iterator[dict]: except ValueError: break # an object still arriving buffer = buffer[end:] - yield value if isinstance(value, dict) else {"log": str(value)} + yield value # a dict: the text started with "{" continue # Plain text: one line of it, once the line is complete. if "\n" not in buffer: break + # Never blank: leading whitespace was stripped above. line, buffer = buffer.split("\n", 1) - if line.strip(): - yield {"log": line.strip()} + yield {"log": line.strip()} tail = (buffer + pending.decode("utf-8", errors="replace")).strip() if tail: try: