Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/machine.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,8 @@ m.update_player(id=3, initials="MAX", full_name="Max M.") # authenticated

# Firmware updates, with streamed progress
info = m.check_for_updates() # note: 10 s server-side cooldown
# progress fires as each record arrives; the board rebooting at the end is
# success, losing it earlier raises
m.apply_update(progress=lambda rec: print(rec.get("percent"), rec.get("log")))
m.wait_until_reachable() # poll /api/version until the board is back

Expand Down
83 changes: 83 additions & 0 deletions tests/test_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import pytest
import requests
import urllib3
from conftest import FakeResponse, FakeSession

from warpedpinball import auth
Expand Down Expand Up @@ -335,3 +336,85 @@ def test_hostnames_and_public_ips_keep_proxy_behavior():
def test_caller_supplied_session_is_left_alone():
session = requests.Session()
assert HttpTransport("192.168.4.7", session=session)._session.trust_env is True


class _RawResponse(FakeResponse):
"""A response whose body is read the way urllib3 2.3+ allows."""

def __init__(self, reads):
super().__init__(200)
self._reads = list(reads)
self.raw = self

def read1(self, amt):
item = self._reads.pop(0) if self._reads else b""
if isinstance(item, Exception):
raise item
return item

def iter_content(self, chunk_size=4096): # pragma: no cover - must not run
raise AssertionError("read1 should be used when available")


def test_stream_returns_what_has_arrived_rather_than_filling_a_buffer():
session = FakeSession()
session.responses = [_RawResponse([b'{"percent": 1}', b'{"percent": 2}'])]
chunks = list(make_transport(session).stream("/api/update/apply"))
assert chunks == [b'{"percent": 1}', b'{"percent": 2}']


@pytest.mark.parametrize(
"error, expected",
[
(
urllib3.exceptions.ProtocolError("Connection broken: reset by peer"),
DeviceUnreachableError,
),
(ConnectionResetError(104, "reset"), DeviceUnreachableError),
(
urllib3.exceptions.ReadTimeoutError(None, "/", "Read timed out."),
DeviceTimeoutError,
),
],
)
def test_stream_raw_read_errors_are_typed(error, expected):
session = FakeSession()
session.responses = [_RawResponse([b"partial", error])]
stream = make_transport(session).stream("/api/update/apply")
assert next(stream) == b"partial"
with pytest.raises(expected):
next(stream)


def test_stream_delivers_progress_before_the_board_closes_the_connection():
"""Against a real socket: no length, records trickle in, then a reset."""
import socket
import threading

server = socket.socket()
server.bind(("127.0.0.1", 0))
server.listen(1)
port = server.getsockname()[1]
release = threading.Event()

def serve():
conn, _ = server.accept()
# Read the whole request, body included: closing with any of it
# unread makes the kernel send a reset instead of a clean close.
request = b""
while not request.endswith(b"}"):
request += conn.recv(65536)
conn.sendall(b"HTTP/1.1 200 200\r\nContent-Type: text/html\r\n\r\n")
conn.sendall(b'{"percent": 1}')
release.wait(5)
conn.sendall(b'{"percent": 2}')
conn.close()
server.close()

threading.Thread(target=serve, daemon=True).start()
transport = HttpTransport(f"127.0.0.1:{port}", timeout=5)
stream = transport.stream("/api/update/apply", body={"url": "u"})
# The first record arrives while the board is still holding the second.
assert next(stream) == b'{"percent": 1}'
release.set()
assert b"".join(stream) == b'{"percent": 2}'
92 changes: 88 additions & 4 deletions tests/test_machine.py
Original file line number Diff line number Diff line change
Expand Up @@ -471,11 +471,95 @@ def test_apply_update_non_dict_check_result_raises():
machine.apply_update()


def test_iter_lines_skips_blank_lines():
from warpedpinball.machine import _iter_lines
def test_iter_records_skips_blank_lines():
from warpedpinball.machine import _iter_records

lines = list(_iter_lines(iter([b"a\n\n\nb\n"])))
assert lines == ["a", "b"] # blank lines between newlines dropped
records = list(_iter_records(iter([b"a\n\n\nb\n"])))
assert records == [{"log": "a"}, {"log": "b"}] # blank lines dropped


def test_iter_records_splits_back_to_back_objects_as_they_arrive():
"""The firmware writes records with nothing between them."""
from warpedpinball.machine import _iter_records

seen = []

def chunks():
yield b'{"log": "Starting update", "percent": 0}{"perc'
# The first record is out before the rest of the stream exists.
seen.append("second chunk")
yield b'ent": 2.5}{"log": "Writing /a.py", "percent": 41}'

records = []
for record in _iter_records(chunks()):
records.append((record, list(seen)))
assert records == [
({"log": "Starting update", "percent": 0}, []),
({"percent": 2.5}, ["second chunk"]),
({"log": "Writing /a.py", "percent": 41}, ["second chunk"]),
]


def test_iter_records_handles_a_character_split_across_chunks():
from warpedpinball.machine import _iter_records

raw = json.dumps({"log": "caf\u00e9"}, ensure_ascii=False).encode()
split = raw.index("\u00e9".encode()) + 1
records = list(_iter_records(iter([raw[:split], raw[split:]])))
assert records == [{"log": "caf\u00e9"}]


def test_iter_records_keeps_stray_text_and_non_object_json():
from warpedpinball.machine import _iter_records

assert list(_iter_records(iter([b"[1, 2]"]))) == [{"log": "[1, 2]"}]
assert list(_iter_records(iter([b'{"percent": 1}trailing']))) == [
{"percent": 1},
{"log": "trailing"},
]
assert list(_iter_records(iter([b"\xff\xfe\xfd\xfc\xfb"]))) == [
{"log": "\ufffd" * 5}
]


def _dropping_stream(chunks):
from warpedpinball.exceptions import DeviceUnreachableError

def _gen():
yield from chunks
raise DeviceUnreachableError("192.168.1.42")

return _gen()


def test_apply_update_treats_the_reboot_after_completion_as_success():
"""A finished board reboots mid-connection; that is the update working."""
machine, transport = make_machine(
responses={"/api/update/check": {"url": "http://u/fw.bin"}},
)
transport.stream = lambda *a, **k: _dropping_stream(
[
b'{"log": "Writing files to board", "percent": 40}',
b'{"log": "Update complete, Device will now reboot", "percent": 99}',
]
)
records = machine.apply_update()
assert records[-1]["percent"] == 99


def test_apply_update_losing_the_board_midway_raises():
from warpedpinball.exceptions import DeviceUnreachableError

machine, transport = make_machine(
responses={"/api/update/check": {"url": "http://u/fw.bin"}},
)
transport.stream = lambda *a, **k: _dropping_stream(
[b'{"log": "Writing files to board", "percent": 40}']
)
seen = []
with pytest.raises(DeviceUnreachableError):
machine.apply_update(progress=seen.append)
assert seen == [{"log": "Writing files to board", "percent": 40}]


def test_diff_status_non_dict_inputs_yield_status_changed():
Expand Down
2 changes: 1 addition & 1 deletion warpedpinball/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
from .origin import OriginAuthError
from .transports.http import HttpTransport

__version__ = "0.5.0"
__version__ = "0.5.1"

__all__ = [
"connect",
Expand Down
94 changes: 73 additions & 21 deletions warpedpinball/machine.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,8 +284,13 @@ def apply_update(
"""Apply a firmware update, streaming progress.

When ``url`` is omitted it is taken from :meth:`check_for_updates`.
``progress`` receives each ``{"log": ..., "percent": ...}`` line as it
streams. Returns the list of all progress records.
``progress`` receives each ``{"log": ..., "percent": ...}`` record as
it streams. Returns the list of all progress records.

A board that finishes an update reboots without closing the
connection, so a connection lost after the board has said the update
is complete is the update succeeding, and returns normally. Lost any
earlier, it raises.
"""
if url is None:
info = self.check_for_updates()
Expand All @@ -300,14 +305,14 @@ def apply_update(
"/api/update/apply", body={"url": url}, authenticated=True
)
records: List[dict] = []
for line in _iter_lines(stream):
try:
record = json.loads(line)
except ValueError:
record = {"log": line}
records.append(record)
if progress is not None:
progress(record)
try:
for record in _iter_records(stream):
records.append(record)
if progress is not None:
progress(record)
except TransportError:
if not _update_finished(records):
raise
return records

# -- clock --------------------------------------------------------------------
Expand Down Expand Up @@ -547,19 +552,66 @@ def _call_gated(
) from None


def _iter_lines(chunks: Iterator[bytes]) -> Iterator[str]:
"""Split a byte-chunk stream into decoded, non-empty lines."""
buffer = b""
#: The board's last words before it reboots into new firmware.
_UPDATE_COMPLETE = "update complete"


def _update_finished(records: List[dict]) -> bool:
"""Whether the board has reported its update as written and rebooting."""
for record in records:
if _UPDATE_COMPLETE in str(record.get("log") or "").lower():
return True
return False


def _iter_records(chunks: Iterator[bytes]) -> Iterator[dict]:
"""Decode a stream of JSON objects, however they are separated.

The firmware writes each record back to back with nothing between them
(``{"percent": 2}{"percent": 3}``), so splitting on newlines would hold
every record until the stream ends. Newline-separated records still parse,
and text that is not JSON comes through as ``{"log": text}``.
"""
decoder = json.JSONDecoder()
buffer = ""
pending = b""
for chunk in chunks:
buffer += chunk
while b"\n" in buffer:
line, buffer = buffer.split(b"\n", 1)
text = line.decode("utf-8", errors="replace").strip()
if text:
yield text
tail = buffer.decode("utf-8", errors="replace").strip()
pending += chunk
try:
buffer += pending.decode("utf-8")
pending = b""
except UnicodeDecodeError as exc:
# A multi-byte character split across chunks: keep its start.
buffer += pending[: exc.start].decode("utf-8")
pending = pending[exc.start :]
if len(pending) > 3: # not a split character, just bad bytes
buffer += pending.decode("utf-8", errors="replace")
pending = b""
while True:
buffer = buffer.lstrip()
if not buffer:
break
if buffer[0] == "{":
try:
value, end = decoder.raw_decode(buffer)
except ValueError:
break # an object still arriving
buffer = buffer[end:]
yield value # a dict: the text started with "{"
continue
# Plain text: one line of it, once the line is complete.
if "\n" not in buffer:
break
# Never blank: leading whitespace was stripped above.
line, buffer = buffer.split("\n", 1)
yield {"log": line.strip()}
tail = (buffer + pending.decode("utf-8", errors="replace")).strip()
if tail:
yield tail
try:
value = json.loads(tail)
except ValueError:
value = None
yield value if isinstance(value, dict) else {"log": tail}


def _find_key(d: Any, *fragments: str) -> Optional[str]:
Expand Down
35 changes: 34 additions & 1 deletion warpedpinball/transports/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from urllib.parse import urlsplit

import requests
import urllib3

from .. import auth
from ..exceptions import (
Expand All @@ -21,6 +22,30 @@

DEFAULT_TIMEOUT = 10.0

#: Most a streamed read hands back at once.
STREAM_CHUNK = 4096


def _available_chunks(resp: requests.Response) -> Iterator[bytes]:
"""Yield a streamed body as it arrives, not once a fixed-size buffer fills.

``iter_content(4096)`` blocks until 4096 bytes are in hand. A board
streaming update progress sends a few dozen bytes at a time and closes
the connection without a length, so every record would sit in that buffer
until the end. ``read1`` returns whatever has arrived; urllib3 older than
2.3 has no ``read1``, and byte-at-a-time reads are the responsive
fallback (these streams are small).
"""
read1 = getattr(getattr(resp, "raw", None), "read1", None)
if callable(read1):
while True:
chunk = read1(STREAM_CHUNK)
if not chunk:
return
yield chunk
else:
yield from resp.iter_content(chunk_size=1)


def _is_lan_address(host: str) -> bool:
"""True for private/link-local/loopback IPs (boards always live on these)."""
Expand Down Expand Up @@ -192,11 +217,19 @@ def stream(

def _iter() -> Iterator[bytes]:
try:
for chunk in resp.iter_content(chunk_size=4096):
for chunk in _available_chunks(resp):
if chunk:
yield chunk
except requests.RequestException as exc:
raise self._wrap_request_exc(exc, path) from None
except (urllib3.exceptions.ReadTimeoutError, TimeoutError) as exc:
raise self._wrap_request_exc(
requests.exceptions.Timeout(exc), path
) from None
except (urllib3.exceptions.HTTPError, OSError) as exc:
raise self._wrap_request_exc(
requests.exceptions.ConnectionError(exc), path
) from None
finally:
resp.close()

Expand Down
Loading