From 4027df3dc2f88df905db0bc12ee0218e6da3c6dc Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Fri, 25 Sep 2026 10:03:45 -0500 Subject: [PATCH 1/2] sidecar: send CircuitPython one raw-paste window at a time (#50) Long exec/run pastes to a CircuitPython board over serial arrived garbled now and then: one 64-byte USB packet lost and the next one received twice, so the paste keeps its length. On CircuitPython's ESP32 ports the VM task can re-arm the CDC OUT endpoint between TinyUSB's USB task marking a transfer complete and copying the packet out (usbd.c clears busy before cdcd_xfer_cb), and mpremote keeps two raw-paste windows in flight, so the host's next packet lands on the uncopied one. The serial transport now waits for the board's window acknowledgement before each 128-byte window when the board runs CircuitPython, so nothing arrives while a packet is still in the endpoint buffer. MicroPython keeps mpremote's writer. On the T-Embed (CircuitPython 10.3.0), 32,000-character pastes through mpftp run: main 22 of 30 intact, this branch 30 of 30 on the same inputs. Through mpremote directly, 89 of 120 stock against 120 of 120 paced. --- CHANGELOG.md | 6 ++ cli/src/mpftp/rawpaste.py | 106 ++++++++++++++++++++++++++ cli/src/mpftp/sidecar.py | 10 ++- cli/tests/test_rawpaste.py | 151 +++++++++++++++++++++++++++++++++++++ 4 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 cli/src/mpftp/rawpaste.py create mode 100644 cli/tests/test_rawpaste.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a2168d..dc75adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,12 @@ - Connecting no longer moves a board clock that's already set, so it stops undoing NTP. An unset clock, and `mpftp rtc --set`, get the host's UTC rather than local time. `rtc` and `rtc --set` work on CircuitPython, where both used to fail (mpftp#58). - `mpftp hard-reset --monitor SECONDS [--log-path FILE]` resets the board, waits for the same port to come back, and streams the boot read-only, so a fast-failing `main.py` can't fall in the gap between two commands. The MCP `hard_reset` tool takes `monitor_seconds` and `log_path` for the same (mpftp#60). - `hard-reset` resets CircuitPython boards. It sent `machine.reset()`, which CircuitPython doesn't have, and reported success while the board kept running. +- Long `exec`, `run` and serial `put` on a CircuitPython board no longer + arrive garbled now and then. On CircuitPython's ESP32 ports a paste that ran + ahead of the board could lose one 64-byte USB packet and get the next one + twice (a race in the firmware's USB stack). mpftp now sends a CircuitPython + board one raw-paste window at a time; on a T-Embed that took 120 of 120 + pastes through intact, against 89 of 120 before (mpftp#50). ## v0.0.8 (2026-09-25) diff --git a/cli/src/mpftp/rawpaste.py b/cli/src/mpftp/rawpaste.py new file mode 100644 index 0000000..a5c75ce --- /dev/null +++ b/cli/src/mpftp/rawpaste.py @@ -0,0 +1,106 @@ +"""Raw-paste over serial, one flow-control window in flight. + +Raw-paste mode (``Ctrl-E A Ctrl-A``) lets the host run ahead of the board: the +board announces a window (128 bytes on MicroPython and CircuitPython), grants +two of them at the start, and sends ``\\x01`` each time it has read another +window's worth. mpremote uses both credits, so up to 256 bytes are on their +way while the board is still reading. + +On CircuitPython's ESP32 ports, where TinyUSB runs in a FreeRTOS task of its +own, that overlap can corrupt the paste (mpftp#50). The VM task reading the +CDC FIFO can re-arm the OUT endpoint in the gap where TinyUSB's USB task has +marked the last transfer complete but not yet copied its 64-byte packet out +of the endpoint buffer +(``lib/tinyusb/src/device/usbd.c`` clears ``busy`` before calling +``cdcd_xfer_cb``). The next packet from the host then lands on the one not +yet copied: that packet is lost and its successor arrives twice. On a T-Embed +(ESP32-S3) running CircuitPython 10.3.0, a 32,000-character paste came through +wrong 31 times in 120, each time as one 64-byte packet replaced by the next. + +Sending one window and waiting for the board's ``\\x01`` before the next +closes that gap: the board asks for more only after it has read everything +sent, so no packet can arrive while an earlier one is still in the endpoint +buffer. The same 120 pastes sent this way all came through. It gives up a +little speed at best (43.8 KB/s against 54.8 KB/s for mpremote's way on its +good run; its other run managed 6.5 KB/s), so MicroPython boards keep +mpremote's way. +""" + +from __future__ import annotations + +import struct +from typing import Any, Callable, Optional + +_TRANSPORT_CLASS: Any = None + + +def paced_raw_paste_write(transport: Any, command_bytes: bytes) -> None: + """mpremote's ``raw_paste_write``, with at most one window unacknowledged.""" + from mpremote.transport import TransportError + + serial = transport.serial + header = serial.read(2) + if len(header) != 2: + raise TransportError("no raw-paste window from the board: {!r}".format(header)) + window = struct.unpack("= len(command_bytes): + break + # Wait until the board has read all of it. + data = serial.read(1) + if data == b"\x04": + # The board stopped reading (a syntax error, say). Acknowledge. + serial.write(b"\x04") + return + if data != b"\x01": + raise TransportError("unexpected read during raw paste: {!r}".format(data)) + + serial.write(b"\x04") + data = transport.read_until(1, b"\x04") + if not data.endswith(b"\x04"): + raise TransportError("could not complete raw paste: {!r}".format(data)) + + +def _transport_class() -> Any: + global _TRANSPORT_CLASS + if _TRANSPORT_CLASS is not None: + return _TRANSPORT_CLASS + + from mpremote.transport_serial import SerialTransport + + class PacedSerialTransport(SerialTransport): + """mpremote's serial transport; raw-paste waits for each window when + ``paced()`` says so (the sidecar's: the board runs CircuitPython).""" + + paced: Optional[Callable[[], bool]] = None + + def raw_paste_write(self, command_bytes: bytes) -> None: + if self.paced is not None and self.paced(): + paced_raw_paste_write(self, command_bytes) + else: + super().raw_paste_write(command_bytes) + + _TRANSPORT_CLASS = PacedSerialTransport + return _TRANSPORT_CLASS + + +def open_serial_transport(device: str, baud: int, paced: Callable[[], bool]) -> Any: + transport = _transport_class()(device, baudrate=baud) + transport.paced = paced + return transport diff --git a/cli/src/mpftp/sidecar.py b/cli/src/mpftp/sidecar.py index b5cacb0..6a97e76 100644 --- a/cli/src/mpftp/sidecar.py +++ b/cli/src/mpftp/sidecar.py @@ -25,7 +25,7 @@ from pathlib import Path from typing import Any, Callable, Optional -from . import ble, config, mdns, webrepl, wifiboard +from . import ble, config, mdns, rawpaste, webrepl, wifiboard def split_fs_path(path: str) -> tuple[bool, str]: @@ -683,9 +683,11 @@ def _open_transport(self, device: str, baud: int) -> Any: raise time.sleep(wait) return webrepl.open_transport(device, password) - from mpremote.transport_serial import SerialTransport - - return SerialTransport(device, baudrate=baud) + # Raw-paste one window at a time on CircuitPython, whose ESP32 native + # USB can garble a paste that runs ahead of it (rawpaste.py, mpftp#50). + return rawpaste.open_serial_transport( + device, baud, paced=lambda: (self.interpreter or "") == "circuitpython" + ) def connect( self, diff --git a/cli/tests/test_rawpaste.py b/cli/tests/test_rawpaste.py new file mode 100644 index 0000000..64aead5 --- /dev/null +++ b/cli/tests/test_rawpaste.py @@ -0,0 +1,151 @@ +"""Raw-paste pacing for CircuitPython over serial (mpftp#50).""" + +from __future__ import annotations + +import unittest +from unittest import mock + +from mpftp import rawpaste + +WINDOW = 128 + + +class FakeBoard: + """The board's side of raw-paste, as shared/runtime/pyexec.c does it. + + It announces a 128-byte window and one extra credit, reads what the host + wrote whenever the host waits on it, and sends ``\\x01`` for every window + it has read. ``most_in_flight`` is the largest amount the host had + written that the board had not read yet: what the board's USB receive + path has to hold at once. + """ + + def __init__(self, stop_after: int | None = None) -> None: + self.out = bytearray(bytes([WINDOW & 0xFF, WINDOW >> 8, 0x01])) + self.received = bytearray() + self.read_upto = 0 + self.since_ack = 0 + self.most_in_flight = 0 + self.finished = False + self.stop_after = stop_after + self.stopped = False + + # pyserial surface + def write(self, data: bytes) -> int: + if self.stopped: + if data == b"\x04": + self.finished = True + return len(data) + self.received += data + self.most_in_flight = max(self.most_in_flight, len(self.received) - self.read_upto) + if self.received.endswith(b"\x04"): + self._consume() + return len(data) + + def _consume(self) -> None: + while self.read_upto < len(self.received): + c = self.received[self.read_upto] + self.read_upto += 1 + if c == 0x04: + self.finished = True + self.out += b"\x04" + return + if self.stop_after is not None and self.read_upto >= self.stop_after: + self.stopped = True + self.out += b"\x04" + return + self.since_ack += 1 + if self.since_ack == WINDOW: + self.since_ack = 0 + self.out += b"\x01" + + def read(self, n: int = 1) -> bytes: + if not self.out: + self._consume() + data = bytes(self.out[:n]) + del self.out[:n] + return data + + def inWaiting(self) -> int: # noqa: N802 (pyserial spelling) + return len(self.out) + + +class FakeTransport: + def __init__(self, board: FakeBoard) -> None: + self.serial = board + + def read_until(self, min_num_bytes: int, ending: bytes, timeout: float = 10) -> bytes: + data = bytearray() + while not data.endswith(ending): + chunk = self.serial.read(1) + if not chunk: + break + data += chunk + return bytes(data) + + +def _source(n: int) -> bytes: + return bytes(0x41 + (i % 26) for i in range(n)) + + +class PacedRawPasteTests(unittest.TestCase): + def test_everything_arrives_in_order(self) -> None: + for n in (1, WINDOW - 1, WINDOW, WINDOW + 1, 5 * WINDOW, 24_000): + board = FakeBoard() + src = _source(n) + rawpaste.paced_raw_paste_write(FakeTransport(board), src) + self.assertTrue(board.finished, n) + self.assertEqual(bytes(board.received), src + b"\x04", n) + + def test_never_more_than_one_window_unread(self) -> None: + board = FakeBoard() + rawpaste.paced_raw_paste_write(FakeTransport(board), _source(24_000)) + self.assertLessEqual(board.most_in_flight, WINDOW + 1) # + the closing Ctrl-D + + def test_mpremotes_own_write_runs_two_windows_ahead(self) -> None: + # The control: the same fake board sees mpremote's stock writer put + # two windows in flight, which is what garbles CircuitPython (#50). + from mpremote.transport_serial import SerialTransport + + board = FakeBoard() + t = FakeTransport(board) + SerialTransport.raw_paste_write(t, _source(24_000)) # type: ignore[arg-type] + self.assertEqual(bytes(board.received), _source(24_000) + b"\x04") + self.assertGreater(board.most_in_flight, WINDOW + 1) + + def test_board_that_stops_reading_is_acknowledged(self) -> None: + board = FakeBoard(stop_after=300) + rawpaste.paced_raw_paste_write(FakeTransport(board), _source(2_000)) + self.assertTrue(board.finished) + self.assertLess(len(board.received), 2_000) + + +class TransportChoiceTests(unittest.TestCase): + def test_paces_only_when_told(self) -> None: + cls = rawpaste._transport_class() + t = cls.__new__(cls) + t.paced = lambda: False + with mock.patch.object(rawpaste, "paced_raw_paste_write") as paced, \ + mock.patch("mpremote.transport_serial.SerialTransport.raw_paste_write") as stock: + t.raw_paste_write(b"x") + self.assertEqual((paced.call_count, stock.call_count), (0, 1)) + t.paced = lambda: True + t.raw_paste_write(b"x") + self.assertEqual((paced.call_count, stock.call_count), (1, 1)) + + def test_sidecar_paces_circuitpython_only(self) -> None: + from mpftp.sidecar import Session + + s = Session.__new__(Session) + s.interpreter = None + with mock.patch.object(rawpaste, "open_serial_transport", side_effect=lambda d, b, paced: paced): + paced = s._open_transport("COM25", 115200) + self.assertFalse(paced()) + s.interpreter = "micropython" + self.assertFalse(paced()) + s.interpreter = "circuitpython" + self.assertTrue(paced()) + + +if __name__ == "__main__": + unittest.main() From 9eaf83a42bdb868d5e6af7eee544050f55b60b20 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Fri, 25 Sep 2026 10:15:35 -0500 Subject: [PATCH 2/2] rawpaste: run the paced-writer tests without mpremote installed --- CHANGELOG.md | 12 ++++++------ cli/src/mpftp/rawpaste.py | 10 ++++++---- cli/tests/test_rawpaste.py | 9 +++++++++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc75adc..9df857f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ asks once per board, and `mpftp.sharePasswords` (`ask`, `always`, `never`) answers for all of them. Enable Wi-Fi Access says where the password ends up (mpftp#43). +- Long `exec`, `run` and serial `put` on a CircuitPython board no longer + arrive garbled now and then. On CircuitPython's ESP32 ports a paste that ran + ahead of the board could lose one 64-byte USB packet and get the next one + twice (a race in the firmware's USB stack). mpftp now sends a CircuitPython + board one raw-paste window at a time; on a T-Embed that took 120 of 120 + pastes through intact, against 89 of 120 before (mpftp#50). ## v0.0.9 (2026-09-25) @@ -23,12 +29,6 @@ - Connecting no longer moves a board clock that's already set, so it stops undoing NTP. An unset clock, and `mpftp rtc --set`, get the host's UTC rather than local time. `rtc` and `rtc --set` work on CircuitPython, where both used to fail (mpftp#58). - `mpftp hard-reset --monitor SECONDS [--log-path FILE]` resets the board, waits for the same port to come back, and streams the boot read-only, so a fast-failing `main.py` can't fall in the gap between two commands. The MCP `hard_reset` tool takes `monitor_seconds` and `log_path` for the same (mpftp#60). - `hard-reset` resets CircuitPython boards. It sent `machine.reset()`, which CircuitPython doesn't have, and reported success while the board kept running. -- Long `exec`, `run` and serial `put` on a CircuitPython board no longer - arrive garbled now and then. On CircuitPython's ESP32 ports a paste that ran - ahead of the board could lose one 64-byte USB packet and get the next one - twice (a race in the firmware's USB stack). mpftp now sends a CircuitPython - board one raw-paste window at a time; on a T-Embed that took 120 of 120 - pastes through intact, against 89 of 120 before (mpftp#50). ## v0.0.8 (2026-09-25) diff --git a/cli/src/mpftp/rawpaste.py b/cli/src/mpftp/rawpaste.py index a5c75ce..2afe6bf 100644 --- a/cli/src/mpftp/rawpaste.py +++ b/cli/src/mpftp/rawpaste.py @@ -10,9 +10,8 @@ own, that overlap can corrupt the paste (mpftp#50). The VM task reading the CDC FIFO can re-arm the OUT endpoint in the gap where TinyUSB's USB task has marked the last transfer complete but not yet copied its 64-byte packet out -of the endpoint buffer -(``lib/tinyusb/src/device/usbd.c`` clears ``busy`` before calling -``cdcd_xfer_cb``). The next packet from the host then lands on the one not +of the endpoint buffer (``lib/tinyusb/src/device/usbd.c`` clears ``busy`` +before calling ``cdcd_xfer_cb``). The next packet from the host then lands on the one not yet copied: that packet is lost and its successor arrives twice. On a T-Embed (ESP32-S3) running CircuitPython 10.3.0, a 32,000-character paste came through wrong 31 times in 120, each time as one 64-byte packet replaced by the next. @@ -36,7 +35,10 @@ def paced_raw_paste_write(transport: Any, command_bytes: bytes) -> None: """mpremote's ``raw_paste_write``, with at most one window unacknowledged.""" - from mpremote.transport import TransportError + try: + from mpremote.transport import TransportError + except ImportError: # the sidecar always has mpremote; unit tests may not + TransportError = RuntimeError # noqa: N806 serial = transport.serial header = serial.read(2) diff --git a/cli/tests/test_rawpaste.py b/cli/tests/test_rawpaste.py index 64aead5..e29f1b5 100644 --- a/cli/tests/test_rawpaste.py +++ b/cli/tests/test_rawpaste.py @@ -7,6 +7,13 @@ from mpftp import rawpaste +try: + import mpremote # noqa: F401 + + HAVE_MPREMOTE = True +except ImportError: + HAVE_MPREMOTE = False + WINDOW = 128 @@ -102,6 +109,7 @@ def test_never_more_than_one_window_unread(self) -> None: rawpaste.paced_raw_paste_write(FakeTransport(board), _source(24_000)) self.assertLessEqual(board.most_in_flight, WINDOW + 1) # + the closing Ctrl-D + @unittest.skipUnless(HAVE_MPREMOTE, "mpremote not installed") def test_mpremotes_own_write_runs_two_windows_ahead(self) -> None: # The control: the same fake board sees mpremote's stock writer put # two windows in flight, which is what garbles CircuitPython (#50). @@ -120,6 +128,7 @@ def test_board_that_stops_reading_is_acknowledged(self) -> None: self.assertLess(len(board.received), 2_000) +@unittest.skipUnless(HAVE_MPREMOTE, "mpremote not installed") class TransportChoiceTests(unittest.TestCase): def test_paces_only_when_told(self) -> None: cls = rawpaste._transport_class()