Garbled long pastes to CircuitPython: one raw-paste window at a time (#50)

CHANGELOG.md

@@ -16,6 +16,12 @@
   asks once per board, and `mpftp.sharePasswords` (`ask`, `always`, `never`)
   answers for all of them. Enable Wi-Fi Access says where the password ends up
   (mpftp#43).
+- Long `exec`, `run` and serial `put` on a CircuitPython board no longer
+  arrive garbled now and then. On CircuitPython's ESP32 ports a paste that ran
+  ahead of the board could lose one 64-byte USB packet and get the next one
+  twice (a race in the firmware's USB stack). mpftp now sends a CircuitPython
+  board one raw-paste window at a time; on a T-Embed that took 120 of 120
+  pastes through intact, against 89 of 120 before (mpftp#50).
 
 ## v0.0.9 (2026-09-25)
 

cli/src/mpftp/rawpaste.py

@@ -0,0 +1,108 @@
+"""Raw-paste over serial, one flow-control window in flight.
+
+Raw-paste mode (``Ctrl-E A Ctrl-A``) lets the host run ahead of the board: the
+board announces a window (128 bytes on MicroPython and CircuitPython), grants
+two of them at the start, and sends ``\\x01`` each time it has read another
+window's worth. mpremote uses both credits, so up to 256 bytes are on their
+way while the board is still reading.
+
+On CircuitPython's ESP32 ports, where TinyUSB runs in a FreeRTOS task of its
+own, that overlap can corrupt the paste (mpftp#50). The VM task reading the
+CDC FIFO can re-arm the OUT endpoint in the gap where TinyUSB's USB task has
+marked the last transfer complete but not yet copied its 64-byte packet out
+of the endpoint buffer (``lib/tinyusb/src/device/usbd.c`` clears ``busy``
+before calling ``cdcd_xfer_cb``). The next packet from the host then lands on the one not
+yet copied: that packet is lost and its successor arrives twice. On a T-Embed
+(ESP32-S3) running CircuitPython 10.3.0, a 32,000-character paste came through
+wrong 31 times in 120, each time as one 64-byte packet replaced by the next.
+
+Sending one window and waiting for the board's ``\\x01`` before the next
+closes that gap: the board asks for more only after it has read everything
+sent, so no packet can arrive while an earlier one is still in the endpoint
+buffer. The same 120 pastes sent this way all came through. It gives up a
+little speed at best (43.8 KB/s against 54.8 KB/s for mpremote's way on its
+good run; its other run managed 6.5 KB/s), so MicroPython boards keep
+mpremote's way.
+"""
+
+from __future__ import annotations
+
+import struct
+from typing import Any, Callable, Optional
+
+_TRANSPORT_CLASS: Any = None
+
+
+def paced_raw_paste_write(transport: Any, command_bytes: bytes) -> None:
+    """mpremote's ``raw_paste_write``, with at most one window unacknowledged."""
+    try:
+        from mpremote.transport import TransportError
+    except ImportError:  # the sidecar always has mpremote; unit tests may not
+        TransportError = RuntimeError  # noqa: N806
+
+    serial = transport.serial
+    header = serial.read(2)
+    if len(header) != 2:
+        raise TransportError("no raw-paste window from the board: {!r}".format(header))
+    window = struct.unpack("<H", header)[0]
+    if window == 0:
+        raise TransportError("the board offered a raw-paste window of 0 bytes")
+
+    # The board grants a second window straight after the header. Take the
+    # byte off the wire and leave the credit unused.
+    first = serial.read(1)
+    if first == b"\x04":
+        serial.write(b"\x04")
+        return
+    if first != b"\x01":
+        raise TransportError("unexpected read during raw paste: {!r}".format(first))
+
+    i = 0
+    while i < len(command_bytes):
+        chunk = command_bytes[i : i + window]
+        serial.write(chunk)
+        i += len(chunk)
+        if i >= len(command_bytes):
+            break
+        # Wait until the board has read all of it.
+        data = serial.read(1)
+        if data == b"\x04":
+            # The board stopped reading (a syntax error, say). Acknowledge.
+            serial.write(b"\x04")
+            return
+        if data != b"\x01":
+            raise TransportError("unexpected read during raw paste: {!r}".format(data))
+
+    serial.write(b"\x04")
+    data = transport.read_until(1, b"\x04")
+    if not data.endswith(b"\x04"):
+        raise TransportError("could not complete raw paste: {!r}".format(data))
+
+
+def _transport_class() -> Any:
+    global _TRANSPORT_CLASS
+    if _TRANSPORT_CLASS is not None:
+        return _TRANSPORT_CLASS
+
+    from mpremote.transport_serial import SerialTransport
+
+    class PacedSerialTransport(SerialTransport):
+        """mpremote's serial transport; raw-paste waits for each window when
+        ``paced()`` says so (the sidecar's: the board runs CircuitPython)."""
+
+        paced: Optional[Callable[[], bool]] = None
+
+        def raw_paste_write(self, command_bytes: bytes) -> None:
+            if self.paced is not None and self.paced():
+                paced_raw_paste_write(self, command_bytes)
+            else:
+                super().raw_paste_write(command_bytes)
+
+    _TRANSPORT_CLASS = PacedSerialTransport
+    return _TRANSPORT_CLASS
+
+
+def open_serial_transport(device: str, baud: int, paced: Callable[[], bool]) -> Any:
+    transport = _transport_class()(device, baudrate=baud)
+    transport.paced = paced
+    return transport

cli/src/mpftp/sidecar.py

@@ -25,7 +25,7 @@
 from pathlib import Path
 from typing import Any, Callable, Optional
 
-from . import ble, config, mdns, webrepl, wifiboard
+from . import ble, config, mdns, rawpaste, webrepl, wifiboard
 
 
 def split_fs_path(path: str) -> tuple[bool, str]:
@@ -683,9 +683,11 @@ def _open_transport(self, device: str, baud: int) -> Any:
                         raise
                 time.sleep(wait)
             return webrepl.open_transport(device, password)
-        from mpremote.transport_serial import SerialTransport
-
-        return SerialTransport(device, baudrate=baud)
+        # Raw-paste one window at a time on CircuitPython, whose ESP32 native
+        # USB can garble a paste that runs ahead of it (rawpaste.py, mpftp#50).
+        return rawpaste.open_serial_transport(
+            device, baud, paced=lambda: (self.interpreter or "") == "circuitpython"
+        )
 
     def connect(
         self,

cli/tests/test_rawpaste.py

@@ -0,0 +1,160 @@
+"""Raw-paste pacing for CircuitPython over serial (mpftp#50)."""
+
+from __future__ import annotations
+
+import unittest
+from unittest import mock
+
+from mpftp import rawpaste
+
+try:
+    import mpremote  # noqa: F401
+
+    HAVE_MPREMOTE = True
+except ImportError:
+    HAVE_MPREMOTE = False
+
+WINDOW = 128
+
+
+class FakeBoard:
+    """The board's side of raw-paste, as shared/runtime/pyexec.c does it.
+
+    It announces a 128-byte window and one extra credit, reads what the host
+    wrote whenever the host waits on it, and sends ``\\x01`` for every window
+    it has read. ``most_in_flight`` is the largest amount the host had
+    written that the board had not read yet: what the board's USB receive
+    path has to hold at once.
+    """
+
+    def __init__(self, stop_after: int | None = None) -> None:
+        self.out = bytearray(bytes([WINDOW & 0xFF, WINDOW >> 8, 0x01]))
+        self.received = bytearray()
+        self.read_upto = 0
+        self.since_ack = 0
+        self.most_in_flight = 0
+        self.finished = False
+        self.stop_after = stop_after
+        self.stopped = False
+
+    # pyserial surface
+    def write(self, data: bytes) -> int:
+        if self.stopped:
+            if data == b"\x04":
+                self.finished = True
+            return len(data)
+        self.received += data
+        self.most_in_flight = max(self.most_in_flight, len(self.received) - self.read_upto)
+        if self.received.endswith(b"\x04"):
+            self._consume()
+        return len(data)
+
+    def _consume(self) -> None:
+        while self.read_upto < len(self.received):
+            c = self.received[self.read_upto]
+            self.read_upto += 1
+            if c == 0x04:
+                self.finished = True
+                self.out += b"\x04"
+                return
+            if self.stop_after is not None and self.read_upto >= self.stop_after:
+                self.stopped = True
+                self.out += b"\x04"
+                return
+            self.since_ack += 1
+            if self.since_ack == WINDOW:
+                self.since_ack = 0
+                self.out += b"\x01"
+
+    def read(self, n: int = 1) -> bytes:
+        if not self.out:
+            self._consume()
+        data = bytes(self.out[:n])
+        del self.out[:n]
+        return data
+
+    def inWaiting(self) -> int:  # noqa: N802 (pyserial spelling)
+        return len(self.out)
+
+
+class FakeTransport:
+    def __init__(self, board: FakeBoard) -> None:
+        self.serial = board
+
+    def read_until(self, min_num_bytes: int, ending: bytes, timeout: float = 10) -> bytes:
+        data = bytearray()
+        while not data.endswith(ending):
+            chunk = self.serial.read(1)
+            if not chunk:
+                break
+            data += chunk
+        return bytes(data)
+
+
+def _source(n: int) -> bytes:
+    return bytes(0x41 + (i % 26) for i in range(n))
+
+
+class PacedRawPasteTests(unittest.TestCase):
+    def test_everything_arrives_in_order(self) -> None:
+        for n in (1, WINDOW - 1, WINDOW, WINDOW + 1, 5 * WINDOW, 24_000):
+            board = FakeBoard()
+            src = _source(n)
+            rawpaste.paced_raw_paste_write(FakeTransport(board), src)
+            self.assertTrue(board.finished, n)
+            self.assertEqual(bytes(board.received), src + b"\x04", n)
+
+    def test_never_more_than_one_window_unread(self) -> None:
+        board = FakeBoard()
+        rawpaste.paced_raw_paste_write(FakeTransport(board), _source(24_000))
+        self.assertLessEqual(board.most_in_flight, WINDOW + 1)  # + the closing Ctrl-D
+
+    @unittest.skipUnless(HAVE_MPREMOTE, "mpremote not installed")
+    def test_mpremotes_own_write_runs_two_windows_ahead(self) -> None:
+        # The control: the same fake board sees mpremote's stock writer put
+        # two windows in flight, which is what garbles CircuitPython (#50).
+        from mpremote.transport_serial import SerialTransport
+
+        board = FakeBoard()
+        t = FakeTransport(board)
+        SerialTransport.raw_paste_write(t, _source(24_000))  # type: ignore[arg-type]
+        self.assertEqual(bytes(board.received), _source(24_000) + b"\x04")
+        self.assertGreater(board.most_in_flight, WINDOW + 1)
+
+    def test_board_that_stops_reading_is_acknowledged(self) -> None:
+        board = FakeBoard(stop_after=300)
+        rawpaste.paced_raw_paste_write(FakeTransport(board), _source(2_000))
+        self.assertTrue(board.finished)
+        self.assertLess(len(board.received), 2_000)
+
+
+@unittest.skipUnless(HAVE_MPREMOTE, "mpremote not installed")
+class TransportChoiceTests(unittest.TestCase):
+    def test_paces_only_when_told(self) -> None:
+        cls = rawpaste._transport_class()
+        t = cls.__new__(cls)
+        t.paced = lambda: False
+        with mock.patch.object(rawpaste, "paced_raw_paste_write") as paced, \
+                mock.patch("mpremote.transport_serial.SerialTransport.raw_paste_write") as stock:
+            t.raw_paste_write(b"x")
+            self.assertEqual((paced.call_count, stock.call_count), (0, 1))
+            t.paced = lambda: True
+            t.raw_paste_write(b"x")
+            self.assertEqual((paced.call_count, stock.call_count), (1, 1))
+
+    def test_sidecar_paces_circuitpython_only(self) -> None:
+        from mpftp.sidecar import Session
+
+        s = Session.__new__(Session)
+        s.interpreter = None
+        with mock.patch.object(rawpaste, "open_serial_transport", side_effect=lambda d, b, paced: paced):
+            paced = s._open_transport("COM25", 115200)
+        self.assertFalse(paced())
+        s.interpreter = "micropython"
+        self.assertFalse(paced())
+        s.interpreter = "circuitpython"
+        self.assertTrue(paced())
+
+
+if __name__ == "__main__":
+    unittest.main()