From 1f91862b80dac22386a29f638595a2fd2c9254b4 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:42:20 -0500 Subject: [PATCH 1/5] bledev.cpble: the CircuitPython backend over _bleio Both roles. Waits are short polls of _bleio's objects; writes land in PacketBuffer (notify characteristics) or CharacteristicBuffer (the rest). Notifications go in as the header of an empty PacketBuffer.write(), which steps around CircuitPython 10.3's stall when NimBLE's buffer pool runs dry and keeps each notification whole. The reported MTU is capped at 247. auto picks cpble where _bleio exists; nus and midi accept a raw _bleio.adapter; midi's clock uses supervisor.ticks_ms on CircuitPython. The board scripts for nus, the GATT contract and midi run on CircuitPython, and cpfiles_client/cpfiles_discovery reach CircuitPython's own file service. --- lib/bledev/auto.py | 10 +- lib/bledev/cpble.py | 994 ++++++++++++++++++++++++ lib/bledev/midi.py | 12 + lib/bledev/nus.py | 4 + pydevices-desktop.toml | 1 + tests/bledev_board/cpfiles_client.py | 158 ++++ tests/bledev_board/cpfiles_discovery.py | 35 + tests/bledev_board/gatt_central.py | 46 +- tests/bledev_board/gatt_peripheral.py | 28 +- tests/bledev_board/midi_server.py | 16 +- tests/bledev_board/nus_server.py | 39 +- tests/test_bledev_cpble.py | 306 ++++++++ 12 files changed, 1614 insertions(+), 35 deletions(-) create mode 100644 lib/bledev/cpble.py create mode 100644 tests/bledev_board/cpfiles_client.py create mode 100644 tests/bledev_board/cpfiles_discovery.py create mode 100644 tests/test_bledev_cpble.py diff --git a/lib/bledev/auto.py b/lib/bledev/auto.py index b14c983a..1b4e4ff0 100644 --- a/lib/bledev/auto.py +++ b/lib/bledev/auto.py @@ -4,7 +4,7 @@ ble = bledev.auto.adapter() picks, in order: ``mpble`` where MicroPython's ``bluetooth`` module exists, -``webble`` in a browser (PyScript, Pyodide, the Workbench simulator), and +``cpble`` where CircuitPython's ``_bleio`` does, ``webble`` in a browser (PyScript, Pyodide, the Workbench simulator), and ``bleak`` on CPython when bleak is installed (``pip install pydevices[ble]``). Pass ``backend="fake"`` (or set ``BLEDEV_BACKEND``) to force one; extra keyword arguments go to the backend's constructor. @@ -19,6 +19,7 @@ #: backend name -> (module, class). Each module stands alone and never imports this one. BACKENDS = { "mpble": ("bledev.mpble", "MPBLE"), + "cpble": ("bledev.cpble", "CPBLE"), "bleak": ("bledev.bleak", "BleakBLE"), "webble": ("bledev.webble", "WebBLE"), "fake": ("bledev.fake", "FakeBLE"), @@ -48,7 +49,9 @@ def backend_name(): return "webble" if _importable("bluetooth"): return "mpble" - return None # CircuitPython's _bleio waits for bledev.cpble + if _importable("_bleio"): + return "cpble" + return None if _in_browser(): return "webble" if _importable("bleak"): @@ -71,7 +74,8 @@ def adapter(backend=None, **kwargs): if name is None: raise UnsupportedError( "no BLE backend on this host: on CPython, pip install 'pydevices[ble]' " - "for bleak; on MicroPython, use firmware built with bluetooth" + "for bleak; on MicroPython, use firmware built with bluetooth; on " + "CircuitPython, a board with _bleio" ) if name not in BACKENDS: raise UnsupportedError("unknown bledev backend {!r}; one of {}".format(name, sorted(BACKENDS))) diff --git a/lib/bledev/cpble.py b/lib/bledev/cpble.py new file mode 100644 index 00000000..f1ef0f0f --- /dev/null +++ b/lib/bledev/cpble.py @@ -0,0 +1,994 @@ +"""CircuitPython backend, over ``_bleio``. Both roles. + + import bledev.cpble + ble = bledev.cpble.get() # the board's one radio, shared + +Needs CircuitPython with ``_bleio`` (every nRF52, and every ESP32 with BLE +except the S2), and the ``asyncio`` library from the Adafruit bundle +(``circup install asyncio``, which brings ``adafruit_ticks``). + +``_bleio`` is the model furthest from aioble's. aioble hands you events; +``_bleio`` hands you objects you poll: a connection's ``connected`` flag, +a buffer a characteristic's writes land in, a scan's iterator. So this backend +turns each wait into a short poll of those objects (every +:data:`POLL_MS`), and a few things work differently from the other backends: + +* **Some calls block the event loop.** ``_bleio`` has no async API. Connecting, + service discovery, a read, a write with response and a subscription each + wait inside ``_bleio`` until the radio answers, usually one or two + connection intervals, and nothing else in asyncio runs meanwhile. A scan + runs in slices of :data:`SCAN_SLICE_MS` for the same reason. +* **Writes land in ring buffers** that CircuitPython fills from the radio's + task. A characteristic you can notify on gets a ``PacketBuffer``, which + keeps each write whole but only takes writes from the central that + subscribed, after it subscribed. Any other writable characteristic gets a + ``CharacteristicBuffer``, a byte stream: with ``capture=True`` each + ``written()`` returns whatever bytes arrived since the last, not one write. + Both drop the oldest data when full and don't say so, so read them promptly; + the stream buffer's fill is checked and a full one raises. +* **One notification in flight per characteristic.** ``notify()`` raises + :class:`BusyError` while the previous one is still waiting for the radio; + every async sender (nus, midi) waits and tries again. That's how this backend + steps around a stall in CircuitPython 10.3's ``PacketBuffer`` (see + :meth:`_CPServerCharacteristic._send`), and it also keeps each notification + whole, where ``PacketBuffer`` would otherwise merge two into one. +* **Setting a value notifies.** ``Characteristic.write(data)`` on a + characteristic with notify or indicate sends it to every subscribed central, + whatever ``send_update`` says, because that's what ``_bleio`` does. +* **Indications** go out, but ``indicate()`` returns once they're queued: + ``_bleio`` doesn't report the acknowledgement. A central here can't receive + indications at all (CircuitPython's ESP32 port ignores them), so + ``capabilities()["indicate"]`` is False. +* **Pairing is "just works" only.** CircuitPython's ESP32 port has no passkey + or numeric comparison (``authenticated=True`` raises + :class:`UnsupportedError`), and it always bonds. Its MTU is fixed at build + time (256 on the ESP32 port), so ``config(mtu=...)`` is unsupported; both + sides exchange at connect. +* **A peripheral doesn't learn the central's address.** ``_bleio`` doesn't + expose it, so ``connection.device.address`` is ``"central"``. +* **The GATT server allows ten characteristics per service and two + descriptors per characteristic**, on the ESP32 port. + +The ``_bleio`` objects are there when you need them: ``ble.raw`` is +``_bleio.adapter`` and ``connection.native`` a ``_bleio.Connection``. +""" + +try: + import asyncio +except ImportError: # pragma: no cover + import uasyncio as asyncio + +import _bleio + +from . import ( + BLE, + BLEError, + BLETimeoutError, + BusyError, + ClientCharacteristic, + ClientDescriptor, + ClientService, + Connection, + DEFAULT_MTU, + Device, + DisconnectedError, + FLAG_INDICATE, + FLAG_NOTIFY, + FLAG_READ, + FLAG_WRITE, + FLAG_WRITE_NO_RESPONSE, + GattError, + INSUFFICIENT_AUTHENTICATION, + INSUFFICIENT_ENCRYPTION, + PairingError, + ScanResult, + Scanner, + UnsupportedError, + UUID, + decode_advertisement, + decode_service_data, + pack_advertisement, + sleep_ms, +) + +#: How often a wait looks at ``_bleio``'s objects, in milliseconds. +POLL_MS = 5 + +#: How long, in milliseconds, a characteristic keeps retrying its last +#: notification after sending it. See :meth:`_CPServerCharacteristic._send`. +RETRY_MS = 5000 + +#: How long one slice of a scan blocks the event loop, in milliseconds. +SCAN_SLICE_MS = 100 + +#: The largest attribute value ``_bleio`` holds (``BLE_ATT_ATTR_MAX_LEN``). +_MAX_VALUE = 512 + +#: The largest ATT payload a peer can send here: CircuitPython's ESP32 port +#: asks for an MTU of 256. +_MAX_PAYLOAD = 253 + +#: The MTU this backend reports, and sends by: 247, one link-layer packet, +#: even when the link agreed 256. A 253-byte notification is split into two +#: packets below NimBLE's ATT layer, which needs buffers the pool may not +#: have, and when NimBLE refuses a notification, CircuitPython 10.3's +#: PacketBuffer switches to a second outgoing buffer it never allocated for +#: notifications, and the next write faults the board (seen: a hard fault in +#: the nus gate, the board restarting in safe mode). +_MAX_MTU = 247 + +# How long a write without response keeps retrying a full buffer by default. +_WRITE_RETRY_MS = 2000 + +try: + from supervisor import ticks_ms as _ticks_ms +except ImportError: # pragma: no cover - CPython tests with a stub _bleio + import time as _time + + def _ticks_ms(): + return int(_time.monotonic() * 1000) & _TICKS_MASK + + +# supervisor.ticks_ms() wraps at 2**29. +_TICKS_MASK = (1 << 29) - 1 + + +def _elapsed(since): + return (_ticks_ms() - since) & _TICKS_MASK + + +def _cuuid(uuid): + """bledev UUID -> _bleio.UUID.""" + uuid = UUID(uuid) + short = uuid.short + if short is not None and short <= 0xFFFF: + return _bleio.UUID(short) + return _bleio.UUID(str(uuid)) + + +def _uuid(cuuid): + """_bleio.UUID -> bledev UUID.""" + if cuuid.size == 16: + return UUID(cuuid.uuid16) + return UUID(bytes(cuuid.uuid128)) + + +# _bleio's property bits are the Bluetooth spec's, as bledev's are; map them +# by name anyway, so a port that numbers them differently still works. +_PROPS = ( + (FLAG_READ, "READ"), + (FLAG_WRITE, "WRITE"), + (FLAG_WRITE_NO_RESPONSE, "WRITE_NO_RESPONSE"), + (FLAG_NOTIFY, "NOTIFY"), + (FLAG_INDICATE, "INDICATE"), +) + + +def _to_cprops(flags): + props = 0 + for flag, name in _PROPS: + if flags & flag: + props |= getattr(_bleio.Characteristic, name) + return props + + +def _from_cprops(props): + flags = 0 + for flag, name in _PROPS: + if props & getattr(_bleio.Characteristic, name): + flags |= flag + return flags + + +def _translate(e, what): + """A bledev error for an exception ``_bleio`` raised, or ``None``.""" + if isinstance(e, BLEError): + return e + if isinstance(e, ConnectionError): + return DisconnectedError("{}: not connected".format(what)) + if isinstance(e, TimeoutError): + return BLETimeoutError("{} timed out".format(what)) + if isinstance(e, MemoryError): + return BusyError("{}: the radio's buffers are full".format(what)) + security = getattr(_bleio, "SecurityError", None) + if security is not None and isinstance(e, security): + text = str(e).lower() + status = INSUFFICIENT_AUTHENTICATION if "authentication" in text else INSUFFICIENT_ENCRYPTION + return GattError(status, "{}: {}".format(what, e)) + if isinstance(e, _bleio.BluetoothError): + # A read or write that the peer refused comes back as NimBLE's + # "Unknown system firmware error: 261", 0x100 plus the ATT code, + # because _bleio checks that status as a NimBLE error. + text = str(e) + tail = text.rsplit(" ", 1)[-1] + try: + code = int(tail) + except ValueError: + code = None + if code is not None and 0x100 <= code < 0x200: + return GattError(code - 0x100, "{}: ATT error 0x{:02x}".format(what, code - 0x100)) + return BLEError("{}: {}".format(what, text)) + return None + + +class _Call: + """``with _Call("read"):`` turns ``_bleio``'s exceptions into bledev's.""" + + def __init__(self, what): + self._what = what + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + if exc_type is None: + return False + error = _translate(exc, self._what) + if error is not None and error is not exc: + raise error + return False + + +_shared = None + + +def get(adapter=None): + """The shared :class:`CPBLE` for this board's radio, made on first use.""" + global _shared + if _shared is None: + _shared = CPBLE(adapter) + return _shared + + +class CPBLE(BLE): + """The board's radio, ``_bleio.adapter`` unless you pass another. + + ``queue_limit`` bounds each notification and capture queue, and sizes the + packet buffers ``_bleio`` fills. ``stream_buffer`` is the byte size of the + buffer behind a write-only characteristic with ``capture=True`` (nus's RX): + what arrives while your code doesn't read. Prefer :func:`get`. + """ + + backend = "cpble" + + def __init__(self, adapter=None, *, queue_limit=64, stream_buffer=32768): + self.raw = adapter if adapter is not None else _bleio.adapter + if self.raw is None: + raise UnsupportedError("this board has no BLE adapter (_bleio.adapter is None)") + if not self.raw.enabled: + self.raw.enabled = True + self.queue_limit = queue_limit + self.stream_buffer = stream_buffer + self._services = () + self._native_services = [] + # _bleio.Connection -> _CPConnection. _bleio hands out one object per + # link, so identity is enough. + self._connections = {} + + def capabilities(self): + caps = BLE.capabilities(self) + caps.update( + central=True, + peripheral=True, + mtu=_MAX_MTU, + max_mtu=_MAX_MTU, + indicate=False, + pairing=True, + long_read=False, + ) + return caps + + def config(self, *names, **settings): + for key, value in settings.items(): + if key == "gap_name": + self.raw.name = value + elif key == "mtu": + if value != _MAX_MTU: + raise UnsupportedError( + "CircuitPython's MTU is fixed when it's built ({} here)".format(_MAX_MTU) + ) + else: + raise UnsupportedError("cpble has no setting {!r}".format(key)) + if names: + if len(names) != 1: + raise ValueError("read one setting at a time") + name = names[0] + if name == "mtu": + return _MAX_MTU + if name == "gap_name": + return self.raw.name + raise UnsupportedError("cpble has no setting {!r}".format(name)) + return None + + def close(self): + try: + self.raw.stop_advertising() + except Exception: + pass + try: + self.raw.stop_scan() + except Exception: + pass + for native in tuple(self.raw.connections): + try: + native.disconnect() + except Exception: + pass + self._connections = {} + + def enable_bonding(self, store=None): + """CircuitPython always bonds and keeps the keys itself (in NVS on an + ESP32), so this only accepts the call. ``forget_bonds()`` erases them.""" + + def forget_bonds(self): + """Erase every bond this board holds (``_bleio.adapter.erase_bonding()``).""" + self.raw.erase_bonding() + + # -- connections + + def _wrap(self, native, role, device=None): + conn = self._connections.get(native) + if conn is None: + if len(self._connections) > 8: + for key in list(self._connections): + if not self._connections[key].is_connected(): + del self._connections[key] + if device is None: + device = Device(self, "central") + conn = _CPConnection(self, device, role, native) + self._connections[native] = conn + return conn + + def _current(self): + """The live peripheral-side connection a write most likely came from. + + ``_bleio`` doesn't say which central wrote, so with more than one this + is the most recent one still connected.""" + found = None + for native in self.raw.connections: + conn = self._connections.get(native) + if conn is not None and conn.role == "peripheral" and conn.is_connected(): + found = conn + if found is None: + for native in self.raw.connections: + found = self._connections.get(native) or found + return found + + # -- peripheral + + def register_services(self, *services): + for service in self._services: + for characteristic in service.characteristics: + if characteristic._impl is not None: + characteristic._impl._release() + characteristic._impl = None + for native in self._native_services: + try: + native.deinit() + except Exception: + pass + self._native_services = [] + open_ = _bleio.Attribute.OPEN + for service in services: + if len(service.characteristics) > 10: + raise UnsupportedError("CircuitPython serves at most 10 characteristics per service") + nservice = _bleio.Service(_cuuid(service.uuid)) + self._native_services.append(nservice) + for c in service.characteristics: + if c.authenticated: + raise UnsupportedError("CircuitPython can't require a passkey (authenticated=True)") + perm = _bleio.Attribute.ENCRYPT_NO_MITM if c.encrypted else open_ + if len(c.descriptors) > 2: + raise UnsupportedError("CircuitPython allows two descriptors per characteristic") + # Every value gets the largest buffer: max_len is kept here, + # in Python, as the other backends keep it (a longer write + # keeps its first max_len bytes). _bleio would refuse the + # write instead, and a notification can't exceed the value's + # buffer. + nchar = _bleio.Characteristic.add_to_service( + nservice, + _cuuid(c.uuid), + properties=_to_cprops(c.flags), + read_perm=perm if c.flags & FLAG_READ or c.flags & (FLAG_NOTIFY | FLAG_INDICATE) else open_, + write_perm=perm if c.flags & (FLAG_WRITE | FLAG_WRITE_NO_RESPONSE) else _bleio.Attribute.NO_ACCESS, + max_length=_MAX_VALUE, + fixed_length=False, + initial_value=c._initial or b"", + ) + for d in c.descriptors: + _bleio.Descriptor.add_to_characteristic( + nchar, + _cuuid(d.uuid), + read_perm=perm, + write_perm=_bleio.Attribute.NO_ACCESS, + max_length=max(1, len(d.value)), + fixed_length=True, + initial_value=d.value or b"\x00", + ) + c._impl = _CPServerCharacteristic(self, c, nchar) + self._services = tuple(services) + + async def advertise( + self, + interval_us=100000, + *, + name=None, + services=None, + appearance=0, + manufacturer=None, + connectable=True, + timeout_ms=None, + service_data=None, + ): + adv_data, resp_data = pack_advertisement(name, services, appearance, manufacturer, service_data) + if name: + # The GAP name, which a paired computer lists, is the advertised one. + self.raw.name = name + interval = min(10.24, max(0.02, interval_us / 1000000)) + known = set(self.raw.connections) + if not known: + # Fresh packet buffers for the next link (see _renew), made + # before a central can connect and subscribe. + for service in self._services: + for c in service.characteristics: + if c._impl is not None and c._impl._packets is not None: + c._impl._renew() + try: + self.raw.stop_advertising() + except Exception: + pass + with _Call("advertise"): + self.raw.start_advertising( + adv_data, + scan_response=resp_data or None, + connectable=connectable, + interval=interval, + ) + start = _ticks_ms() + try: + while True: + for native in self.raw.connections: + if native not in known and native.connected: + # A connect stops advertising. + return self._wrap(native, "peripheral") + if timeout_ms is not None and _elapsed(start) >= timeout_ms: + raise BLETimeoutError("advertise: nobody connected in {} ms".format(timeout_ms)) + await sleep_ms(POLL_MS * 4) + finally: + if self.raw.advertising: + try: + self.raw.stop_advertising() + except Exception: + pass + + # -- central + + def scan(self, duration_ms, *, active=False, interval_us=None, window_us=None): + return _CPScanner(self, duration_ms, active, interval_us, window_us) + + async def _connect(self, device, timeout_ms, **options): + address = device.native + if not isinstance(address, _bleio.Address): + if device.addr is None: + raise BLEError("cpble connects by address; {!r} has none".format(device)) + address = _bleio.Address(bytes(reversed(device.addr)), device.addr_type) + try: + self.raw.stop_scan() + except Exception: + pass + await sleep_ms(0) + with _Call("connect"): + native = self.raw.connect(address, timeout=(timeout_ms or 10000) / 1000) + if native is None or not native.connected: + raise BLETimeoutError("connect timed out") + conn = self._wrap(native, "central", device) + interval_us = options.get("max_conn_interval_us") or options.get("min_conn_interval_us") + if interval_us: + try: + native.connection_interval = interval_us / 1000 + except Exception: + pass # the peripheral may refuse; the link keeps its interval + return conn + + +class _CPScanner(Scanner): + """Runs ``_bleio``'s scan in slices, so the event loop gets a turn between them.""" + + def __init__(self, ble, duration_ms, active, interval_us, window_us): + Scanner.__init__(self, ble, duration_ms, active, interval_us, window_us) + self._seen = {} + self._started = None + + async def _start(self): + self._started = _ticks_ms() + + async def _stop(self): + try: + self._ble.raw.stop_scan() + except Exception: + pass + + def _slice(self): + interval = (self.interval_us or 30000) / 1000000 + window = (self.window_us or self.interval_us or 30000) / 1000000 + left = self.duration_ms - _elapsed(self._started) + if left <= 0: + self._finish() + return + length = min(SCAN_SLICE_MS, left) / 1000 + with _Call("scan"): + results = self._ble.raw.start_scan( + timeout=length, + interval=interval, + window=min(window, interval), + active=self.active, + minimum_rssi=-127, + buffer_size=2048, + ) + try: + for entry in results: + self._entry(entry) + finally: + try: + self._ble.raw.stop_scan() + except Exception: + pass + + def _entry(self, entry): + raw = bytes(entry.advertisement_bytes) + key = bytes(entry.address.address_bytes) + seen = self._seen.get(key) + if seen is None: + seen = self._seen[key] = [b"", b"", True] + if entry.scan_response: + seen[1] = raw + else: + seen[0] = raw + seen[2] = entry.connectable + name, services, appearance, manufacturer = decode_advertisement(seen[0], seen[1]) + device = Device( + self._ble, bytes(reversed(key)), entry.address.type, name=name, native=entry.address + ) + self._push( + ScanResult( + device, + rssi=entry.rssi, + name=name, + services=services, + appearance=appearance, + manufacturer=manufacturer, + connectable=seen[2], + service_data=decode_service_data(seen[0], seen[1]), + ) + ) + + async def __anext__(self): + while True: + if self._queue: + return self._queue.pop(0) + if self._done: + raise StopAsyncIteration + self._slice() + await sleep_ms(0) + + +class _CPConnection(Connection): + def __init__(self, ble, device, role, native): + Connection.__init__(self, ble, device, role) + #: The ``_bleio.Connection``. + self.native = native + self._remote = None + self._last_mtu = DEFAULT_MTU + + @property + def mtu(self): + try: + self._last_mtu = min(self.native.max_packet_length + 3, _MAX_MTU) + except Exception: + pass # disconnected: keep the last value + return self._last_mtu + + def is_connected(self): + try: + return bool(self.native.connected) + except Exception: + return False + + async def disconnect(self, timeout_ms=2000): + if self.is_connected(): + try: + self.native.disconnect() + except Exception: + pass + await self.disconnected(timeout_ms) + + async def disconnected(self, timeout_ms=None): + start = _ticks_ms() + while self.is_connected(): + if timeout_ms is not None and _elapsed(start) >= timeout_ms: + raise BLETimeoutError("disconnected timed out after {} ms".format(timeout_ms)) + await sleep_ms(POLL_MS * 4) + + async def exchange_mtu(self, mtu=None, timeout_ms=1000): + # CircuitPython exchanges at every connect, from either side; wait for + # the result rather than asking again. + start = _ticks_ms() + while self.is_connected() and self.mtu <= DEFAULT_MTU and _elapsed(start) < (timeout_ms or 0): + await sleep_ms(POLL_MS * 4) + if not self.is_connected(): + raise DisconnectedError("exchange_mtu: disconnected") + return self.mtu + + @property + def encrypted(self): + try: + return bool(self.native.paired) + except Exception: + return False + + async def pair(self, bond=True, timeout_ms=30000, passkey=None): + if passkey is not None: + raise UnsupportedError("CircuitPython pairs only 'just works': no passkey") + if not bond: + raise UnsupportedError("CircuitPython always bonds") + if not self.is_connected(): + raise DisconnectedError("pair: not connected") + # _bleio's pair() blocks until the link is encrypted; a refusal leaves + # it waiting, which only a disconnect or Ctrl-C ends. + with _Call("pair"): + self.native.pair(bond=True) + if not self.encrypted: + if not self.is_connected(): + raise PairingError("pair: the peer hung up") + raise PairingError("pair: refused") + + async def unpair(self): + raise UnsupportedError("CircuitPython forgets one peer only by erasing every bond: ble.forget_bonds()") + + def _discover(self): + if self._remote is None: + if not self.is_connected(): + raise DisconnectedError("service discovery: not connected") + with _Call("service discovery"): + self._remote = [_CPClientService(self, s) for s in self.native.discover_remote_services()] + return self._remote + + async def _discover_services(self, uuid, timeout_ms): + await sleep_ms(0) + return [s for s in self._discover() if uuid is None or s.uuid == uuid] + + +class _CPClientService(ClientService): + def __init__(self, connection, native): + ClientService.__init__(self, connection, _uuid(native.uuid)) + self.native = native + self._characteristics = None + + async def _discover_characteristics(self, uuid, timeout_ms): + if self._characteristics is None: + self._characteristics = [_CPClientCharacteristic(self, c) for c in self.native.characteristics] + return [c for c in self._characteristics if uuid is None or c.uuid == uuid] + + +class _CPClientCharacteristic(ClientCharacteristic): + def __init__(self, service, native): + ClientCharacteristic.__init__(self, service, _uuid(native.uuid), _from_cprops(native.properties)) + self.native = native + self._buffer = None + self._packet = bytearray(_MAX_VALUE) + self._queue = [] + self._overrun = 0 + + def _live(self, what): + if not self.connection.is_connected(): + raise DisconnectedError("{}: not connected".format(what)) + + async def read(self, timeout_ms=1000): + self._check(FLAG_READ, "read") + self._live("read") + await sleep_ms(0) + with _Call("read"): + value = self.native.value + self._live("read") + return bytes(value) + + async def _discover_descriptors(self, uuid, timeout_ms): + self._live("descriptor discovery") + found = [_CPClientDescriptor(self, d) for d in self.native.descriptors] + return [d for d in found if uuid is None or d.uuid == uuid] + + async def write(self, data, response=None, timeout_ms=1000): + data = bytes(data) + response = self._want_response(response) + if response and self.properties & FLAG_WRITE_NO_RESPONSE: + # _bleio picks the write type from the properties, and a + # characteristic that allows both always gets one without. + raise UnsupportedError( + "CircuitPython writes {!r} without response; it can't ask for one".format(self) + ) + self._live("write") + if len(data) > self.connection.mtu - 3: + raise BLEError( + "write of {} bytes exceeds the ATT payload of {} (mtu {} - 3)".format( + len(data), self.connection.mtu - 3, self.connection.mtu + ) + ) + start = _ticks_ms() + limit = timeout_ms if timeout_ms is not None else _WRITE_RETRY_MS + while True: + try: + with _Call("write"): + self.native.value = data + return + except BusyError: + self._live("write") + if _elapsed(start) >= limit: + raise BLETimeoutError("write: buffers stayed full for {} ms".format(limit)) + await sleep_ms(2) + except BLEError: + self._live("write") + raise + + async def subscribe(self, notify=True, indicate=False): + if indicate: + raise UnsupportedError("CircuitPython can't receive indications") + if notify: + self._check(FLAG_NOTIFY, "notify") + self._live("subscribe") + await sleep_ms(0) + if notify and self._buffer is None: + # Making the buffer writes the CCCD. + with _Call("subscribe"): + self._buffer = _bleio.PacketBuffer( + self.native, buffer_size=self.connection._ble.queue_limit, max_packet_size=_MAX_PAYLOAD + ) + elif not notify and self._buffer is not None: + self._buffer.deinit() + self._buffer = None + with _Call("unsubscribe"): + self.native.set_cccd(notify=False, indicate=False) + + def _drain(self): + buffer = self._buffer + if buffer is None: + return + limit = self.connection._ble.queue_limit + while True: + try: + n = buffer.readinto(self._packet) + except Exception: + return + if n <= 0: + return + if len(self._queue) >= limit: + self._overrun += 1 + else: + self._queue.append(bytes(self._packet[:n])) + + async def notified(self, timeout_ms=None): + self._check(FLAG_NOTIFY, "notify") + start = _ticks_ms() + while True: + self._drain() + if self._overrun: + raise BLEError( + "notify {} queue overran: {} dropped because the reader fell behind".format( + self.uuid, self._overrun + ) + ) + if self._queue: + return self._queue.pop(0) + if not self.connection.is_connected(): + raise DisconnectedError("notified: disconnected") + if timeout_ms is not None and _elapsed(start) >= timeout_ms: + raise BLETimeoutError("notified timed out after {} ms".format(timeout_ms)) + await sleep_ms(POLL_MS) + + async def indicated(self, timeout_ms=None): + raise UnsupportedError("CircuitPython can't receive indications") + + +class _CPClientDescriptor(ClientDescriptor): + def __init__(self, characteristic, native): + ClientDescriptor.__init__(self, characteristic, _uuid(native.uuid)) + self.native = native + + async def read(self, timeout_ms=1000): + if not self.connection.is_connected(): + raise DisconnectedError("read: not connected") + await sleep_ms(0) + with _Call("descriptor read"): + return bytes(self.native.value) + + +_retrying = [] +_retry_task = None + + +def _retry(impl): + """Retry ``impl``'s last notification every :data:`POLL_MS` for + :data:`RETRY_MS`, in case the radio couldn't take it yet.""" + global _retry_task + impl._sent_at = _ticks_ms() + if impl not in _retrying: + _retrying.append(impl) + if _retry_task is None: + _retry_task = asyncio.create_task(_retry_loop()) + + +async def _retry_loop(): + global _retry_task + try: + while _retrying: + await sleep_ms(POLL_MS) + for impl in list(_retrying): + packets = impl._packets + if packets is None or _elapsed(impl._sent_at) >= RETRY_MS: + _retrying.remove(impl) + continue + try: + packets.write(b"") + except Exception: + _retrying.remove(impl) + finally: + _retry_task = None + + +class _CPServerCharacteristic: + """cpble's half of a :class:`bledev.Characteristic`.""" + + def __init__(self, ble, characteristic, native): + self._ble = ble + self._char = characteristic + self.native = native + self._packets = None # a PacketBuffer: notifications out, writes in + self._stream = None # a CharacteristicBuffer: writes in, as bytes + self._stream_size = 0 + self._queue = [] + self._overrun = 0 + self._scratch = None + self._sent_at = 0 + flags = characteristic.flags + writable = flags & (FLAG_WRITE | FLAG_WRITE_NO_RESPONSE) + if flags & (FLAG_NOTIFY | FLAG_INDICATE): + self._renew() + self._scratch = bytearray(_MAX_PAYLOAD) + elif writable: + self._stream_size = ble.stream_buffer if characteristic.capture else 1024 + self._stream = _bleio.CharacteristicBuffer(native, timeout=0, buffer_size=self._stream_size) + + def _renew(self): + """A fresh PacketBuffer, for each new connection. One that saw NimBLE + refuse a notification is left pointing at a buffer it never + allocated (see _MAX_MTU), so none is kept past its link.""" + if self._packets is not None: + try: + self._packets.deinit() + except Exception: + pass + writable = self._char.flags & (FLAG_WRITE | FLAG_WRITE_NO_RESPONSE) + self._packets = _bleio.PacketBuffer( + self.native, buffer_size=self._ble.queue_limit if writable else 1, max_packet_size=_MAX_PAYLOAD + ) + self._queue = [] + self._overrun = 0 + + def _release(self): + for buffer in (self._packets, self._stream): + if buffer is not None: + try: + buffer.deinit() + except Exception: + pass + self._packets = self._stream = None + + def read(self): + value = bytes(self.native.value) + if self._char.flags & (FLAG_WRITE | FLAG_WRITE_NO_RESPONSE): + return value[: self._char.max_len] + return value + + def write(self, data, send_update): + # Setting the value notifies subscribers whatever send_update says. + self.native.value = data + + def _check_size(self, connection, data): + if len(data) > connection.mtu - 3: + raise BLEError( + "notification of {} bytes exceeds the ATT payload of {} (mtu {} - 3)".format( + len(data), connection.mtu - 3, connection.mtu + ) + ) + + def _send(self, connection, data, what): + if not connection.is_connected(): + raise DisconnectedError("{}: disconnected".format(what)) + if data is None: + data = bytes(self.native.value) + self._check_size(connection, data) + # CircuitPython 10.3's PacketBuffer.write() keeps a packet it couldn't + # hand to NimBLE (the buffer pool was dry) and retries it only on the + # next "sent" event, which then never comes; the write after that + # waits for it forever inside _bleio. Ten notifications back to back + # were enough. So the packet goes in as the header of an empty write: + # _bleio copies a header only into an empty packet, and an empty + # write never waits, but it does retry what's left. 0 back means the + # previous packet is still there: busy, and it was just retried. + if self._packets.outgoing_packet_length < 0: + return # the central hasn't subscribed: as on every backend, it gets nothing + with _Call(what): + n = self._packets.write(b"", header=data) + if n == 0 and data: + if not connection.is_connected(): + raise DisconnectedError("{}: disconnected".format(what)) + raise BusyError("{}: the previous packet is still waiting for the radio".format(what)) + if n > 0: + # It may be the one left waiting; keep retrying it for a while. + _retry(self) + if n < 0 and not connection.is_connected(): + raise DisconnectedError("{}: disconnected".format(what)) + # n < 0 on a live link: the central hasn't subscribed, so, as on + # every backend, it gets nothing. + + def notify(self, connection, data): + self._send(connection, data, "notify") + + async def indicate(self, connection, data, timeout_ms): + if not self._char.flags & FLAG_INDICATE: + raise UnsupportedError("{!r} was not created with indicate=True".format(self._char)) + self._send(connection, data, "indicate") + await sleep_ms(0) + + def _drain(self): + """Move what ``_bleio`` buffered into the queue. True if anything came.""" + got = False + limit = self._ble.queue_limit + max_len = self._char.max_len + if self._packets is not None: + while True: + try: + n = self._packets.readinto(self._scratch) + except Exception: + break + if n <= 0: + break + got = True + if len(self._queue) >= limit: + self._overrun += 1 + else: + self._queue.append(bytes(self._scratch[: min(n, max_len)])) + elif self._stream is not None: + waiting = self._stream.in_waiting + if waiting: + if waiting >= self._stream_size - 1: + # A full ring drops what doesn't fit, silently. Say so. + self._overrun += 1 + data = self._stream.read(waiting) + if data: + got = True + if len(self._queue) >= limit: + self._overrun += 1 + else: + self._queue.append(bytes(data)) + return got + + async def written(self, timeout_ms): + start = _ticks_ms() + while True: + self._drain() + if self._overrun: + raise BLEError( + "writes to {} overran: {} lost because the reader fell behind".format( + self._char.uuid, self._overrun + ) + ) + if self._queue: + data = self._queue.pop(0) + connection = self._ble._current() + if self._char.capture: + return connection, data + self._queue = [] + return connection + if timeout_ms is not None and _elapsed(start) >= timeout_ms: + raise BLETimeoutError("written timed out after {} ms".format(timeout_ms)) + await sleep_ms(POLL_MS) diff --git a/lib/bledev/midi.py b/lib/bledev/midi.py index 3c3f5f64..8ab20c1d 100644 --- a/lib/bledev/midi.py +++ b/lib/bledev/midi.py @@ -134,6 +134,14 @@ def now_ms(): """This host's BLE-MIDI clock: milliseconds, 13 bits.""" return time.ticks_ms() & TIMESTAMP_MASK +elif sys.implementation.name == "circuitpython": + # time.monotonic() is a float that loses milliseconds after a few hours. + from supervisor import ticks_ms as _ticks_ms + + def now_ms(): + """This host's BLE-MIDI clock: milliseconds, 13 bits.""" + return _ticks_ms() & TIMESTAMP_MASK + else: import time @@ -149,6 +157,10 @@ def _adapter(ble): from . import mpble return mpble.get(ble) + if hasattr(ble, "start_advertising") and hasattr(ble, "erase_bonding"): + from . import cpble # CircuitPython's _bleio.adapter + + return cpble.get(ble) raise TypeError("expected a bledev adapter, not {!r}".format(ble)) diff --git a/lib/bledev/nus.py b/lib/bledev/nus.py index 07f7388c..612b44a2 100644 --- a/lib/bledev/nus.py +++ b/lib/bledev/nus.py @@ -67,6 +67,10 @@ def _adapter(ble): from . import mpble return mpble.get(ble) + if hasattr(ble, "start_advertising") and hasattr(ble, "erase_bonding"): + from . import cpble # CircuitPython's _bleio.adapter + + return cpble.get(ble) raise TypeError("expected a bledev adapter, not {!r}".format(ble)) diff --git a/pydevices-desktop.toml b/pydevices-desktop.toml index cb093a19..500cecdb 100644 --- a/pydevices-desktop.toml +++ b/pydevices-desktop.toml @@ -23,6 +23,7 @@ "https://raw.githubusercontent.com/PyDevices/pydevices/main/lib/bledev/__init__.py" = "/lib/bledev/__init__.py" "https://raw.githubusercontent.com/PyDevices/pydevices/main/lib/bledev/auto.py" = "/lib/bledev/auto.py" "https://raw.githubusercontent.com/PyDevices/pydevices/main/lib/bledev/bleak.py" = "/lib/bledev/bleak.py" +"https://raw.githubusercontent.com/PyDevices/pydevices/main/lib/bledev/cpble.py" = "/lib/bledev/cpble.py" "https://raw.githubusercontent.com/PyDevices/pydevices/main/lib/bledev/fake.py" = "/lib/bledev/fake.py" "https://raw.githubusercontent.com/PyDevices/pydevices/main/lib/bledev/filetransfer.py" = "/lib/bledev/filetransfer.py" "https://raw.githubusercontent.com/PyDevices/pydevices/main/lib/bledev/hid.py" = "/lib/bledev/hid.py" diff --git a/tests/bledev_board/cpfiles_client.py b/tests/bledev_board/cpfiles_client.py new file mode 100644 index 00000000..8d2928cd --- /dev/null +++ b/tests/bledev_board/cpfiles_client.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: 2026 Brad Barnett +# +# SPDX-License-Identifier: MIT +"""CircuitPython's own BLE file service, reached with bledev's file client. + +This is the other half of "one client serves both interpreters" +(docs/ble.md ยง4): ``files_client.py`` checks a MicroPython board serving +``bledev.filetransfer``, and this checks a CircuitPython board's supervisor, +which serves the same protocol and wants pairing ("just works") first. + +The board must be advertising its file service publicly, which CircuitPython +does only in discovery mode: a reset pressed during the blue blink after boot, +or ``cpfiles_discovery.py`` run on the board, which does the same without +hands. Runs on a laptop (bleak, Windows Python):: + + PYTHONPATH="$(wslpath -w lib)" python.exe "$(wslpath -w tests/bledev_board/cpfiles_client.py)" [--plant] [--runs N] [--keep] + +Checks, in order: a client that won't pair is refused; one that pairs by +itself (``pair=None``, the default) gets in, on an encrypted link; 20 KB up +and back down byte for byte (timed, ``--runs N`` times), a read from an +offset, mkdir, listdir, move, delete; then a reconnect from the bond, with +no pairing. Then it unpairs, unless ``--keep``, +so Windows' Bluetooth list ends as it started. ``--plant`` flips a bit in +one packet the client receives, and the run must FAIL. Ends with +``RESULT PASS`` or ``RESULT FAIL``. +""" +import asyncio +import sys +import time + +import bledev +import bledev.bleak +import bledev.filetransfer as ft + +ARGS = sys.argv[1:] +PLANT = "--plant" in ARGS +KEEP = "--keep" in ARGS +RUNS = 1 +for i, arg in enumerate(ARGS): + if arg == "--runs": + RUNS = int(ARGS[i + 1]) + +failures = [] + + +def check(ok, what, detail=""): + print("PASS" if ok else "FAIL", what, detail) + if not ok: + failures.append(what) + + +def ticks(): + return time.monotonic() * 1000 + + +def pattern(n, seed): + out = bytearray(n) + x = seed + for i in range(n): + x = (x * 1103515245 + 12345) & 0x7FFFFFFF + out[i] = (x >> 16) & 0xFF + return bytes(out) + + +async def main(): + ble = bledev.bleak.BleakBLE() + device = await ble.find(service=ft.SERVICE, timeout_ms=20000) + print("found", device) + + try: + files = await ft.connect(ble, device=device, pair=False, timeout_ms=15000) + check(False, "a client that won't pair is refused", "it got in") + await files.close() + except bledev.BLEError as e: + check(True, "a client that won't pair is refused", type(e).__name__) + await bledev.sleep_ms(2000) + + t0 = ticks() + files = await ft.connect(ble, device=device, timeout_ms=15000) + check(files.connection.encrypted, "the client paired by itself", "{:.0f} ms to connect and pair".format(ticks() - t0)) + print("connected, mtu", files.connection.mtu, "protocol version", files.version) + if PLANT: + feed = files._feed + state = {"n": 0} + + def planted(data): + if len(data) > 28: + state["n"] += 1 + if state["n"] == 3: + data = bytes(data[:-1]) + bytes((data[-1] ^ 0x10,)) + feed(data) + + files._feed = planted + + try: + await files.delete("/ftgate") + except ft.FileTransferError: + pass + # CircuitPython's mkdir makes one level; bledev's server makes the parents too. + await files.mkdir("/ftgate") + await files.mkdir("/ftgate/sub") + data = pattern(20 * 1024, 7) + for run in range(RUNS): + t0 = ticks() + await files.write("/ftgate/big.bin", data) + t1 = ticks() + back = await files.read("/ftgate/big.bin") + t2 = ticks() + print("run {}: 20 KB up {:.0f} ms ({:.1f} KB/s), down {:.0f} ms ({:.1f} KB/s)".format( + run + 1, t1 - t0, len(data) / (t1 - t0), t2 - t1, len(back) / max(1, t2 - t1))) + check(back == data, "20 KB round trip byte for byte", "{} bytes".format(len(back))) + check(await files.read("/ftgate/big.bin", offset=20000) == data[20000:], "read from an offset") + entries = dict((e[0], e) for e in await files.listdir("/ftgate")) + check(entries.get("big.bin", (0, 0))[1] == len(data) and entries["sub"][2], "listdir", repr(sorted(entries))) + await files.move("/ftgate/big.bin", "/ftgate/sub/moved.bin") + names = [e[0] for e in await files.listdir("/ftgate/sub")] + check(names == ["moved.bin"], "move", repr(names)) + await files.delete("/ftgate") + names = [e[0] for e in await files.listdir("/")] + check("ftgate" not in names, "delete a directory and its contents", repr(names)) + await files.close() + + # Bonded now, the supervisor advertises privately, with neither a name nor + # its service, so find it by address. No pairing this time: the bond does it. + await bledev.sleep_ms(2000) + again = None + async with ble.scan(20000, active=True) as scanner: + async for result in scanner: + if result.device.address == device.address: + again = result.device + break + check(again is not None, "the bonded board advertises again") + t0 = ticks() + files = await ft.connect(ble, device=again, pair=False, timeout_ms=15000) + names = [e[0] for e in await files.listdir("/")] + check("boot_out.txt" in names, "a reconnect from the bond, no pairing", + "{:.0f} ms to connect and list".format(ticks() - t0)) + await files.close() + if not KEEP: + await bledev.sleep_ms(1000) + await bledev.bleak.unpair(device.address) + print("unpaired", device.address) + + +async def guarded(): + try: + await main() + except Exception as e: + failures.append("exception") + print("EXCEPTION", type(e).__name__, e) + import traceback + + traceback.print_exc() + print("RESULT", "FAIL {}".format(failures) if failures else "PASS") + + +asyncio.run(guarded()) +sys.exit(1 if failures else 0) diff --git a/tests/bledev_board/cpfiles_discovery.py b/tests/bledev_board/cpfiles_discovery.py new file mode 100644 index 00000000..d12769b5 --- /dev/null +++ b/tests/bledev_board/cpfiles_discovery.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2026 Brad Barnett +# +# SPDX-License-Identifier: MIT +"""Restart a CircuitPython ESP32-S3 in BLE discovery mode, with no hand on the board. + +CircuitPython's BLE workflow (the file service and the serial console) only +advertises to a new host in discovery mode. You get there by pressing reset +during the second of blue blinking after boot: the supervisor writes a marker +into an RTC register that survives a reset (``port_set_saved_word``), and +clears it once the second is up, so a reset inside that second boots with the +marker set (``supervisor/shared/bluetooth/bluetooth.c``). Discovery mode also +erases the board's bonds. + +This writes the same marker and resets, which is the same thing without the +button. ``memorymap`` may write the RTC peripheral's registers on the S3, and +the marker lives in ``RTC_CNTL_STORE0_REG``. Run it with +``mpftp exec -d COMx --no-follow "$(cat cpfiles_discovery.py)"`` or copy it +over; the board restarts at once. Then ``cpfiles_client.py`` on a laptop. +""" +import memorymap +import microcontroller +import os +import struct + +# RTC_CNTL_STORE0_REG: DR_REG_RTCCNTL_BASE (0x60008000) + 0x50, on the ESP32-S3. +STORE0 = {"esp32s3": 0x60008050} +# BLE_DISCOVERY_DATA_GUARD with a mode of 1, as the supervisor writes it. +MARKER = 0xBB0000BB | (1 << 8) + +chip = os.uname().sysname.lower() # "ESP32S3" +if chip not in STORE0: + raise RuntimeError("no saved-word register known for " + chip) +register = memorymap.AddressRange(start=STORE0[chip], length=4) +register[0:4] = struct.pack(" F7``: * mode 1, the gate: this side reads 1,000 mixed messages (``mix()``) and @@ -24,7 +24,11 @@ import sys import bledev.midi as midi -import bledev.mpble + +if sys.implementation.name == "circuitpython": + import bledev.cpble as backend # mpftp run keeps Bluetooth on here +else: + import bledev.mpble as backend PLANT = None # None, "drop" or "flip" LOG = "/midi_server.log" @@ -137,7 +141,7 @@ async def main(): os.remove(LOG) except OSError: pass - ble = bledev.mpble.get() + ble = backend.get() log("serving BLE-MIDI as", NAME, "plant", PLANT) while True: port = await midi.serve(ble, name=NAME) @@ -158,5 +162,5 @@ async def main(): await port.aclose() -if sys.implementation.name == "micropython": +if sys.implementation.name in ("micropython", "circuitpython"): asyncio.run(main()) diff --git a/tests/bledev_board/nus_server.py b/tests/bledev_board/nus_server.py index e7320163..6c90b8e4 100644 --- a/tests/bledev_board/nus_server.py +++ b/tests/bledev_board/nus_server.py @@ -9,10 +9,23 @@ Set PLANT = True to flip one bit at offset 5000 of everything this side sends; the client must then report FAIL. Logs to /gate_server.log. """ -import asyncio, time -import bledev.mpble +import asyncio, sys, time import bledev.nus as nus +if sys.implementation.name == "circuitpython": + # CircuitPython: bledev.cpble, and no time.ticks_ms. Its filesystem is + # read-only to code while USB mounts it, so the log is printed only. + import bledev.cpble as backend + from supervisor import ticks_ms + + def ticks_diff(a, b): + return (a - b) & ((1 << 29) - 1) + +else: + import bledev.mpble as backend + + ticks_ms, ticks_diff = time.ticks_ms, time.ticks_diff + PLANT = False # the planted run sets this: flip one bit at offset 5000 of what we send LOG = "/gate_server.log" @@ -20,8 +33,11 @@ def log(*parts): line = " ".join(str(p) for p in parts) print(line) - with open(LOG, "a") as f: - f.write(line + "\n") + try: + with open(LOG, "a") as f: + f.write(line + "\n") + except OSError: + pass def pattern(n, seed): @@ -48,13 +64,16 @@ def compare(got, want): async def main(): - open(LOG, "w").close() + try: + open(LOG, "w").close() + except OSError: + pass await gate(log) async def gate(log): """Serve one gate run; ``coex_server.py`` imports this.""" - ble = bledev.mpble.get() + ble = backend.get() log("serving as bledev-gate, plant", PLANT) link = await nus.serve(ble, name="bledev-gate", timeout_ms=120000) log("connected, mtu", link.connection.mtu) @@ -65,18 +84,18 @@ async def gate(log): op, n = cmd.split() n = int(n) if op == "UP": - t0 = time.ticks_ms() + t0 = ticks_ms() got = await link.readexactly(n) - ms = time.ticks_diff(time.ticks_ms(), t0) + ms = ticks_diff(ticks_ms(), t0) bad, first = compare(got, pattern(n, 1)) verdict = "OK" if bad == 0 and len(got) == n else "BAD" log("UP", verdict, n, "bytes", ms, "ms mismatches", bad, "first", first, "mtu", link.connection.mtu) await link.write("UP {} {} {} {}\n".format(verdict, ms, bad, first).encode()) elif op == "DOWN": data = plant(pattern(n, 2)) - t0 = time.ticks_ms() + t0 = ticks_ms() await link.write(data) - log("DOWN sent", n, "bytes in", time.ticks_diff(time.ticks_ms(), t0), "ms") + log("DOWN sent", n, "bytes in", ticks_diff(ticks_ms(), t0), "ms") elif op == "ECHO": left = n sent = 0 diff --git a/tests/test_bledev_cpble.py b/tests/test_bledev_cpble.py new file mode 100644 index 00000000..1b83c537 --- /dev/null +++ b/tests/test_bledev_cpble.py @@ -0,0 +1,306 @@ +# SPDX-FileCopyrightText: 2026 Brad Barnett +# +# SPDX-License-Identifier: MIT +"""bledev.cpble without a board: a stand-in ``_bleio`` that behaves the way +CircuitPython 10.3's does where it matters, and checks of the parts cpble adds. + +The stand-in's ``PacketBuffer`` does what the real one does with a buffer pool +that has run dry: it keeps the packet, appends the next write to it (so two +notifications arrive as one), and a write that doesn't fit beside the kept +packet would wait forever inside ``_bleio``, which here raises instead. The +checks prove notifications leave one per ``notify()`` and never hit that wait, +and a planted fault (cpble's send reverted to a plain ``write(data)``) must +fail them. The same code over the real radio is ``bledev_board/``. +""" + +import asyncio +import sys +import types +import unittest + +import _env # noqa: F401 (puts lib/ on sys.path) + +PAYLOAD = 244 + + +class _Stall(Exception): + """What _bleio would do instead: wait forever for a packet nothing retries.""" + + +def _stub(): + m = types.ModuleType("_bleio") + + class BluetoothError(Exception): + pass + + class SecurityError(BluetoothError): + pass + + class UUID: + def __init__(self, value): + if isinstance(value, int): + self.size, self.uuid16, self._raw = 16, value, None + else: + raw = bytes.fromhex(value.replace("-", "")) + self.size, self.uuid16, self._raw = 128, None, raw + + @property + def uuid128(self): + return bytes(reversed(self._raw)) # little-endian, as _bleio's + + class Attribute: + NO_ACCESS, OPEN, ENCRYPT_NO_MITM, ENCRYPT_WITH_MITM = 0, 1, 2, 3 + + class Service: + def __init__(self, uuid): + self.uuid = uuid + self.characteristics = [] + + def deinit(self): + pass + + class Characteristic: + BROADCAST, READ, WRITE_NO_RESPONSE, WRITE, NOTIFY, INDICATE = 1, 2, 4, 8, 16, 32 + + def __init__(self): + self.value = b"" + + @classmethod + def add_to_service(cls, service, uuid, *, properties, read_perm, write_perm, max_length, fixed_length, initial_value): + c = cls() + c.uuid, c.properties, c.value = uuid, properties, bytes(initial_value) + service.characteristics.append(c) + return c + + class Descriptor: + @staticmethod + def add_to_characteristic(*args, **kwargs): + return None + + class PacketBuffer: + def __init__(self, characteristic, *, buffer_size, max_packet_size=None): + self.characteristic = characteristic + characteristic.packets = self + self.pending = b"" + self.sent = [] + self.incoming = [] + self.subscribed = True + self.pool = True # False: NimBLE has no mbufs left + + @property + def outgoing_packet_length(self): + return PAYLOAD if self.subscribed else -1 + + def _try(self): + if self.pending and self.pool: + self.sent.append(self.pending) + self.pending = b"" + + def write(self, data, *, header=None): + if not self.subscribed: + return 0 + data, header = bytes(data), bytes(header or b"") + if self.pending and len(data) + len(self.pending) > PAYLOAD: + raise _Stall("PacketBuffer.write would wait forever") + n = 0 + if not self.pending: + self.pending += header + n += len(header) + self.pending += data + self._try() + return n + len(data) + + def readinto(self, buf): + if not self.incoming: + return 0 + packet = self.incoming.pop(0) + buf[: len(packet)] = packet + return len(packet) + + def deinit(self): + pass + + class CharacteristicBuffer: + def __init__(self, characteristic, *, timeout, buffer_size): + characteristic.stream = self + self.data = b"" + self.size = buffer_size + + def feed(self, data): + room = self.size - len(self.data) + self.data += data[:room] # a full ring drops the rest, silently + + @property + def in_waiting(self): + return len(self.data) + + def read(self, n): + out, self.data = self.data[:n], self.data[n:] + return out + + def deinit(self): + pass + + class Connection: + connected = True + max_packet_length = 253 + paired = False + + def disconnect(self): + self.connected = False + + class Adapter: + enabled = True + name = "stub" + advertising = False + + def __init__(self): + self.connections = () + + def start_advertising(self, data, *, scan_response=None, connectable=True, interval=0.1): + self.advertised = (bytes(data), scan_response) + self.advertising = True + self.connections = (Connection(),) # a central connects at once + + def stop_advertising(self): + self.advertising = False + + def stop_scan(self): + pass + + for name, value in list(locals().items()): + if isinstance(value, type): + setattr(m, name, value) + m.adapter = Adapter() + return m + + +sys.modules["_bleio"] = _stub() + +import bledev # noqa: E402 +import bledev.cpble as cpble # noqa: E402 +import bledev.nus as nus # noqa: E402 + +_bleio = sys.modules["_bleio"] + + +def run(coro): + return asyncio.run(coro) + + +async def _serve(queue_limit=64, stream_buffer=32768): + _bleio.adapter = _bleio.Adapter() + cpble._shared = None + ble = cpble.CPBLE(queue_limit=queue_limit, stream_buffer=stream_buffer) + service = bledev.Service(nus.SERVICE) + rx = bledev.Characteristic(service, nus.RX, write=True, write_no_response=True, capture=True, max_len=244) + tx = bledev.Characteristic(service, nus.TX, notify=True) + stream = bledev.Characteristic(service, "12345678-1234-5678-1234-56789abcdef2", + write_no_response=True, notify=True, capture=True, max_len=100) + ble.register_services(service) + connection = await ble.advertise(name="gate", services=[nus.SERVICE]) + return ble, connection, rx, tx, stream + + +class CPBLEChecks(unittest.TestCase): + def test_uuids_round_trip(self): + for uuid in (bledev.UUID(0x180F), nus.SERVICE): + self.assertEqual(cpble._uuid(cpble._cuuid(uuid)), uuid) + + def test_errors_translate(self): + e = cpble._translate(_bleio.BluetoothError("Unknown system firmware error: 271"), "read") + self.assertIsInstance(e, bledev.GattError) + self.assertEqual(e.status, bledev.INSUFFICIENT_ENCRYPTION) + self.assertIsInstance(cpble._translate(ConnectionError("Not connected"), "w"), bledev.DisconnectedError) + self.assertIsInstance(cpble._translate(MemoryError("Nimble out of memory"), "w"), bledev.BusyError) + self.assertIsInstance(cpble._translate(TimeoutError(), "w"), bledev.BLETimeoutError) + + def test_advertises_what_the_other_backends_do(self): + async def main(): + ble, connection, *_ = await _serve() + adv, resp = bledev.pack_advertisement("gate", [nus.SERVICE]) + self.assertEqual(ble.raw.advertised[0], adv) + self.assertEqual(ble.raw.name, "gate") + self.assertEqual(connection.role, "peripheral") + self.assertEqual(connection.mtu, 247) # capped: one link-layer packet + + run(main()) + + def test_notifications_leave_whole_and_never_stall(self): + async def main(): + ble, connection, rx, tx, stream = await _serve() + pb = tx._impl._packets + sent = [] + for i in range(40): + packet = bytes([i]) * 200 + pb.pool = i % 3 != 1 # the pool runs dry every third packet + while True: + try: + tx.notify(connection, packet) + break + except bledev.BusyError: + pb.pool = True # the radio caught up + await asyncio.sleep(0) + sent.append(packet) + pb.pool = True + await asyncio.sleep(0.02) # the retry loop sends what's left + return pb.sent, sent + + got, sent = run(main()) + self.assertEqual(got, sent) + + def test_an_unsubscribed_central_gets_nothing(self): + async def main(): + ble, connection, rx, tx, stream = await _serve() + tx._impl._packets.subscribed = False + tx.notify(connection, b"early") + return tx._impl._packets.sent + + self.assertEqual(run(main()), []) + + def test_captured_writes_in_order_and_cut_to_max_len(self): + async def main(): + ble, connection, rx, tx, stream = await _serve() + stream._impl._packets.incoming = [bytes([i]) * 120 for i in range(5)] + got = [] + for _ in range(5): + conn, data = await stream.written(timeout_ms=100) + self.assertIs(conn, connection) + got.append(data) + return got + + self.assertEqual(run(main()), [bytes([i]) * 100 for i in range(5)]) + + def test_a_full_stream_buffer_is_an_error(self): + async def main(): + ble, connection, rx, tx, stream = await _serve(stream_buffer=1024) + rx._impl._stream.feed(bytes(2000)) + with self.assertRaises(bledev.BLEError): + await rx.written(timeout_ms=100) + + run(main()) + + +class Planted(unittest.TestCase): + """cpble's send reverted to a plain PacketBuffer.write(data): the checks above must fail.""" + + def test_plain_write_fails_the_checks(self): + original = cpble._CPServerCharacteristic._send + + def plain(self, connection, data, what): + if self._packets.outgoing_packet_length < 0: + return + self._packets.write(data) + + cpble._CPServerCharacteristic._send = plain + try: + checks = CPBLEChecks("test_notifications_leave_whole_and_never_stall") + result = unittest.TestResult() + checks.run(result) + self.assertFalse(result.wasSuccessful(), "the planted plain write went unnoticed") + finally: + cpble._CPServerCharacteristic._send = original + + +if __name__ == "__main__": + unittest.main() From 257125c2f454269ab93e2eca300bde5eb3b1b62b Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:45:31 -0500 Subject: [PATCH 2/5] bledev: run the contract and codec checks on CircuitPython's own interpreter; nus_client runs on CircuitPython --- lib/bledev/midi.py | 5 ++- tests/bledev_board/nus_client.py | 34 +++++++++++++---- tests/test_bledev_cpble.py | 64 ++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 9 deletions(-) diff --git a/lib/bledev/midi.py b/lib/bledev/midi.py index 8ab20c1d..2a7481b6 100644 --- a/lib/bledev/midi.py +++ b/lib/bledev/midi.py @@ -136,7 +136,10 @@ def now_ms(): elif sys.implementation.name == "circuitpython": # time.monotonic() is a float that loses milliseconds after a few hours. - from supervisor import ticks_ms as _ticks_ms + try: + from supervisor import ticks_ms as _ticks_ms + except ImportError: # the unix port has no supervisor, and MicroPython's time + from time import ticks_ms as _ticks_ms def now_ms(): """This host's BLE-MIDI clock: milliseconds, 13 bits.""" diff --git a/tests/bledev_board/nus_client.py b/tests/bledev_board/nus_client.py index 1ad2e4c8..2a3ecf17 100644 --- a/tests/bledev_board/nus_client.py +++ b/tests/bledev_board/nus_client.py @@ -14,9 +14,17 @@ import asyncio, time, gc, sys import bledev.nus as nus -MICROPYTHON = sys.implementation.name == "micropython" -if MICROPYTHON: - import bledev.mpble +CIRCUITPYTHON = sys.implementation.name == "circuitpython" +MICROPYTHON = sys.implementation.name == "micropython" or CIRCUITPYTHON # a board +if CIRCUITPYTHON: + import bledev.cpble as backend + from supervisor import ticks_ms + + def ticks_diff(a, b): + return (a - b) & ((1 << 29) - 1) + +elif MICROPYTHON: + import bledev.mpble as backend ticks_ms, ticks_diff = time.ticks_ms, time.ticks_diff else: @@ -43,8 +51,11 @@ def log(*parts): line = " ".join(str(p) for p in parts) print(line) if LOG: - with open(LOG, "a") as f: - f.write(line + "\n") + try: + with open(LOG, "a") as f: + f.write(line + "\n") + except OSError: + pass # CircuitPython: the filesystem is USB's while it's mounted def pattern(n, seed): @@ -68,7 +79,7 @@ def kbps(n, ms): async def main(): gc.collect() - ble = bledev.mpble.get() if MICROPYTHON else bledev.bleak.BleakBLE() + ble = backend.get() if MICROPYTHON else bledev.bleak.BleakBLE() t0 = ticks_ms() link = await nus.connect(ble, name="bledev-gate", timeout_ms=20000, **CONN) log("connected in", ticks_diff(ticks_ms(), t0), "ms, mtu", link.connection.mtu, "conn", CONN, "plant", PLANT) @@ -126,7 +137,11 @@ async def guarded(): try: await main() except BaseException as e: - if MICROPYTHON: + if CIRCUITPYTHON: + import traceback + + log("EXCEPTION", "".join(traceback.format_exception(e))) + elif MICROPYTHON: import io buf = io.StringIO() @@ -140,6 +155,9 @@ async def guarded(): if LOG: - open(LOG, "w").close() + try: + open(LOG, "w").close() + except OSError: + pass log("start") asyncio.run(guarded()) diff --git a/tests/test_bledev_cpble.py b/tests/test_bledev_cpble.py index 1b83c537..5c28a098 100644 --- a/tests/test_bledev_cpble.py +++ b/tests/test_bledev_cpble.py @@ -281,6 +281,70 @@ async def main(): run(main()) +def _circuitpython(): + """A CircuitPython unix binary and a directory holding Adafruit's asyncio + and adafruit_ticks, or ``(None, why)``. ``PYDEVICES_CIRCUITPYTHON`` and + ``PYDEVICES_CP_ASYNCIO`` name them; otherwise the workspace layout, where + ``bin/circuitpython`` and the ``circuitpython`` checkout (whose ``frozen/`` + carries both libraries) sit beside this repository.""" + import os + from pathlib import Path + + binary = os.environ.get("PYDEVICES_CIRCUITPYTHON") + libs = os.environ.get("PYDEVICES_CP_ASYNCIO") + here = Path(__file__).resolve().parent + for parent in here.parents: + if binary is None and (parent / "bin" / "circuitpython").is_file(): + binary = str(parent / "bin" / "circuitpython") + frozen = parent / "circuitpython" / "frozen" + if libs is None and (frozen / "Adafruit_CircuitPython_asyncio" / "asyncio").is_dir(): + libs = frozen + if binary is None: + return None, "no circuitpython binary" + if libs is None: + return None, "no Adafruit asyncio library" + return (binary, libs), None + + +class OnCircuitPython(unittest.TestCase): + """bledev's contract and codec checks, run by CircuitPython's own + interpreter (the unix port, with Adafruit's asyncio). The file-transfer + server and pairing checks are MicroPython's and aren't run here.""" + + def test_contract_and_codecs(self): + import os + import shutil + import subprocess + import tempfile + from pathlib import Path + + found, why = _circuitpython() + if found is None: + message = why + ": bledev's checks did NOT run on CircuitPython" + if os.environ.get("PYDEVICES_REQUIRE_CIRCUITPYTHON") == "1": + self.fail(message) + sys.stderr.write("\n*** " + message + " ***\n") + self.skipTest(message) + binary, libs = found + here = Path(__file__).resolve().parent + if isinstance(libs, Path): # the checkout's frozen/: gather the two libraries + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + shutil.copytree(str(libs / "Adafruit_CircuitPython_asyncio" / "asyncio"), os.path.join(scratch.name, "asyncio")) + shutil.copy(str(libs / "Adafruit_CircuitPython_Ticks" / "adafruit_ticks.py"), scratch.name) + libs = scratch.name + env = dict(os.environ, MICROPYPATH=str(here.parent / "lib") + ":" + libs) + for script, tail in ( + ("bledev_contract.py", " 0 failed (circuitpython)"), + ("bledev_midi_codec.py", " 0 failed"), + ("bledev_hid_checks.py", " 0 failed (circuitpython)"), + ): + with self.subTest(script=script): + result = subprocess.run([binary, str(here / script)], capture_output=True, text=True, timeout=600, env=env) + self.assertIn(tail, result.stdout, result.stdout[-2000:] + result.stderr[-2000:]) + self.assertIn("circuitpython", result.stdout) + + class Planted(unittest.TestCase): """cpble's send reverted to a plain PacketBuffer.write(data): the checks above must fail.""" From ee0c06d4879449ba6ff38cf657521734e1a46119 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Thu, 24 Sep 2026 23:19:57 -0500 Subject: [PATCH 3/5] cpfiles gate: plant in the read path, unpair on failure; docs: cpble, the CircuitPython PacketBuffer draft --- docs/bledev-internals.md | 74 +++++- docs/bledev.md | 38 +++- .../cp-packetbuffer-notify-stall.md | 84 +++++++ .../cp-packetbuffer-notify-stall.patch | 215 ++++++++++++++++++ tests/bledev_board/cpfiles_client.py | 20 +- 5 files changed, 424 insertions(+), 7 deletions(-) create mode 100644 docs/upstream-reports/cp-packetbuffer-notify-stall.md create mode 100644 docs/upstream-reports/cp-packetbuffer-notify-stall.patch diff --git a/docs/bledev-internals.md b/docs/bledev-internals.md index 2f634d26..ed3c3860 100644 --- a/docs/bledev-internals.md +++ b/docs/bledev-internals.md @@ -1,7 +1,7 @@ # bledev internals: writing a backend This page is for you if you're adding a backend (`bledev.bleak`, -`bledev.webble`, later `bledev.cpble`) or changing one. What an app can rely +`bledev.webble`, `bledev.cpble`) or changing one. What an app can rely on is in [bledev.md](bledev.md#the-rules-every-backend-keeps); this is how a backend delivers it. @@ -16,6 +16,7 @@ standalone module per backend, and an optional selector. | `fake.py` | In-process loopback. The reference implementation of every hook. | | `mpble.py` | MicroPython, over aioble. | | `bleak.py` | CPython on Windows, Linux and macOS, over bleak. Central only. | +| `cpble.py` | CircuitPython, over `_bleio`. Both roles. | | `nus.py` | Nordic UART byte stream, built only on the contract. | | `repl.py` | The REPL over nus: MicroPython's raw `bluetooth` API from the IRQ on the board, `nus` on the client. | | `improv.py` | Improv Wi-Fi setup, both sides, built only on the contract. | @@ -227,6 +228,77 @@ bounds the setup at 4 s and tries again, up to three times. `nus.connect()` and `midi.connect()` use it, and the contract checks both the retry and the giving up. +## How cpble gets there + +`_bleio` has no events and no async calls. An adapter hands out objects you +look at: `adapter.connections`, `connection.connected`, and the ring buffers +that a characteristic's writes and notifications land in. So every wait in +cpble is a poll of one of those every `POLL_MS` (5 ms), and each blocking +`_bleio` call (connect, discovery, a read, a write with response, making a +subscription) holds the event loop until the radio answers, which is one or +two connection intervals. A scan runs in slices of `SCAN_SLICE_MS` (100 ms), +each a fresh `start_scan()`, with the scan response merged per address the +way aioble does. + +**Where writes land.** A characteristic with notify or indicate gets a +`_bleio.PacketBuffer`, which keeps each write whole but takes writes only from +the central that subscribed, and only after it did (it learns the connection +from the subscribe event). Any other writable characteristic gets a +`_bleio.CharacteristicBuffer`, a byte stream: with `capture=True`, each +`written()` returns what arrived since the last one. Both drop the oldest data +when full and don't say so; cpble checks the stream buffer's fill and raises +when it's full, and sizes it at 32 KB (`stream_buffer`), which is nus's whole +16 KB echo phase in flight twice over. `_bleio` doesn't say which central +wrote, so `written()` names the most recent live one. Every server value +buffer is `_bleio`'s largest (512 bytes), because a notification can't be +longer than the value's buffer; `max_len` is kept in Python (`read()` gives +the first `max_len` bytes) rather than by `_bleio` refusing the write. + +**Notifications, and a stall in CircuitPython 10.3.** `PacketBuffer.write()` +on the ESP32 keeps a packet it couldn't get an mbuf for and retries it on the +next `BLE_GAP_EVENT_NOTIFY_TX`. NimBLE sends that event when a notification is +handed to the stack, not when it leaves the radio, so a packet that never got +an mbuf never produces one, and the next write that doesn't fit beside it +waits inside `_bleio` until the central disconnects. Writing 253-byte packets +back to back from the laptop gate delivered 10 (2,530 bytes) and stopped, +every time. So cpble passes each notification as the `header=` of an empty +write: `_bleio` copies a header only into an empty packet, and a write of +nothing never waits, but it does retry what's pending. A return of 0 means +the previous packet is still there, and `notify()` raises `BusyError`; nus, +midi and the other async senders wait and try again. A task retries the last +packet every 5 ms for 5 s after it was sent, since nothing else will. It +also keeps notifications whole, where `PacketBuffer` would append the next +write to a pending packet. + +**And a hard fault.** When NimBLE refuses a notification after the mbuf was +allocated (it needs one more for the ATT header), `PacketBuffer` "undoes" by +switching to a second outgoing buffer that it allocates only for indications +and writes, and the next write copies into a null pointer: the board restarts +in safe mode. cpble can make it rarer but not impossible: notifications are +held to 244 bytes (the reported MTU is at most 247, one link-layer packet, so +NimBLE never fragments), and each new connection gets fresh packet buffers. +The fix is in CircuitPython; the draft for upstream, with the patch, is +[upstream-reports/cp-packetbuffer-notify-stall.md](upstream-reports/cp-packetbuffer-notify-stall.md). + +**Discovery mode, without a hand.** CircuitPython's own file service +advertises to a new host only in discovery mode, which normally takes a reset +pressed during the blue blink after boot. The supervisor marks that second in +an RTC register that survives a reset, and `memorymap` can write that register +on the S3 (`RTC_CNTL_STORE0_REG`, 0x60008050), so +`tests/bledev_board/cpfiles_discovery.py` writes the mark and resets. It also +erases the board's bonds, as the button would. + +**What `_bleio` can't do here**, on the ESP32 port: a central can't receive +indications (they're ignored below Python); pairing is "just works" only +(`authenticated=True` raises); there are no long reads; the MTU is fixed at +256 when CircuitPython is built; a service has at most 10 characteristics and +a characteristic at most 2 descriptors; a peripheral doesn't learn its +central's address; `Service.deinit()` is the only way to take a service away, +which `register_services()` uses; and a read that times out inside `_bleio` +returns a full buffer as if it succeeded. A refused read or write comes back as +`BluetoothError("Unknown system firmware error: 261")`, NimBLE's 0x100 plus +the ATT code, and cpble turns that into `GattError(5)`. + ## How the REPL gets there The REPL has to work when nothing runs asyncio, at the `>>>` prompt, so the diff --git a/docs/bledev.md b/docs/bledev.md index 8cd97e2a..472f554c 100644 --- a/docs/bledev.md +++ b/docs/bledev.md @@ -31,11 +31,12 @@ transport can talk to it too. | Backend | Host | Central (scan, connect) | Peripheral (advertise, serve) | |---|---|---|---| | `bledev.mpble` | MicroPython with `bluetooth` (ESP32, S3, P4 + C6, Pico W) | yes | yes | +| `bledev.cpble` | CircuitPython with `_bleio` (ESP32-S3 measured) | yes | yes | | `bledev.bleak` | CPython on Windows, Linux, macOS | yes | no | | `bledev.webble` | Browsers: PyScript, the Workbench simulator | yes | no | | `bledev.fake` | Anywhere: tests | yes | yes | -`bledev.webble` is being written; the other three are here. +`bledev.webble` is being written; the other four are here. Roles aren't symmetric. A laptop or a browser can only scan and connect, so when a board talks to one, the board advertises. Only board-to-board links put @@ -86,11 +87,18 @@ import bledev.bleak ble = bledev.bleak.BleakBLE() # central only ``` +On CircuitPython (it needs the `asyncio` library: `circup install asyncio`): + +```python +import bledev.cpble +ble = bledev.cpble.get() # CircuitPython's _bleio.adapter, both roles +``` + Anywhere, if you don't want to name the backend: ```python import bledev.auto -ble = bledev.auto.adapter() # mpble, webble or bleak, whichever this host has +ble = bledev.auto.adapter() # mpble, cpble, webble or bleak, whichever this host has ``` `bledev.auto` is optional, and nothing else in bledev imports it. Set @@ -438,6 +446,32 @@ USB host can use it the same way. How it numbers axes, what it does with a rollover, and what a board central can't read are in [bledev-internals.md](bledev-internals.md#how-hid-gets-there). +## On CircuitPython + +The same code runs on a CircuitPython board with `bledev.cpble`, in either +role: nus, MIDI, the contract's checks and the file client all passed over the +radio there (the numbers are [below](#measured-on-circuitpython)). A few +things differ, because `_bleio` works differently from aioble underneath: + +- Connecting, discovery, a read, a write with response and subscribing each + hold up your other tasks until the radio answers, usually a connection + interval or two. `_bleio` has no async calls. +- It pairs "just works" only. `authenticated=True` and passkeys raise + `UnsupportedError`, and it always bonds. +- A central there can't receive indications, and the MTU is 247 at most. +- `Characteristic.write()` on a characteristic that notifies sends the value + to subscribed centrals even without `send_update`. +- CircuitPython 10.3's `PacketBuffer` can stall or fault the board when a + notification stream outruns the radio. bledev steps around the stall; the + fault needs a CircuitPython fix, which is drafted for upstream. Until it + lands, a board that streams notifications fast can restart in safe mode. + +CircuitPython's own BLE file service, the one its supervisor serves, is +reached with the same `bledev.filetransfer.connect()` a MicroPython board +uses; it pairs by itself. The service is only public in CircuitPython's +discovery mode. What cpble does about each of these, and why, is in +[bledev-internals.md](bledev-internals.md#how-cpble-gets-there). + ## The rules every backend keeps These hold on every backend. The fake enforces the strict version of each, so diff --git a/docs/upstream-reports/cp-packetbuffer-notify-stall.md b/docs/upstream-reports/cp-packetbuffer-notify-stall.md new file mode 100644 index 00000000..75378b1c --- /dev/null +++ b/docs/upstream-reports/cp-packetbuffer-notify-stall.md @@ -0,0 +1,84 @@ +# Draft: `_bleio.PacketBuffer` on the ESP32 stalls when NimBLE's buffers run dry, and can hard-fault after a refused notification + +**Target:** adafruit/circuitpython, an issue with a PR (the fix is small and +local to `ports/espressif/common-hal/_bleio/PacketBuffer.c`). +**Status:** ready to file, awaiting Brad's word. Post only what is below the +`---`. The patch is `cp-packetbuffer-notify-stall.patch` beside this file, +against tag 10.3.0. The upstream-emissary agent can turn it into a branch +and a PR body; nothing has been pushed. + +Found while building `bledev.cpble` (pydevices, `lib/bledev/cpble.py`), which +works around the stall on unpatched firmware (it writes each notification as +the `header=` of an empty write, which never waits). The workaround doesn't +avoid the hard fault; only the fix does. The measurements are in +`docs/bledev-internals.md`, "How cpble gets there". + +Verified on a LilyGO T-Embed (ESP32-S3), CircuitPython 10.3.0, the official +build and the same tag with this patch, against Windows 11 through bleak +3.0.2, 2026-09-24. + +--- + +### `_bleio.PacketBuffer` on the ESP32: notifications stall for good when NimBLE's mbufs run out, and a refused notification can hard-fault the next write + +**CircuitPython 10.3.0, ESP32-S3 (espressif port, NimBLE).** Two faults in +`ports/espressif/common-hal/_bleio/PacketBuffer.c`, both reachable from the +BLE file-transfer service without any user code. + +**1. A stall.** `queue_next_write()` keeps the pending packet when +`ble_hs_mbuf_from_flat()` returns `NULL`, expecting the next +`BLE_GAP_EVENT_NOTIFY_TX` to retry it. For a notification that event fires +synchronously inside `ble_gatts_notify_custom()`, when the packet is handed +to the stack (`ble_gattc.c`, `ble_gap_notify_tx_event(rc, conn_handle, +chr_val_handle, 0)`), not when it leaves the radio. A packet that never got an +mbuf never produces one, so nothing retries it. The next +`common_hal_bleio_packet_buffer_write()` that doesn't fit beside it then waits +in its `while (self->pending_size != 0 ...)` loop until the central +disconnects. + +Seen three ways: + +- A user `PacketBuffer` on a notify characteristic: 64 writes of 253 bytes + back to back delivered 10 notifications (2,530 bytes), then nothing, every + time, until the central gave up. +- Half-packet writes with an empty `write(b"")` before each (which does retry + the pending packet): 3,780 and 4,662 bytes, then the same stall. +- **The BLE file-transfer service**, paired, listing `/` on a board with a + few files: the listing stops part way through the fourth entry (18 + notifications, because `_process_listdir` writes the path in small pieces), + and every later command times out until the host disconnects. + +**2. A hard fault.** When `ble_gatts_notify_custom()` itself fails (it +allocates another mbuf for the ATT header, `ble_att_cmd_get()`, so it can fail +with `BLE_HS_ENOMEM` right after `ble_hs_mbuf_from_flat()` succeeded), the +"undo" path flips `pending_index`. Notifications only ever allocate +`outgoing[0]` (`outgoing2` is allocated only for `CHAR_PROP_WRITE` and +`CHAR_PROP_INDICATE`), so the next write `memcpy`s into `outgoing[1] == +NULL`. The same path also sets `pending_size` back and then zeroes it three +lines later, so the refused packet is lost as well. On the T-Embed this +restarted the board in safe mode (`SafeModeReason.HARD_FAULT`) in 2 of 11 +16 KB notify-heavy runs. + +**The fix** (patch attached, `PacketBuffer.c` and `PacketBuffer.h`): + +- In `queue_next_write()`, on any failure, keep the packet pending (don't + zero `pending_size`), and flip `pending_index` back only for indications, + which are the only kind that flipped it. +- Retry a kept packet from a background callback, rescheduled while it keeps + failing, since no BLE event will say the buffers are free. Without this, the + last packet of a response sits pending until the next command's reply + pushes it out, so the host sees each answer one command late. Also retry in + `common_hal_bleio_packet_buffer_write()`'s wait loop and in + `common_hal_bleio_packet_buffer_flush()`. +- Drop a pending packet when its central unsubscribes or disconnects, so a + kept packet can't reach the next central. + +With it (same board, same tests): RESULTS_PATCHED + +Not addressed here, noticed on the way: `bleio_packet_buffer_extend()` drops +the oldest packets when its ring is full, with a `// set an overflow flag?` +comment; `bleio_gattc_read()` reports a timeout as success with the whole +buffer's length; and ATT errors on reads and writes reach Python as +`BluetoothError("Unknown system firmware error: 261")` rather than +`SecurityError`, because `_wait_for_completion()` is checked with +`CHECK_NIMBLE_ERROR` instead of `CHECK_BLE_ERROR`. diff --git a/docs/upstream-reports/cp-packetbuffer-notify-stall.patch b/docs/upstream-reports/cp-packetbuffer-notify-stall.patch new file mode 100644 index 00000000..3ffee676 --- /dev/null +++ b/docs/upstream-reports/cp-packetbuffer-notify-stall.patch @@ -0,0 +1,215 @@ +diff --git a/ports/espressif/common-hal/_bleio/PacketBuffer.c b/ports/espressif/common-hal/_bleio/PacketBuffer.c +index a14c627f11..08337fc07a 100644 +--- a/ports/espressif/common-hal/_bleio/PacketBuffer.c ++++ b/ports/espressif/common-hal/_bleio/PacketBuffer.c +@@ -60,6 +60,23 @@ void bleio_packet_buffer_extend(bleio_packet_buffer_obj_t *self, uint16_t conn_h + static int packet_buffer_on_ble_client_evt(struct ble_gap_event *event, void *param); + static int queue_next_write(bleio_packet_buffer_obj_t *self); + ++// Runs on the VM task, from background callbacks. ++static void _retry_pending(void *self_in) { ++ bleio_packet_buffer_obj_t *self = (bleio_packet_buffer_obj_t *)self_in; ++ if (common_hal_bleio_packet_buffer_deinited(self) || ++ self->conn_handle == BLEIO_HANDLE_INVALID || ++ self->packet_queued) { ++ return; ++ } ++ queue_next_write(self); ++} ++ ++// NimBLE couldn't take the pending packet. It reports buffers freeing up with no event, so ++// try again from the next background pass, and every pass after while it keeps failing. ++static void _schedule_retry(bleio_packet_buffer_obj_t *self) { ++ background_callback_add(&self->retry_callback, _retry_pending, self); ++} ++ + static int _write_cb(uint16_t conn_handle, + const struct ble_gatt_error *error, + struct ble_gatt_attr *attr, +@@ -81,61 +98,78 @@ static int queue_next_write(bleio_packet_buffer_obj_t *self) { + // transmission (when packet_queued is true) and the other is `pending` and can still be + // modified. By primarily appending to the `pending` buffer we can reduce the protocol overhead + // of the lower level link and ATT layers. ++ // ++ // If NimBLE can't take the packet (no mbufs, most often), the packet stays pending, and ++ // write() and flush() try again while they wait. Nothing else would: a notification that ++ // was never sent produces no BLE_GAP_EVENT_NOTIFY_TX to retry from. + self->packet_queued = false; +- if (self->pending_size > 0) { +- uint16_t conn_handle = self->conn_handle; +- int err_code = NIMBLE_OK; +- if (self->client) { +- if (self->write_type == CHAR_PROP_WRITE_NO_RESPONSE) { +- err_code = ble_gattc_write_no_rsp_flat(conn_handle, +- self->characteristic->handle, +- self->outgoing[self->pending_index], +- self->pending_size); +- // We don't set packet_queued because NimBLE will buffer our +- // outgoing packets. +- } else { +- err_code = ble_gattc_write_flat(conn_handle, +- self->characteristic->handle, +- self->outgoing[self->pending_index], +- self->pending_size, +- _write_cb, self); +- self->pending_index = (self->pending_index + 1) % 2; +- self->packet_queued = true; +- } ++ if (self->pending_size == 0) { ++ return NIMBLE_OK; ++ } ++ uint16_t conn_handle = self->conn_handle; ++ int err_code = NIMBLE_OK; ++ if (self->client) { ++ if (self->write_type == CHAR_PROP_WRITE_NO_RESPONSE) { ++ err_code = ble_gattc_write_no_rsp_flat(conn_handle, ++ self->characteristic->handle, ++ self->outgoing[self->pending_index], ++ self->pending_size); ++ // We don't set packet_queued because NimBLE will buffer our ++ // outgoing packets. + } else { +- // Allocate an mbuf because the functions below consume it. +- struct os_mbuf *om = ble_hs_mbuf_from_flat(self->outgoing[self->pending_index], self->pending_size); +- if (om == NULL) { +- // We may not have any more mbufs if BLE busy. It isn't a problem (yet) so we'll +- // just skip queueing for now. +- return BLE_HS_ENOMEM; +- } +- size_t pending_size = self->pending_size; +- self->pending_size = 0; +- if (self->write_type == CHAR_PROP_NOTIFY) { +- err_code = ble_gatts_notify_custom(conn_handle, self->characteristic->handle, om); +- } else if (self->write_type == CHAR_PROP_INDICATE) { +- err_code = ble_gatts_indicate_custom(conn_handle, self->characteristic->handle, om); ++ err_code = ble_gattc_write_flat(conn_handle, ++ self->characteristic->handle, ++ self->outgoing[self->pending_index], ++ self->pending_size, ++ _write_cb, self); ++ if (err_code == NIMBLE_OK) { + self->pending_index = (self->pending_index + 1) % 2; + self->packet_queued = true; +- } else { +- // Placeholder error. +- err_code = BLE_HS_EUNKNOWN; +- } +- // Undo our queueing if it fails. We need to do it early because we may recurse back +- // to here from the above ble_gatts functions. +- if (err_code != NIMBLE_OK) { +- self->pending_index = (self->pending_index + 1) % 2; +- self->packet_queued = false; +- self->pending_size = pending_size; + } + } +- self->pending_size = 0; + if (err_code != NIMBLE_OK) { +- // On error, simply skip updating the pending buffers so that the next HVC or WRITE +- // complete event triggers another attempt. ++ // Keep the packet pending for the next attempt. ++ _schedule_retry(self); + return err_code; + } ++ self->pending_size = 0; ++ return NIMBLE_OK; ++ } ++ ++ // Allocate an mbuf because the functions below consume it. ++ struct os_mbuf *om = ble_hs_mbuf_from_flat(self->outgoing[self->pending_index], self->pending_size); ++ if (om == NULL) { ++ // No mbufs while BLE is busy. Keep the packet pending for the next attempt. ++ _schedule_retry(self); ++ return BLE_HS_ENOMEM; ++ } ++ // Clear pending_size before sending, because the ble_gatts functions below may recurse back ++ // here through the NOTIFY_TX event. ++ size_t pending_size = self->pending_size; ++ self->pending_size = 0; ++ if (self->write_type == CHAR_PROP_NOTIFY) { ++ // Only one outgoing buffer is allocated for notifications (see construct), so the ++ // pending index doesn't change. ++ err_code = ble_gatts_notify_custom(conn_handle, self->characteristic->handle, om); ++ } else if (self->write_type == CHAR_PROP_INDICATE) { ++ self->pending_index = (self->pending_index + 1) % 2; ++ self->packet_queued = true; ++ err_code = ble_gatts_indicate_custom(conn_handle, self->characteristic->handle, om); ++ if (err_code != NIMBLE_OK) { ++ self->pending_index = (self->pending_index + 1) % 2; ++ self->packet_queued = false; ++ } ++ } else { ++ // Placeholder error. ++ os_mbuf_free_chain(om); ++ err_code = BLE_HS_EUNKNOWN; ++ } ++ if (err_code != NIMBLE_OK) { ++ // NimBLE freed the mbuf. The data is still in our buffer: keep it pending for the next ++ // attempt rather than dropping it. ++ self->pending_size = pending_size; ++ _schedule_retry(self); ++ return err_code; + } + return NIMBLE_OK; + } +@@ -145,6 +179,8 @@ static int packet_buffer_on_ble_client_evt(struct ble_gap_event *event, void *pa + bleio_packet_buffer_obj_t *self = (bleio_packet_buffer_obj_t *)param; + if (event->type == BLE_GAP_EVENT_DISCONNECT && self->conn_handle == event->disconnect.conn.conn_handle) { + self->conn_handle = BLEIO_HANDLE_INVALID; ++ // A packet still pending was for that link; don't hand it to the next one. ++ self->pending_size = 0; + return false; + } + if (event->type == BLE_GAP_EVENT_SUBSCRIBE) { +@@ -152,6 +188,10 @@ static int packet_buffer_on_ble_client_evt(struct ble_gap_event *event, void *pa + self->conn_handle = event->subscribe.conn_handle; + } else if (self->conn_handle == event->subscribe.conn_handle && event->subscribe.cur_notify == 0 && event->subscribe.cur_indicate == 0) { + self->conn_handle = BLEIO_HANDLE_INVALID; ++ if (!self->client) { ++ // As above: whatever is pending was for the central that left. ++ self->pending_size = 0; ++ } + } + return false; + } +@@ -328,6 +368,10 @@ mp_int_t common_hal_bleio_packet_buffer_write(bleio_packet_buffer_obj_t *self, c + while (self->pending_size != 0 && + self->conn_handle != BLEIO_HANDLE_INVALID && + !mp_hal_is_interrupted()) { ++ if (!self->packet_queued) { ++ // The pending packet is waiting for NimBLE to have room; nothing else retries it. ++ queue_next_write(self); ++ } + RUN_BACKGROUND_TASKS; + } + } +@@ -443,6 +487,9 @@ void common_hal_bleio_packet_buffer_flush(bleio_packet_buffer_obj_t *self) { + self->packet_queued) && + self->conn_handle != BLEIO_HANDLE_INVALID && + !mp_hal_is_interrupted()) { ++ if (!self->packet_queued) { ++ queue_next_write(self); ++ } + RUN_BACKGROUND_TASKS; + } + } +diff --git a/ports/espressif/common-hal/_bleio/PacketBuffer.h b/ports/espressif/common-hal/_bleio/PacketBuffer.h +index 0ab084c5b0..278f9fbe06 100644 +--- a/ports/espressif/common-hal/_bleio/PacketBuffer.h ++++ b/ports/espressif/common-hal/_bleio/PacketBuffer.h +@@ -9,6 +9,7 @@ + #include "py/ringbuf.h" + #include "shared-bindings/_bleio/Characteristic.h" + #include "common-hal/_bleio/ble_events.h" ++#include "supervisor/background_callback.h" + + typedef struct { + mp_obj_base_t base; +@@ -27,6 +28,8 @@ typedef struct { + uint8_t write_type; + bool client; + bool packet_queued; ++ // Retries a pending packet that NimBLE had no room for. No BLE event will. ++ background_callback_t retry_callback; + } bleio_packet_buffer_obj_t; + + typedef ble_event_handler_entry_t ble_event_handler_t; diff --git a/tests/bledev_board/cpfiles_client.py b/tests/bledev_board/cpfiles_client.py index 8d2928cd..2e22787e 100644 --- a/tests/bledev_board/cpfiles_client.py +++ b/tests/bledev_board/cpfiles_client.py @@ -21,7 +21,7 @@ offset, mkdir, listdir, move, delete; then a reconnect from the bond, with no pairing. Then it unpairs, unless ``--keep``, so Windows' Bluetooth list ends as it started. ``--plant`` flips a bit in -one packet the client receives, and the run must FAIL. Ends with +one packet of the file data it reads back first, and the run must FAIL. Ends with ``RESULT PASS`` or ``RESULT FAIL``. """ import asyncio @@ -41,6 +41,7 @@ RUNS = int(ARGS[i + 1]) failures = [] +FOUND = [] # the board's address, to unpair even when a check throws def check(ok, what, detail=""): @@ -65,6 +66,7 @@ def pattern(n, seed): async def main(): ble = bledev.bleak.BleakBLE() device = await ble.find(service=ft.SERVICE, timeout_ms=20000) + FOUND.append(device.address) print("found", device) try: @@ -79,14 +81,16 @@ async def main(): files = await ft.connect(ble, device=device, timeout_ms=15000) check(files.connection.encrypted, "the client paired by itself", "{:.0f} ms to connect and pair".format(ticks() - t0)) print("connected, mtu", files.connection.mtu, "protocol version", files.version) + state = {"armed": False, "n": 0} if PLANT: feed = files._feed - state = {"n": 0} def planted(data): - if len(data) > 28: + # The fifth packet to arrive while the first read runs: file data + # (CircuitPython sends each READ_DATA header in a packet of its own). + if state["armed"]: state["n"] += 1 - if state["n"] == 3: + if state["n"] == 5: data = bytes(data[:-1]) + bytes((data[-1] ^ 0x10,)) feed(data) @@ -104,7 +108,9 @@ def planted(data): t0 = ticks() await files.write("/ftgate/big.bin", data) t1 = ticks() + state["armed"] = run == 0 back = await files.read("/ftgate/big.bin") + state["armed"] = False t2 = ticks() print("run {}: 20 KB up {:.0f} ms ({:.1f} KB/s), down {:.0f} ms ({:.1f} KB/s)".format( run + 1, t1 - t0, len(data) / (t1 - t0), t2 - t1, len(back) / max(1, t2 - t1))) @@ -146,6 +152,12 @@ async def guarded(): try: await main() except Exception as e: + if not KEEP and FOUND: + try: # leave Windows' Bluetooth list as it was, even on a failure + await bledev.bleak.unpair(FOUND[0]) + print("unpaired", FOUND[0]) + except Exception: + pass failures.append("exception") print("EXCEPTION", type(e).__name__, e) import traceback From a89e21bd46f3d42dfa4c12cdc7947cb34f35ef60 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Fri, 25 Sep 2026 00:16:24 -0500 Subject: [PATCH 4/5] cpble: discover one service when asked, 30 ms interval by default, slow-setup hint, sticky scan names, order across characteristics; board scripts and docs --- lib/bledev/__init__.py | 4 ++ lib/bledev/cpble.py | 69 ++++++++++++++++++++++++------ tests/bledev_board/gatt_central.py | 10 ++++- 3 files changed, 68 insertions(+), 15 deletions(-) diff --git a/lib/bledev/__init__.py b/lib/bledev/__init__.py index 5bc3aef3..4ee533fe 100644 --- a/lib/bledev/__init__.py +++ b/lib/bledev/__init__.py @@ -1023,6 +1023,10 @@ async def connect_and_set_up( """ pair = options.pop("pair", False) passkey = options.pop("passkey", None) + # A backend whose GATT calls are slow by nature says how long setup may + # take at least (cpble: CircuitPython waits 2 s on every write with a + # response, subscribing included). + setup_timeout_ms = max(setup_timeout_ms, getattr(ble, "min_setup_ms", 0)) error = None stale = False for _ in range(attempts): diff --git a/lib/bledev/cpble.py b/lib/bledev/cpble.py index f1ef0f0f..c52e5cc1 100644 --- a/lib/bledev/cpble.py +++ b/lib/bledev/cpble.py @@ -251,6 +251,13 @@ class CPBLE(BLE): backend = "cpble" + #: How long a profile's setup after connecting may take here (see + #: ``bledev.connect_and_set_up``). CircuitPython's ESP32 port waits out + #: its whole 2 s timeout on every read and every write with response, + #: because it waits for a status to leave 0 and success is 0; making a + #: subscription is such a write. + min_setup_ms = 12000 + def __init__(self, adapter=None, *, queue_limit=64, stream_buffer=32768): self.raw = adapter if adapter is not None else _bleio.adapter if self.raw is None: @@ -482,12 +489,13 @@ async def _connect(self, device, timeout_ms, **options): if native is None or not native.connected: raise BLETimeoutError("connect timed out") conn = self._wrap(native, "central", device) - interval_us = options.get("max_conn_interval_us") or options.get("min_conn_interval_us") - if interval_us: - try: - native.connection_interval = interval_us / 1000 - except Exception: - pass # the peripheral may refuse; the link keeps its interval + # _bleio asks for 15 to 300 ms and may get the slow end; ask for what + # MicroPython's NimBLE asks by default (30 ms) unless told otherwise. + interval_us = options.get("max_conn_interval_us") or options.get("min_conn_interval_us") or 30000 + try: + native.connection_interval = interval_us / 1000 + except Exception: + pass # the peripheral may refuse; the link keeps its interval return conn @@ -539,13 +547,21 @@ def _entry(self, entry): key = bytes(entry.address.address_bytes) seen = self._seen.get(key) if seen is None: - seen = self._seen[key] = [b"", b"", True] + seen = self._seen[key] = [b"", b"", True, None] if entry.scan_response: seen[1] = raw else: seen[0] = raw seen[2] = entry.connectable name, services, appearance, manufacturer = decode_advertisement(seen[0], seen[1]) + # A name seen once stays (a host may put it in only one of the two + # packets), and a UUID in both packets is listed once. + name = seen[3] = name or seen[3] + unique = [] + for uuid in services: + if uuid not in unique: + unique.append(uuid) + services = unique device = Device( self._ble, bytes(reversed(key)), entry.address.type, name=name, native=entry.address ) @@ -578,6 +594,7 @@ def __init__(self, ble, device, role, native): #: The ``_bleio.Connection``. self.native = native self._remote = None + self._by_uuid = {} self._last_mtu = DEFAULT_MTU @property @@ -645,17 +662,29 @@ async def pair(self, bond=True, timeout_ms=30000, passkey=None): async def unpair(self): raise UnsupportedError("CircuitPython forgets one peer only by erasing every bond: ble.forget_bonds()") - def _discover(self): - if self._remote is None: - if not self.is_connected(): - raise DisconnectedError("service discovery: not connected") - with _Call("service discovery"): - self._remote = [_CPClientService(self, s) for s in self.native.discover_remote_services()] + def _discover(self, uuid=None): + # _bleio discovers characteristics and descriptors too, one round trip + # each, so asking for one service is far quicker than asking for all + # (7 s against a MicroPython nus server at CircuitPython's default + # interval). Services found either way are kept. + if self._remote is not None: + return self._remote + if not self.is_connected(): + raise DisconnectedError("service discovery: not connected") + if uuid is not None: + found = self._by_uuid.get(uuid) + if found is None: + with _Call("service discovery"): + found = [_CPClientService(self, s) for s in self.native.discover_remote_services((_cuuid(uuid),))] + self._by_uuid[uuid] = found + return found + with _Call("service discovery"): + self._remote = [_CPClientService(self, s) for s in self.native.discover_remote_services()] return self._remote async def _discover_services(self, uuid, timeout_ms): await sleep_ms(0) - return [s for s in self._discover() if uuid is None or s.uuid == uuid] + return [s for s in self._discover(uuid) if uuid is None or s.uuid == uuid] class _CPClientService(ClientService): @@ -988,6 +1017,18 @@ async def written(self, timeout_ms): if self._char.capture: return connection, data self._queue = [] + # Every buffer is polled on its own, so a task reading + # another characteristic may not have seen writes that came + # before this one. Move them into their queues now, and give + # those tasks a poll to take them first, which keeps the + # central's order across characteristics (the contract check + # that counts 100 captured writes and then asks: 96 of 100, + # once in three, before this). + for service in self._ble._services: + for c in service.characteristics: + if c._impl is not None and c._impl is not self: + c._impl._drain() + await sleep_ms(POLL_MS * 2) return connection if timeout_ms is not None and _elapsed(start) >= timeout_ms: raise BLETimeoutError("written timed out after {} ms".format(timeout_ms)) diff --git a/tests/bledev_board/gatt_central.py b/tests/bledev_board/gatt_central.py index 257db473..e3651690 100644 --- a/tests/bledev_board/gatt_central.py +++ b/tests/bledev_board/gatt_central.py @@ -153,7 +153,15 @@ async def main(): await stream.write(struct.pack(" Date: Fri, 25 Sep 2026 00:25:55 -0500 Subject: [PATCH 5/5] docs: cpble measurements, the file service's conditions, the 2 s GATT wait --- docs/bledev-internals.md | 53 +++++++++++++++++++ docs/bledev.md | 13 +++-- .../cp-packetbuffer-notify-stall.md | 9 +++- 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/docs/bledev-internals.md b/docs/bledev-internals.md index ed3c3860..7af66e9d 100644 --- a/docs/bledev-internals.md +++ b/docs/bledev-internals.md @@ -761,6 +761,59 @@ KB/s), overwriting the same file each time. The planted bit flip on the client failed the round-trip check. mpftp's `ble://` transport, and its gate, are in [PyDevices/mpftp](https://github.com/PyDevices/mpftp) `docs/plans/ble.md`. +### CircuitPython (cpble) + +The LilyGO T-Embed (ESP32-S3) on CircuitPython 10.3.0, bledev as `.py` +source, against Windows 11 and bleak 3.0.2 one desk apart, 2026-09-24. +"Official" is circuitpython.org's build; "patched" is the same tag with +[upstream-reports/cp-packetbuffer-notify-stall.patch](upstream-reports/cp-packetbuffer-notify-stall.patch), +built here because the official build can't carry a notification stream +reliably (below). + +| Gate | Official 10.3.0 | Patched | +|---|---|---| +| nus, 16 KB up, down and echoed (`nus_server.py` on the board) | 9 of 11 passed; 2 restarted the board in safe mode (hard fault) | 12 of 12 | +| nus throughput, Windows' defaults | 50-90 KB/s up, 31-49 down, 21-37 echo | 50-91 up, 25-49 down, 16-28 echo | +| nus, `--fast` (throughput parameters) | | 88 up, 44 down, 34 echo | +| the GATT contract, laptop central (`gatt_central.py --mtu 256`) | pass | 5 of 5 | +| BLE-MIDI, 1,000 messages each way | pass (0.8 s up, 0.7 s down) | pass | +| CircuitPython's own file service, 20 KB up and down (`cpfiles_client.py`) | 0 of 2: the download stalls | 7 of 7 round trips in 3 sessions | +| MicroPython LCD-7 central, CircuitPython serving nus | | pass: 90 up, 37 down, 25 echo | + +Every planted fault failed its gate: a bit flipped in the nus stream (either +side), notification 7 skipped by the contract's peripheral, the three BLE-MIDI +plants (a flip going up, a dropped and a flipped packet going down), and a bit +flipped in the file data read back. + +**BLE-MIDI latency** against the T-Embed, half a note-on's round trip, 400 +notes: a median of 52.1 ms (p99 103.2) at Windows' defaults and 13.3 ms (p99 +23.4) with throughput parameters, the same as against a MicroPython S3 +(52.3 and 13.4). Polling every 5 ms costs nothing you can see there. + +**CircuitPython's file service.** The client paired by itself in 2.1-3.2 s, +reconnected from the bond with no pairing in 1.0-1.7 s, and moved 20 KB up in +4.6-10.1 s (2.0-4.4 KB/s: CircuitPython writes flash between 512-byte +windows) and down in 0.8-1.8 s (11-26 KB/s). Two conditions, both +CircuitPython's design: the service writes nothing while a computer has the +CIRCUITPY drive mounted (USB takes the filesystem's lock as soon as the host +asks whether it may write; `storage.disable_usb_drive()` in `boot.py` for the +gate), and a command that arrives while a write's automatic reload is +restarting `code.py` isn't answered until another command arrives +(`supervisor.runtime.autoreload = False` in `code.py` for the gate). mkdir +there makes one level, where bledev's server makes the parents too. + +**CircuitPython as central**, against the LCD-7 serving nus: found, connected, +discovered, subscribed and wrote, with every write with a response (making the +subscription is one) taking 2 s, because CircuitPython's ESP32 port waits out +its whole timeout for a status to leave 0, and success is 0. The full nus gate +in that direction didn't run; see ble.md. + +**mpftp on CircuitPython**, noticed on the way: a long `exec` or `run` sent +over the serial REPL arrived garbled now and then (a base64 chunk of 6,000 +characters decoded with 11 wrong bytes, twice), and `put` with the drive +disabled failed with a syntax error. Copying through the CIRCUITPY drive and +`exec`-ing `import script` was reliable. + ### Pairing The laptop (Windows 11, bleak 3.0.2, WinRT custom pairing) and the LCD-7 diff --git a/docs/bledev.md b/docs/bledev.md index 472f554c..24f0bb25 100644 --- a/docs/bledev.md +++ b/docs/bledev.md @@ -450,7 +450,7 @@ rollover, and what a board central can't read are in The same code runs on a CircuitPython board with `bledev.cpble`, in either role: nus, MIDI, the contract's checks and the file client all passed over the -radio there (the numbers are [below](#measured-on-circuitpython)). A few +radio there (the numbers are in [the internals](bledev-internals.md#circuitpython-cpble)). A few things differ, because `_bleio` works differently from aioble underneath: - Connecting, discovery, a read, a write with response and subscribing each @@ -463,13 +463,18 @@ things differ, because `_bleio` works differently from aioble underneath: to subscribed centrals even without `send_update`. - CircuitPython 10.3's `PacketBuffer` can stall or fault the board when a notification stream outruns the radio. bledev steps around the stall; the - fault needs a CircuitPython fix, which is drafted for upstream. Until it - lands, a board that streams notifications fast can restart in safe mode. + fault needs a CircuitPython fix, drafted for upstream with a patch. On the + official build, 2 of 11 fast 16 KB streams restarted the board in safe mode; + on the patched build, none did. +- A central there waits 2 s on every read and every write with a response + (subscribing is one), so connecting and setting up takes several seconds. CircuitPython's own BLE file service, the one its supervisor serves, is reached with the same `bledev.filetransfer.connect()` a MicroPython board uses; it pairs by itself. The service is only public in CircuitPython's -discovery mode. What cpble does about each of these, and why, is in +discovery mode, it won't write while a computer has the CIRCUITPY drive +mounted, and on the official 10.3.0 build a 20 KB download stalls (the same +`PacketBuffer` fault). What cpble does about each of these, and why, is in [bledev-internals.md](bledev-internals.md#how-cpble-gets-there). ## The rules every backend keeps diff --git a/docs/upstream-reports/cp-packetbuffer-notify-stall.md b/docs/upstream-reports/cp-packetbuffer-notify-stall.md index 75378b1c..21a35a8a 100644 --- a/docs/upstream-reports/cp-packetbuffer-notify-stall.md +++ b/docs/upstream-reports/cp-packetbuffer-notify-stall.md @@ -73,12 +73,17 @@ restarted the board in safe mode (`SafeModeReason.HARD_FAULT`) in 2 of 11 - Drop a pending packet when its central unsubscribes or disconnects, so a kept packet can't reach the next central. -With it (same board, same tests): RESULTS_PATCHED +With it (same board, same tests): the 16 KB notify stream passed 12 of 12 +with no safe-mode restart, and the file service listed `/` in 50 ms and moved +20 KB up and down byte for byte in 7 of 7 round trips, where the official +build stalled on the download 2 of 2 times. Not addressed here, noticed on the way: `bleio_packet_buffer_extend()` drops the oldest packets when its ring is full, with a `// set an overflow flag?` comment; `bleio_gattc_read()` reports a timeout as success with the whole -buffer's length; and ATT errors on reads and writes reach Python as +buffer's length; `bleio_gattc_read()` and `bleio_gattc_write()` always wait out the whole +2 s, because `_wait_for_completion()` waits for `_completion_status` to leave +0 and a success sets it to 0; and ATT errors on reads and writes reach Python as `BluetoothError("Unknown system firmware error: 261")` rather than `SecurityError`, because `_wait_for_completion()` is checked with `CHECK_NIMBLE_ERROR` instead of `CHECK_BLE_ERROR`.