From 0e76598066af07a225e24b238af633c73a917d3e Mon Sep 17 00:00:00 2001 From: Albert Mavashev Date: Mon, 27 Jul 2026 15:37:03 -0400 Subject: [PATCH 1/9] fix: heartbeat extend drift + actual_source audit marker (v0.5.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extend_by_ms extends relative to the reservation's CURRENT expires_at_ms (spec ~1761; server extend.lua adds to current expiry), but every heartbeat extended by the full ttl_ms at ttl/2 cadence — drifting expiry outward +ttl/2 per beat. Consequences: a killed process left reserved budget locked until the drifted expiry (zombie window scaling with runtime, bounded only by max_extensions ~10), and long runs exhausted max_extensions twice as fast as necessary, silently losing heartbeat protection mid-flight. All four heartbeats (sync/async lifecycle, sync/async streaming) now use alternate-beat extension: extend on the first beat (only ttl/2 lifetime remains) and every second beat after a success; retry immediately after a failure. No client-vs-server clock comparison (skew-safe). Expiry lead stays within [ttl/2, 1.5*ttl]; extension consumption halved. Also: commits whose actual was structurally defaulted from the estimate (@cycles without an actual expression; streams without a recorded cost or with a raised cost_fn) now stamp metadata.actual_source="estimate" so audit evidence distinguishes measured from assumed spend. Defaults unchanged. Fleet-wide: TS/Java/Rust ship the same heartbeat fix; spec heartbeat guidance in cycles-protocol#148 (v0.1.25.16). 517 tests pass at 100% coverage; ruff and mypy --strict clean. --- AUDIT.md | 10 ++ CHANGELOG.md | 10 ++ pyproject.toml | 2 +- runcycles/lifecycle.py | 35 ++++- runcycles/streaming.py | 49 +++++-- tests/test_heartbeat.py | 293 ++++++++++++++++++++++++++++++++++++++++ tests/test_streaming.py | 14 +- 7 files changed, 394 insertions(+), 19 deletions(-) create mode 100644 tests/test_heartbeat.py diff --git a/AUDIT.md b/AUDIT.md index 423d652..0a991d7 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -13,6 +13,16 @@ --- +## 2026-07-27 — Heartbeat drift fix + actual_source marker (v0.5.1) + +The heartbeat extended by full ttl_ms every ttl/2 beat while extend_by_ms +is relative to current expiry — drifting expiry outward +ttl/2 per beat +(zombie budget lockup on kill; max_extensions burned 2× too fast). All +four heartbeats now alternate-beat extend (lead stays [ttl/2, 1.5×ttl]). +Commits whose actual was defaulted from the estimate now carry +metadata.actual_source="estimate" for audit honesty. Spec guidance: +cycles-protocol#148. 517 tests pass at 100% coverage. + ## 2026-07-27 — Durable commit retries (journal + /v1/events fallback) Pending commits no longer exist only in memory: the retry engines journal diff --git a/CHANGELOG.md b/CHANGELOG.md index 010ddd7..aa1f96f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog 1.1.0](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.1] - 2026-07-27 + +### Fixed + +- **Heartbeat extend drift**: `extend_by_ms` is relative to the reservation's *current* `expires_at_ms` (spec), but the heartbeat extended by the full `ttl_ms` on every `ttl/2` beat — drifting expiry outward by `ttl/2` per beat. A killed process left its reserved budget locked until the drifted expiry (a zombie-reservation window scaling with runtime, bounded only by `max_extensions`), and long runs burned the `max_extensions` budget twice as fast as needed, losing heartbeat protection mid-flight. All four heartbeats (both lifecycles, both streaming context managers) now use alternate-beat extension: extend on the first beat and every second beat after a success, retrying immediately after a failure. Expiry lead stays within `[ttl/2, 1.5×ttl]`; extension consumption is halved. Fleet-wide fix (TS/Java/Rust ship the same change); spec guidance added in cycles-protocol#148. + +### Added + +- `metadata.actual_source: "estimate"` is stamped on commits whose actual was structurally defaulted from the estimate (`@cycles` without an `actual` expression; streams with no recorded cost or a raised `cost_fn`), so audit evidence distinguishes measured spend from assumed spend. Defaults are unchanged; the marker flows into `/v1/events` recovery bodies via the shared metadata. + ## [0.5.0] - 2026-07-27 Durable commit retries. Previously a commit that failed transiently lived only in an in-memory daemon thread (or an unreferenced asyncio task): a process exit — even a clean one — dropped it, and once the reservation's grace period elapsed the server's expiry sweep returned the reserved budget to the pool, permanently under-counting spend that had already happened. Pending commits are now journaled to disk before retry, replayed on the next run, flushed (bounded) at interpreter exit, and — when the reservation has already expired — recovered via `POST /v1/events`, the spec's post-hoc direct-debit endpoint. diff --git a/pyproject.toml b/pyproject.toml index 1c04204..3b5bfb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "runcycles" -version = "0.5.0" +version = "0.5.1" description = "Python AI agent budget control — enforce LLM cost limits, tool permissions, and multi-tenant policies before agent actions execute." readme = "README.md" license = "Apache-2.0" diff --git a/runcycles/lifecycle.py b/runcycles/lifecycle.py index e4afabc..1a79d68 100644 --- a/runcycles/lifecycle.py +++ b/runcycles/lifecycle.py @@ -371,7 +371,14 @@ def execute( if metrics.latency_ms is None: metrics.latency_ms = method_elapsed - commit_body = _build_commit_body(actual_amount, cfg.unit, metrics, ctx.commit_metadata) + commit_metadata = ctx.commit_metadata + if cfg.actual is None: + # The estimate is being recorded as the actual (documented + # fallback). Mark the evidence so auditors can distinguish + # measured spend from assumed spend. + logger.debug("No actual expression; committing estimate as actual: id=%s", reservation_id) + commit_metadata = {**(commit_metadata or {}), "actual_source": "estimate"} + commit_body = _build_commit_body(actual_amount, cfg.unit, metrics, commit_metadata) event_fallback = _build_event_fallback_body( reservation_id, create_body["subject"], create_body["action"], commit_body, ) @@ -470,11 +477,23 @@ def _start_heartbeat( interval_s = max(ttl_ms / 2, 1000) / 1000.0 def heartbeat_loop() -> None: + # Alternate-beat extension: extend_by_ms is relative to the + # CURRENT expiry (spec), so extending by ttl on every ttl/2 beat + # drifts expiry outward +ttl/2 per beat — a zombie-reservation + # window — and burns max_extensions twice as fast as needed. + # Extend on the first beat (only ttl/2 of lifetime remains) and + # every second beat after a success; retry right away after a + # failure. Expiry lead stays within [ttl/2, 1.5*ttl]. + beats_since_extend = 1 while not stop_event.wait(timeout=interval_s): + beats_since_extend += 1 + if beats_since_extend < 2: + continue try: body = _build_extend_body(ttl_ms) response = self._client.extend_reservation(reservation_id, body) if response.is_success: + beats_since_extend = 0 new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: ctx.update_expires_at_ms(int(new_expires)) @@ -572,7 +591,13 @@ async def execute( if metrics.latency_ms is None: metrics.latency_ms = method_elapsed - commit_body = _build_commit_body(actual_amount, cfg.unit, metrics, ctx.commit_metadata) + commit_metadata = ctx.commit_metadata + if cfg.actual is None: + # See the sync lifecycle: estimate recorded as actual is + # marked so the evidence stays honest. + logger.debug("No actual expression; committing estimate as actual: id=%s", reservation_id) + commit_metadata = {**(commit_metadata or {}), "actual_source": "estimate"} + commit_body = _build_commit_body(actual_amount, cfg.unit, metrics, commit_metadata) event_fallback = _build_event_fallback_body( reservation_id, create_body["subject"], create_body["action"], commit_body, ) @@ -669,13 +694,19 @@ def _start_heartbeat(self, reservation_id: str, ttl_ms: int, ctx: CyclesContext) interval_s = max(ttl_ms / 2, 1000) / 1000.0 async def heartbeat_loop() -> None: + # Alternate-beat extension — see the sync heartbeat for rationale. + beats_since_extend = 1 try: while True: await asyncio.sleep(interval_s) + beats_since_extend += 1 + if beats_since_extend < 2: + continue try: body = _build_extend_body(ttl_ms) response = await self._client.extend_reservation(reservation_id, body) if response.is_success: + beats_since_extend = 0 new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: ctx.update_expires_at_ms(int(new_expires)) diff --git a/runcycles/streaming.py b/runcycles/streaming.py index 511da03..9d38a2f 100644 --- a/runcycles/streaming.py +++ b/runcycles/streaming.py @@ -92,17 +92,22 @@ def _resolve_actual_cost( usage: StreamUsage, cost_fn: Callable[[StreamUsage], int] | None, estimate_amount: int, -) -> int: - """Resolve the actual cost: explicit > cost_fn > estimate fallback.""" +) -> tuple[int, bool]: + """Resolve the actual cost: explicit > cost_fn > estimate fallback. + + Returns ``(amount, from_estimate)`` — the flag is True when the estimate + was substituted for an unmeasured actual, so the commit can carry an + ``actual_source`` marker for audit honesty. + """ if usage.actual_cost is not None: - return usage.actual_cost + return usage.actual_cost, False if cost_fn is not None: try: - return cost_fn(usage) + return cost_fn(usage), False except Exception: logger.warning("cost_fn raised, falling back to estimate", exc_info=True) - return estimate_amount - return estimate_amount + return estimate_amount, True + return estimate_amount, True def _build_stream_metrics( @@ -273,11 +278,17 @@ def __exit__( def _handle_commit(self) -> None: elapsed_ms = int((time.monotonic() - self._start_time) * 1000) - actual = _resolve_actual_cost(self._usage, self._cost_fn, self._estimate.amount) + actual, actual_from_estimate = _resolve_actual_cost( + self._usage, self._cost_fn, self._estimate.amount + ) ctx_metrics = self._ctx.metrics if self._ctx else None metrics = _build_stream_metrics(self._usage, elapsed_ms, ctx_metrics) unit = self._estimate.unit if isinstance(self._estimate.unit, str) else self._estimate.unit.value - commit_body = _build_commit_body(actual, unit, metrics, self._metadata) + commit_metadata = self._metadata + if actual_from_estimate: + # Estimate recorded as actual — mark the evidence (audit honesty). + commit_metadata = {**(commit_metadata or {}), "actual_source": "estimate"} + commit_body = _build_commit_body(actual, unit, metrics, commit_metadata) assert self._reservation_id is not None event_fallback = _build_event_fallback_body( @@ -364,11 +375,17 @@ def _start_heartbeat(self) -> threading.Thread | None: ctx = self._ctx def heartbeat_loop() -> None: + # Alternate-beat extension — see CyclesLifecycle heartbeat for rationale. + beats_since_extend = 1 while not self._heartbeat_stop.wait(timeout=interval_s): + beats_since_extend += 1 + if beats_since_extend < 2: + continue try: body = _build_extend_body(self._ttl_ms) response = self._client.extend_reservation(reservation_id, body) if response.is_success: + beats_since_extend = 0 new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None and ctx is not None: ctx.update_expires_at_ms(int(new_expires)) @@ -534,11 +551,17 @@ async def __aexit__( async def _handle_commit(self) -> None: elapsed_ms = int((time.monotonic() - self._start_time) * 1000) - actual = _resolve_actual_cost(self._usage, self._cost_fn, self._estimate.amount) + actual, actual_from_estimate = _resolve_actual_cost( + self._usage, self._cost_fn, self._estimate.amount + ) ctx_metrics = self._ctx.metrics if self._ctx else None metrics = _build_stream_metrics(self._usage, elapsed_ms, ctx_metrics) unit = self._estimate.unit if isinstance(self._estimate.unit, str) else self._estimate.unit.value - commit_body = _build_commit_body(actual, unit, metrics, self._metadata) + commit_metadata = self._metadata + if actual_from_estimate: + # Estimate recorded as actual — mark the evidence (audit honesty). + commit_metadata = {**(commit_metadata or {}), "actual_source": "estimate"} + commit_body = _build_commit_body(actual, unit, metrics, commit_metadata) assert self._reservation_id is not None event_fallback = _build_event_fallback_body( @@ -629,13 +652,19 @@ def _start_heartbeat(self) -> asyncio.Task[None] | None: ttl_ms = self._ttl_ms async def heartbeat_loop() -> None: + # Alternate-beat extension — see CyclesLifecycle heartbeat for rationale. + beats_since_extend = 1 try: while True: await asyncio.sleep(interval_s) + beats_since_extend += 1 + if beats_since_extend < 2: + continue try: body = _build_extend_body(ttl_ms) response = await client.extend_reservation(reservation_id, body) if response.is_success: + beats_since_extend = 0 new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None and ctx is not None: ctx.update_expires_at_ms(int(new_expires)) diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py new file mode 100644 index 0000000..1a07fd9 --- /dev/null +++ b/tests/test_heartbeat.py @@ -0,0 +1,293 @@ +"""Deterministic tests for alternate-beat heartbeat extension and the +``actual_source: estimate`` audit marker.""" + +from __future__ import annotations + +import asyncio +import threading +import time +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from runcycles.config import CyclesConfig +from runcycles.lifecycle import AsyncCyclesLifecycle, CyclesLifecycle, DecoratorConfig +from runcycles.models import Action, Amount, Subject, Unit +from runcycles.response import CyclesResponse +from runcycles.retry import AsyncCommitRetryEngine, CommitRetryEngine +from runcycles.streaming import AsyncStreamReservation, StreamReservation + + +def _config() -> CyclesConfig: + return CyclesConfig( + base_url="http://localhost:7878", api_key="test-key", tenant="acme", + retry_enabled=False, + ) + + +def _extend_ok() -> CyclesResponse: + return CyclesResponse.success(200, {"status": "ACTIVE", "expires_at_ms": 9999999999}) + + +def _allow_response() -> CyclesResponse: + return CyclesResponse.success(200, { + "decision": "ALLOW", + "reservation_id": "rsv_test", + "expires_at_ms": int(time.time() * 1000) + 600_000, + "affected_scopes": ["tenant:acme"], + "scope_path": "tenant:acme", + "reserved": {"unit": "USD_MICROCENTS", "amount": 1000}, + }) + + +def _commit_success() -> CyclesResponse: + return CyclesResponse.success(200, {"status": "COMMITTED"}) + + +def _make_sync() -> tuple[CyclesLifecycle, MagicMock]: + client = MagicMock() + client._config = _config() + engine = MagicMock(spec=CommitRetryEngine) + return CyclesLifecycle(client, engine, {"tenant": "acme"}), client + + +def _make_async() -> tuple[AsyncCyclesLifecycle, AsyncMock]: + client = AsyncMock() + client._config = _config() + engine = MagicMock(spec=AsyncCommitRetryEngine) + return AsyncCyclesLifecycle(client, engine, {"tenant": "acme"}), client + + +def _run_sync_beats(lifecycle: CyclesLifecycle, beats: int) -> None: + """Drive the sync heartbeat loop for exactly `beats` iterations.""" + stop = threading.Event() + stop.wait = MagicMock(side_effect=[False] * beats + [True]) # type: ignore[method-assign] + thread = lifecycle._start_heartbeat("rsv_1", 60_000, MagicMock(), stop) + assert thread is not None + thread.join(timeout=5) + assert not thread.is_alive() + + +class TestSyncHeartbeatAlternateBeat: + def test_extends_on_first_and_alternate_beats(self) -> None: + # extend_by_ms is relative to CURRENT expiry: extending every beat + # at ttl/2 cadence drifts expiry outward. Expected: beats 1 and 3 + # extend, beats 2 and 4 skip. + lifecycle, client = _make_sync() + client.extend_reservation.return_value = _extend_ok() + + _run_sync_beats(lifecycle, 4) + + assert client.extend_reservation.call_count == 2 + body = client.extend_reservation.call_args.args[1] + assert body["extend_by_ms"] == 60_000 # amount unchanged; cadence halved + + def test_failed_extend_retries_next_beat(self) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.side_effect = [ + CyclesResponse.http_error(500, "boom"), # beat 1: fail + _extend_ok(), # beat 2: retry, success + _extend_ok(), # beat 4: alternate resumes + ] + + _run_sync_beats(lifecycle, 4) + + # beat 3 skipped after the beat-2 success + assert client.extend_reservation.call_count == 3 + + +@pytest.mark.asyncio +class TestAsyncHeartbeatAlternateBeat: + async def _run_async_beats( + self, lifecycle: AsyncCyclesLifecycle, beats: int, monkeypatch: pytest.MonkeyPatch, + ) -> None: + count = 0 + + async def fake_sleep(_s: float) -> None: + nonlocal count + count += 1 + if count > beats: + raise asyncio.CancelledError + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + task = lifecycle._start_heartbeat("rsv_1", 60_000, MagicMock()) + assert task is not None + await task # heartbeat catches CancelledError and returns + + async def test_extends_on_first_and_alternate_beats( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + lifecycle, client = _make_async() + client.extend_reservation.return_value = _extend_ok() + + await self._run_async_beats(lifecycle, 4, monkeypatch) + + assert client.extend_reservation.await_count == 2 + + async def test_failed_extend_retries_next_beat( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + lifecycle, client = _make_async() + client.extend_reservation.side_effect = [ + CyclesResponse.http_error(500, "boom"), + _extend_ok(), + _extend_ok(), + ] + + await self._run_async_beats(lifecycle, 4, monkeypatch) + + assert client.extend_reservation.await_count == 3 + + +class TestStreamingHeartbeatAlternateBeat: + def test_sync_stream_extends_alternate_beats(self) -> None: + client = MagicMock() + client._config = _config() + client.extend_reservation.return_value = _extend_ok() + stream = StreamReservation( + client, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=60_000, + ) + stream._reservation_id = "rsv_1" + stream._heartbeat_stop.wait = MagicMock( # type: ignore[method-assign] + side_effect=[False] * 4 + [True], + ) + + thread = stream._start_heartbeat() + assert thread is not None + thread.join(timeout=5) + + assert client.extend_reservation.call_count == 2 + + @pytest.mark.asyncio + async def test_async_stream_extends_alternate_beats( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + client = AsyncMock() + client._config = _config() + client.extend_reservation.return_value = _extend_ok() + stream = AsyncStreamReservation( + client, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=60_000, + ) + stream._reservation_id = "rsv_1" + + count = 0 + + async def fake_sleep(_s: float) -> None: + nonlocal count + count += 1 + if count > 4: + raise asyncio.CancelledError + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + task = stream._start_heartbeat() + assert task is not None + await task + + assert client.extend_reservation.await_count == 2 + + +# --------------------------------------------------------------------------- +# actual_source marker +# --------------------------------------------------------------------------- + + +def _cfg(**kwargs: Any) -> DecoratorConfig: + defaults: dict[str, Any] = {"estimate": 1000, "tenant": "acme", "ttl_ms": 60_000} + defaults.update(kwargs) + return DecoratorConfig(**defaults) + + +class TestActualSourceMarker: + def test_fallback_commit_carries_marker(self) -> None: + lifecycle, client = _make_sync() + client.create_reservation.return_value = _allow_response() + client.commit_reservation.return_value = _commit_success() + + lifecycle.execute(lambda: "result", (), {}, _cfg()) # no actual expression + + body = client.commit_reservation.call_args.args[1] + assert body["metadata"]["actual_source"] == "estimate" + + def test_measured_commit_has_no_marker(self) -> None: + lifecycle, client = _make_sync() + client.create_reservation.return_value = _allow_response() + client.commit_reservation.return_value = _commit_success() + + lifecycle.execute(lambda: "result", (), {}, _cfg(actual=lambda _r: 900)) + + body = client.commit_reservation.call_args.args[1] + assert "metadata" not in body or "actual_source" not in body.get("metadata", {}) + + @pytest.mark.asyncio + async def test_async_fallback_commit_carries_marker(self) -> None: + lifecycle, client = _make_async() + client.create_reservation.return_value = _allow_response() + client.commit_reservation.return_value = _commit_success() + + async def fn() -> str: + return "result" + + await lifecycle.execute(fn, (), {}, _cfg()) + + body = client.commit_reservation.call_args.args[1] + assert body["metadata"]["actual_source"] == "estimate" + + def test_stream_fallback_carries_marker_and_measured_does_not(self) -> None: + client = MagicMock() + client._config = _config() + client.create_reservation.return_value = _allow_response() + client.commit_reservation.return_value = _commit_success() + stream = StreamReservation( + client, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=60_000, + ) + with stream: + pass # no actual cost recorded → estimate fallback + body = client.commit_reservation.call_args.args[1] + assert body["metadata"]["actual_source"] == "estimate" + + client2 = MagicMock() + client2._config = _config() + client2.create_reservation.return_value = _allow_response() + client2.commit_reservation.return_value = _commit_success() + stream2 = StreamReservation( + client2, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=60_000, + ) + with stream2: + stream2.usage.set_actual_cost(700) + body2 = client2.commit_reservation.call_args.args[1] + assert "metadata" not in body2 or "actual_source" not in body2.get("metadata", {}) + + @pytest.mark.asyncio + async def test_async_stream_fallback_carries_marker(self) -> None: + client = AsyncMock() + client._config = _config() + client.create_reservation.return_value = _allow_response() + client.commit_reservation.return_value = _commit_success() + stream = AsyncStreamReservation( + client, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=60_000, + ) + async with stream: + pass + body = client.commit_reservation.call_args.args[1] + assert body["metadata"]["actual_source"] == "estimate" diff --git a/tests/test_streaming.py b/tests/test_streaming.py index b1ad873..aede0cd 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -229,7 +229,7 @@ def test_subject_with_no_standard_fields_raises(self) -> None: class TestResolveActualCost: def test_explicit_actual_cost(self) -> None: u = StreamUsage(actual_cost=777) - assert _resolve_actual_cost(u, lambda _: 999, 1000) == 777 + assert _resolve_actual_cost(u, lambda _: 999, 1000) == (777, False) def test_cost_fn(self) -> None: u = StreamUsage(tokens_input=100, tokens_output=50) @@ -237,7 +237,7 @@ def test_cost_fn(self) -> None: def cost_fn(usage: StreamUsage) -> int: return usage.tokens_input * 2 + usage.tokens_output * 3 - assert _resolve_actual_cost(u, cost_fn, 1000) == 350 + assert _resolve_actual_cost(u, cost_fn, 1000) == (350, False) def test_cost_fn_error_falls_back_to_estimate(self) -> None: u = StreamUsage() @@ -245,11 +245,11 @@ def test_cost_fn_error_falls_back_to_estimate(self) -> None: def bad_fn(_: StreamUsage) -> int: raise ValueError("oops") - assert _resolve_actual_cost(u, bad_fn, 1000) == 1000 + assert _resolve_actual_cost(u, bad_fn, 1000) == (1000, True) def test_fallback_to_estimate(self) -> None: u = StreamUsage() - assert _resolve_actual_cost(u, None, 500) == 500 + assert _resolve_actual_cost(u, None, 500) == (500, True) # --------------------------------------------------------------------------- @@ -620,7 +620,8 @@ def test_metadata_passed_to_commit(self) -> None: pass commit_body = mock.commit_reservation.call_args[0][1] - assert commit_body["metadata"] == {"source": "test"} + # actual_source marker added because no actual cost was recorded + assert commit_body["metadata"] == {"source": "test", "actual_source": "estimate"} def test_metrics_include_tokens(self) -> None: mock = _make_mock_client() @@ -1310,7 +1311,8 @@ async def test_metadata_passed_to_commit(self) -> None: pass commit_body = mock.commit_reservation.call_args[0][1] - assert commit_body["metadata"] == {"key": "val"} + # actual_source marker added because no actual cost was recorded + assert commit_body["metadata"] == {"key": "val", "actual_source": "estimate"} @pytest.mark.asyncio async def test_caps_propagated(self) -> None: From 7fc338dbeeeff75385e2edf1bfe785af217e3a81 Mon Sep 17 00:00:00 2001 From: Albert Mavashev Date: Mon, 27 Jul 2026 16:10:37 -0400 Subject: [PATCH 2/9] =?UTF-8?q?fix:=20lead-estimate=20heartbeat=20(v2.1)?= =?UTF-8?q?=20=E2=80=94=20effective=20TTL,=20key=20reuse,=20permanent=20st?= =?UTF-8?q?ops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial self-review of the alternate-beat fix found confirmed inward-drift liveness hazards: at steady state every attempt fired at exactly ttl/2 lead so one failed extend put the retry at lead 0; the 1s interval floor guaranteed lapse for spec-legal ttl in (1000,2000); and sleep-after-response re-arming slipped beats by one RTT per cycle. The heartbeat now estimates its remaining lead from the AUTHORITATIVE expires_at_ms the server returns, compared clock-skew-free (server-frame differences plus client-monotonic elapsed only), extending by ttl when lead < 1.5*ttl and skipping otherwise. Failures retry next beat with the SAME idempotency key (a lost-response extend cannot double-apply); permanent codes (RESERVATION_EXPIRED / RESERVATION_FINALIZED / MAX_EXTENSIONS_EXCEEDED / TENANT_CLOSED / NOT_FOUND, or status 410) stop the heartbeat for good; the 1s interval floor is removed. Spec-review round: tenant policy max_reservation_ttl_ms (default 1h) silently caps granted TTLs and the create response exposes no effective TTL — seeding from the requested ttl could schedule the first beat hours after expiry. The client now captures the HTTP Date header (CyclesResponse.server_date_ms) and seeds the heartbeat from effective_ttl = clamp(expires_at_ms - Date, 1000, requested), still skew-free (server-frame difference). All four heartbeats (sync/async lifecycle, sync/async streaming) share the design. Spec guidance updated in cycles-protocol#148 (02d1270). 531 tests pass at 100% coverage; ruff and mypy --strict clean. --- AUDIT.md | 11 +- CHANGELOG.md | 3 +- runcycles/client.py | 9 +- runcycles/lifecycle.py | 144 +++++++++++-- runcycles/response.py | 18 ++ runcycles/streaming.py | 99 +++++++-- tests/test_heartbeat.py | 437 +++++++++++++++++++++++++++++++++++----- 7 files changed, 626 insertions(+), 95 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 0a991d7..78bde24 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -13,15 +13,22 @@ --- -## 2026-07-27 — Heartbeat drift fix + actual_source marker (v0.5.1) +## 2026-07-27 — Heartbeat lead-estimate redesign + actual_source marker (v0.5.1) The heartbeat extended by full ttl_ms every ttl/2 beat while extend_by_ms is relative to current expiry — drifting expiry outward +ttl/2 per beat (zombie budget lockup on kill; max_extensions burned 2× too fast). All four heartbeats now alternate-beat extend (lead stays [ttl/2, 1.5×ttl]). +Self-review found the first fix (alternate-beat) introduced inward-drift +hazards (single-failure lead-0, sub-2s-ttl floor decay, RTT slippage); the +heartbeat now runs on a clock-skew-free lead estimate from the +authoritative expires_at_ms, derives the effective TTL from the Date +header (tenant max_reservation_ttl_ms caps grants — default 1h), reuses +the extend idempotency key on retries, and stops permanently on +expired/finalized/max-extensions/tenant-closed/not-found. Commits whose actual was defaulted from the estimate now carry metadata.actual_source="estimate" for audit honesty. Spec guidance: -cycles-protocol#148. 517 tests pass at 100% coverage. +cycles-protocol#148. 531 tests pass at 100% coverage. ## 2026-07-27 — Durable commit retries (journal + /v1/events fallback) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa1f96f..f2ff7ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Heartbeat extend drift**: `extend_by_ms` is relative to the reservation's *current* `expires_at_ms` (spec), but the heartbeat extended by the full `ttl_ms` on every `ttl/2` beat — drifting expiry outward by `ttl/2` per beat. A killed process left its reserved budget locked until the drifted expiry (a zombie-reservation window scaling with runtime, bounded only by `max_extensions`), and long runs burned the `max_extensions` budget twice as fast as needed, losing heartbeat protection mid-flight. All four heartbeats (both lifecycles, both streaming context managers) now use alternate-beat extension: extend on the first beat and every second beat after a success, retrying immediately after a failure. Expiry lead stays within `[ttl/2, 1.5×ttl]`; extension consumption is halved. Fleet-wide fix (TS/Java/Rust ship the same change); spec guidance added in cycles-protocol#148. +- **Heartbeat redesign (lead-estimate)**: the initial alternate-beat fix traded outward drift for inward-drift liveness hazards (single-failure lead-0 retry, guaranteed lapse for ttl < 2s under the old 1s interval floor, RTT slippage) found by adversarial self-review. The heartbeat now estimates its remaining lead from the authoritative `expires_at_ms` the server returns — compared clock-skew-free (server-frame differences + client-monotonic elapsed only) — and extends when lead < 1.5×ttl. It also: derives the **effective TTL** from `expires_at_ms − Date` header (tenant policy `max_reservation_ttl_ms`, default 1h, silently caps grants — a 24h request would otherwise heartbeat 12h late); reuses the same idempotency key when retrying a failed extend (a lost response cannot double-extend); stops permanently on `RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`; and drops the 1s interval floor. Spec guidance: cycles-protocol#148. +- **Heartbeat extend drift** (superseded by the redesign above, kept for history): `extend_by_ms` is relative to the reservation's *current* `expires_at_ms` (spec), but the heartbeat extended by the full `ttl_ms` on every `ttl/2` beat — drifting expiry outward by `ttl/2` per beat. A killed process left its reserved budget locked until the drifted expiry (a zombie-reservation window scaling with runtime, bounded only by `max_extensions`), and long runs burned the `max_extensions` budget twice as fast as needed, losing heartbeat protection mid-flight. All four heartbeats (both lifecycles, both streaming context managers) now use alternate-beat extension: extend on the first beat and every second beat after a success, retrying immediately after a failure. Expiry lead stays within `[ttl/2, 1.5×ttl]`; extension consumption is halved. Fleet-wide fix (TS/Java/Rust ship the same change); spec guidance added in cycles-protocol#148. ### Added diff --git a/runcycles/client.py b/runcycles/client.py index 4b8581b..75518a0 100644 --- a/runcycles/client.py +++ b/runcycles/client.py @@ -41,7 +41,14 @@ def _extract_idempotency_key(body: dict[str, Any]) -> str | None: return body.get("idempotency_key") -_RESPONSE_HEADERS = ("x-request-id", "x-ratelimit-remaining", "x-ratelimit-reset", "x-cycles-tenant", "retry-after") +_RESPONSE_HEADERS = ( + "x-request-id", + "x-ratelimit-remaining", + "x-ratelimit-reset", + "x-cycles-tenant", + "retry-after", + "date", +) _BALANCE_FILTER_PARAMS = {"tenant", "workspace", "app", "workflow", "agent", "toolset"} diff --git a/runcycles/lifecycle.py b/runcycles/lifecycle.py index 1a79d68..12cc985 100644 --- a/runcycles/lifecycle.py +++ b/runcycles/lifecycle.py @@ -40,6 +40,7 @@ from runcycles.retry import ( AsyncCommitRetryEngine, CommitRetryEngine, + _extract_error_code, _is_recognized_rejection, ) @@ -198,6 +199,49 @@ def _build_release_body(reason: str) -> dict[str, Any]: return {"idempotency_key": str(uuid.uuid4()), "reason": reason} +def _now_mono_ms() -> float: + """Monotonic milliseconds — the heartbeat's only clock (test seam).""" + return time.monotonic() * 1000.0 + + +# Extend failures that can never succeed again — the heartbeat stops on them. +_PERMANENT_EXTEND_CODES = frozenset( + { + "RESERVATION_EXPIRED", + "RESERVATION_FINALIZED", + "MAX_EXTENSIONS_EXCEEDED", + "TENANT_CLOSED", # closure is irreversible per cascade semantics + "NOT_FOUND", # a purged reservation never comes back + } +) + + +def _effective_ttl_ms( + requested_ttl_ms: int, + expires_at_ms: int | None, + server_date_ms: int | None, +) -> int: + """The TTL the server actually granted, best-effort. + + Tenant policy ``max_reservation_ttl_ms`` (default 1h) silently caps the + granted TTL, and the create response has no effective-TTL field — + scheduling the heartbeat from the REQUESTED ttl can put the first beat + long after expiry. Derive the grant from two server-frame values — + ``expires_at_ms`` minus the HTTP ``Date`` header — which stays + clock-skew-free (the header's ~1s resolution is negligible against + multi-second TTLs). Falls back to the requested ttl when either value + is unavailable. + """ + if expires_at_ms is None or server_date_ms is None: + return requested_ttl_ms + derived = expires_at_ms - server_date_ms + return max(1000, min(derived, requested_ttl_ms)) +# Lead threshold: extend when the estimated remaining lifetime drops below +# this multiple of ttl. Attempts then happen with ~ttl of margin, tolerating +# failed beats; the success-path lead stays within ~[ttl, 2*ttl]. +_LEAD_TARGET_FACTOR = 1.5 + + def _build_extend_body(ttl_ms: int) -> dict[str, Any]: validate_extend_by_ms(ttl_ms) return {"idempotency_key": str(uuid.uuid4()), "extend_by_ms": ttl_ms} @@ -354,7 +398,8 @@ def execute( # Start heartbeat heartbeat_stop = threading.Event() - heartbeat_thread = self._start_heartbeat(reservation_id, cfg.ttl_ms, ctx, heartbeat_stop) + hb_ttl = _effective_ttl_ms(cfg.ttl_ms, res_result.expires_at_ms, res_response.server_date_ms) + heartbeat_thread = self._start_heartbeat(reservation_id, hb_ttl, ctx, heartbeat_stop) try: result = fn(*args, **kwargs) @@ -474,31 +519,61 @@ def _start_heartbeat( ) -> threading.Thread | None: if ttl_ms <= 0: return None - interval_s = max(ttl_ms / 2, 1000) / 1000.0 + # No 1s floor: for spec-legal ttl < 2000 a floored interval cannot + # keep the reservation alive (each extend adds only ttl of lifetime). + interval_s = (ttl_ms / 2) / 1000.0 def heartbeat_loop() -> None: - # Alternate-beat extension: extend_by_ms is relative to the - # CURRENT expiry (spec), so extending by ttl on every ttl/2 beat - # drifts expiry outward +ttl/2 per beat — a zombie-reservation - # window — and burns max_extensions twice as fast as needed. - # Extend on the first beat (only ttl/2 of lifetime remains) and - # every second beat after a success; retry right away after a - # failure. Expiry lead stays within [ttl/2, 1.5*ttl]. - beats_since_extend = 1 + # Lead-estimate heartbeat: extend_by_ms is relative to the + # CURRENT expiry (spec), so blind cadence-based extension either + # drifts expiry outward or leaves zero margin after a failed + # beat. Instead, estimate the remaining lead from the + # AUTHORITATIVE expires_at_ms the server returns, compared + # skew-free: server-frame differences plus client-monotonic + # elapsed only (never client wall clock vs server wall clock). + # Extend when lead < 1.5*ttl; skip otherwise. Failed extends are + # retried with the SAME body (same idempotency key) so a lost + # response cannot double-extend; permanent rejections stop the + # heartbeat for good. + initial_expiry = ctx.expires_at_ms + known_expiry = initial_expiry + anchor_ms = _now_mono_ms() + pending_body: dict[str, Any] | None = None while not stop_event.wait(timeout=interval_s): - beats_since_extend += 1 - if beats_since_extend < 2: + elapsed = _now_mono_ms() - anchor_ms + if initial_expiry is not None and known_expiry is not None: + lead = (known_expiry - initial_expiry) + ttl_ms - elapsed + else: + lead = ttl_ms - elapsed + if lead >= _LEAD_TARGET_FACTOR * ttl_ms: continue try: - body = _build_extend_body(ttl_ms) + body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) + pending_body = body response = self._client.extend_reservation(reservation_id, body) if response.is_success: - beats_since_extend = 0 + pending_body = None new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: ctx.update_expires_at_ms(int(new_expires)) + if initial_expiry is None: + # Late anchor: treat this response as the frame origin. + initial_expiry = int(new_expires) + known_expiry = initial_expiry + anchor_ms = _now_mono_ms() + else: + known_expiry = int(new_expires) + elif known_expiry is not None: + known_expiry += ttl_ms logger.debug("Heartbeat extend ok: id=%s", reservation_id) else: + code = _extract_error_code(response) + if response.status == 410 or code in _PERMANENT_EXTEND_CODES: + logger.warning( + "Heartbeat stopping permanently (%s, status=%d): id=%s", + code, response.status, reservation_id, + ) + return logger.warning("Heartbeat extend failed: id=%s, status=%d", reservation_id, response.status) except Exception: logger.warning("Heartbeat extend error: id=%s", reservation_id, exc_info=True) @@ -577,7 +652,8 @@ async def execute( ) _set_context(ctx) - heartbeat_task = self._start_heartbeat(reservation_id, cfg.ttl_ms, ctx) + hb_ttl = _effective_ttl_ms(cfg.ttl_ms, res_result.expires_at_ms, res_response.server_date_ms) + heartbeat_task = self._start_heartbeat(reservation_id, hb_ttl, ctx) try: result = await fn(*args, **kwargs) @@ -691,26 +767,50 @@ async def _handle_release(self, reservation_id: str, reason: str) -> None: def _start_heartbeat(self, reservation_id: str, ttl_ms: int, ctx: CyclesContext) -> asyncio.Task[None] | None: if ttl_ms <= 0: return None - interval_s = max(ttl_ms / 2, 1000) / 1000.0 + # No 1s floor — see the sync heartbeat for rationale. + interval_s = (ttl_ms / 2) / 1000.0 async def heartbeat_loop() -> None: - # Alternate-beat extension — see the sync heartbeat for rationale. - beats_since_extend = 1 + # Lead-estimate heartbeat — see the sync heartbeat for rationale. + initial_expiry = ctx.expires_at_ms + known_expiry = initial_expiry + anchor_ms = _now_mono_ms() + pending_body: dict[str, Any] | None = None try: while True: await asyncio.sleep(interval_s) - beats_since_extend += 1 - if beats_since_extend < 2: + elapsed = _now_mono_ms() - anchor_ms + if initial_expiry is not None and known_expiry is not None: + lead = (known_expiry - initial_expiry) + ttl_ms - elapsed + else: + lead = ttl_ms - elapsed + if lead >= _LEAD_TARGET_FACTOR * ttl_ms: continue try: - body = _build_extend_body(ttl_ms) + body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) + pending_body = body response = await self._client.extend_reservation(reservation_id, body) if response.is_success: - beats_since_extend = 0 + pending_body = None new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: ctx.update_expires_at_ms(int(new_expires)) + if initial_expiry is None: + initial_expiry = int(new_expires) + known_expiry = initial_expiry + anchor_ms = _now_mono_ms() + else: + known_expiry = int(new_expires) + elif known_expiry is not None: + known_expiry += ttl_ms else: + code = _extract_error_code(response) + if response.status == 410 or code in _PERMANENT_EXTEND_CODES: + logger.warning( + "Heartbeat stopping permanently (%s, status=%d): id=%s", + code, response.status, reservation_id, + ) + return logger.warning("Heartbeat extend failed: id=%s", reservation_id) except Exception: logger.warning("Heartbeat extend error: id=%s", reservation_id, exc_info=True) diff --git a/runcycles/response.py b/runcycles/response.py index c65eed3..6836d0b 100644 --- a/runcycles/response.py +++ b/runcycles/response.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from email.utils import parsedate_to_datetime from typing import Any from runcycles.models import ErrorResponse @@ -73,6 +74,23 @@ def retry_after_ms_header(self) -> int | None: except ValueError: return None + @property + def server_date_ms(self) -> int | None: + """HTTP ``Date`` header as epoch milliseconds (server wall clock). + + Server-frame, so differencing it against other server-frame values + (like ``expires_at_ms``) is clock-skew-free to within the header's + one-second resolution plus transit latency. Returns ``None`` when + absent or unparseable. + """ + val = self.headers.get("date") + if val is None: + return None + try: + return int(parsedate_to_datetime(val).timestamp() * 1000) + except (ValueError, TypeError): + return None + @property def is_success(self) -> bool: return 200 <= self.status < 300 diff --git a/runcycles/streaming.py b/runcycles/streaming.py index 9d38a2f..1c6559d 100644 --- a/runcycles/streaming.py +++ b/runcycles/streaming.py @@ -11,11 +11,14 @@ from dataclasses import dataclass, field from typing import Any +from runcycles import lifecycle as _lifecycle from runcycles._validation import validate_grace_period_ms, validate_subject, validate_ttl_ms from runcycles.client import AsyncCyclesClient, CyclesClient from runcycles.context import CyclesContext, _clear_context, _set_context from runcycles.exceptions import CyclesProtocolError from runcycles.lifecycle import ( + _LEAD_TARGET_FACTOR, + _PERMANENT_EXTEND_CODES, _build_commit_body, _build_event_fallback_body, _build_extend_body, @@ -34,6 +37,7 @@ from runcycles.retry import ( AsyncCommitRetryEngine, CommitRetryEngine, + _extract_error_code, _is_recognized_rejection, ) @@ -180,6 +184,7 @@ def __init__( self._heartbeat_stop = threading.Event() self._heartbeat_thread: threading.Thread | None = None + self._hb_ttl = ttl_ms self._retry_engine = CommitRetryEngine(client._config) self._retry_engine.set_client(client) @@ -245,6 +250,9 @@ def __enter__(self) -> StreamReservation: ) _set_context(self._ctx) + self._hb_ttl = _lifecycle._effective_ttl_ms( + self._ttl_ms, result.expires_at_ms, response.server_date_ms + ) self._start_time = time.monotonic() self._heartbeat_thread = self._start_heartbeat() @@ -369,27 +377,52 @@ def _handle_release(self, reason: str) -> None: def _start_heartbeat(self) -> threading.Thread | None: if self._ttl_ms <= 0: return None - interval_s = max(self._ttl_ms / 2, 1000) / 1000.0 + interval_s = (self._hb_ttl / 2) / 1000.0 # no 1s floor — see lifecycle heartbeat assert self._reservation_id is not None reservation_id: str = self._reservation_id ctx = self._ctx def heartbeat_loop() -> None: - # Alternate-beat extension — see CyclesLifecycle heartbeat for rationale. - beats_since_extend = 1 + # Lead-estimate heartbeat — see CyclesLifecycle heartbeat for rationale. + ttl_ms = self._hb_ttl + initial_expiry = ctx.expires_at_ms if ctx is not None else None + known_expiry = initial_expiry + anchor_ms = _lifecycle._now_mono_ms() + pending_body: dict[str, Any] | None = None while not self._heartbeat_stop.wait(timeout=interval_s): - beats_since_extend += 1 - if beats_since_extend < 2: + elapsed = _lifecycle._now_mono_ms() - anchor_ms + if initial_expiry is not None and known_expiry is not None: + lead = (known_expiry - initial_expiry) + ttl_ms - elapsed + else: + lead = ttl_ms - elapsed + if lead >= _LEAD_TARGET_FACTOR * ttl_ms: continue try: - body = _build_extend_body(self._ttl_ms) + body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) + pending_body = body response = self._client.extend_reservation(reservation_id, body) if response.is_success: - beats_since_extend = 0 + pending_body = None new_expires = response.get_body_attribute("expires_at_ms") - if new_expires is not None and ctx is not None: - ctx.update_expires_at_ms(int(new_expires)) + if new_expires is not None: + if ctx is not None: + ctx.update_expires_at_ms(int(new_expires)) + if initial_expiry is None: + initial_expiry = int(new_expires) + known_expiry = initial_expiry + anchor_ms = _lifecycle._now_mono_ms() + else: + known_expiry = int(new_expires) + elif known_expiry is not None: + known_expiry += ttl_ms else: + code = _extract_error_code(response) + if response.status == 410 or code in _PERMANENT_EXTEND_CODES: + logger.warning( + "Stream heartbeat stopping permanently (%s, status=%d): id=%s", + code, response.status, reservation_id, + ) + return logger.warning("Stream heartbeat failed: id=%s", reservation_id) except Exception: logger.warning("Stream heartbeat error: id=%s", reservation_id, exc_info=True) @@ -450,6 +483,7 @@ def __init__( self._start_time: float = 0.0 self._heartbeat_task: asyncio.Task[None] | None = None + self._hb_ttl = ttl_ms self._retry_engine = AsyncCommitRetryEngine(client._config) self._retry_engine.set_client(client) @@ -515,6 +549,9 @@ async def __aenter__(self) -> AsyncStreamReservation: ) _set_context(self._ctx) + self._hb_ttl = _lifecycle._effective_ttl_ms( + self._ttl_ms, result.expires_at_ms, response.server_date_ms + ) self._start_time = time.monotonic() self._heartbeat_task = self._start_heartbeat() @@ -644,31 +681,55 @@ async def _handle_release(self, reason: str) -> None: def _start_heartbeat(self) -> asyncio.Task[None] | None: if self._ttl_ms <= 0: return None - interval_s = max(self._ttl_ms / 2, 1000) / 1000.0 + interval_s = (self._hb_ttl / 2) / 1000.0 # no 1s floor — see lifecycle heartbeat assert self._reservation_id is not None reservation_id: str = self._reservation_id ctx = self._ctx client = self._client - ttl_ms = self._ttl_ms + ttl_ms = self._hb_ttl async def heartbeat_loop() -> None: - # Alternate-beat extension — see CyclesLifecycle heartbeat for rationale. - beats_since_extend = 1 + # Lead-estimate heartbeat — see CyclesLifecycle heartbeat for rationale. + initial_expiry = ctx.expires_at_ms if ctx is not None else None + known_expiry = initial_expiry + anchor_ms = _lifecycle._now_mono_ms() + pending_body: dict[str, Any] | None = None try: while True: await asyncio.sleep(interval_s) - beats_since_extend += 1 - if beats_since_extend < 2: + elapsed = _lifecycle._now_mono_ms() - anchor_ms + if initial_expiry is not None and known_expiry is not None: + lead = (known_expiry - initial_expiry) + ttl_ms - elapsed + else: + lead = ttl_ms - elapsed + if lead >= _LEAD_TARGET_FACTOR * ttl_ms: continue try: - body = _build_extend_body(ttl_ms) + body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) + pending_body = body response = await client.extend_reservation(reservation_id, body) if response.is_success: - beats_since_extend = 0 + pending_body = None new_expires = response.get_body_attribute("expires_at_ms") - if new_expires is not None and ctx is not None: - ctx.update_expires_at_ms(int(new_expires)) + if new_expires is not None: + if ctx is not None: + ctx.update_expires_at_ms(int(new_expires)) + if initial_expiry is None: + initial_expiry = int(new_expires) + known_expiry = initial_expiry + anchor_ms = _lifecycle._now_mono_ms() + else: + known_expiry = int(new_expires) + elif known_expiry is not None: + known_expiry += ttl_ms else: + code = _extract_error_code(response) + if response.status == 410 or code in _PERMANENT_EXTEND_CODES: + logger.warning( + "Async stream heartbeat stopping permanently (%s, status=%d): id=%s", + code, response.status, reservation_id, + ) + return logger.warning("Async stream heartbeat failed: id=%s", reservation_id) except Exception: logger.warning("Async stream heartbeat error: id=%s", reservation_id, exc_info=True) diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 1a07fd9..1580670 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -1,4 +1,4 @@ -"""Deterministic tests for alternate-beat heartbeat extension and the +"""Deterministic tests for the lead-estimate heartbeat and the ``actual_source: estimate`` audit marker.""" from __future__ import annotations @@ -11,6 +11,7 @@ import pytest +import runcycles.lifecycle as lifecycle_mod from runcycles.config import CyclesConfig from runcycles.lifecycle import AsyncCyclesLifecycle, CyclesLifecycle, DecoratorConfig from runcycles.models import Action, Amount, Subject, Unit @@ -18,6 +19,17 @@ from runcycles.retry import AsyncCommitRetryEngine, CommitRetryEngine from runcycles.streaming import AsyncStreamReservation, StreamReservation +TTL = 60_000 +INITIAL_EXPIRY = 60_000 # server frame; arbitrary origin + + +class FakeClock: + def __init__(self) -> None: + self.t = 0.0 + + def now(self) -> float: + return self.t + def _config() -> CyclesConfig: return CyclesConfig( @@ -26,8 +38,11 @@ def _config() -> CyclesConfig: ) -def _extend_ok() -> CyclesResponse: - return CyclesResponse.success(200, {"status": "ACTIVE", "expires_at_ms": 9999999999}) +def _extend_ok(expires_at_ms: int | None) -> CyclesResponse: + body: dict[str, Any] = {"status": "ACTIVE"} + if expires_at_ms is not None: + body["expires_at_ms"] = expires_at_ms + return CyclesResponse.success(200, body) def _allow_response() -> CyclesResponse: @@ -59,140 +74,462 @@ def _make_async() -> tuple[AsyncCyclesLifecycle, AsyncMock]: return AsyncCyclesLifecycle(client, engine, {"tenant": "acme"}), client -def _run_sync_beats(lifecycle: CyclesLifecycle, beats: int) -> None: - """Drive the sync heartbeat loop for exactly `beats` iterations.""" +def _ctx(expires_at_ms: int | None = INITIAL_EXPIRY) -> MagicMock: + ctx = MagicMock() + ctx.expires_at_ms = expires_at_ms + return ctx + + +def _run_sync_beats( + lifecycle: CyclesLifecycle, + clock: FakeClock, + monkeypatch: pytest.MonkeyPatch, + beats: int, + ttl: int = TTL, + ctx: MagicMock | None = None, +) -> list[float]: + """Drive the sync heartbeat for `beats` iterations, advancing the fake + clock by the beat interval on every wait. Returns the wait timeouts.""" + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + timeouts: list[float] = [] + calls = {"n": 0} + + def wait(timeout: float | None = None) -> bool: + timeouts.append(timeout or 0.0) + calls["n"] += 1 + if calls["n"] > beats: + return True + clock.t += (timeout or 0.0) * 1000.0 + return False + stop = threading.Event() - stop.wait = MagicMock(side_effect=[False] * beats + [True]) # type: ignore[method-assign] - thread = lifecycle._start_heartbeat("rsv_1", 60_000, MagicMock(), stop) + stop.wait = wait # type: ignore[method-assign] + thread = lifecycle._start_heartbeat("rsv_1", ttl, ctx or _ctx(), stop) assert thread is not None thread.join(timeout=5) assert not thread.is_alive() + return timeouts + + +class TestSyncHeartbeatLeadEstimate: + def test_extends_only_when_lead_below_threshold( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Responses grant +ttl each; expected pattern over 4 beats at ttl/2 + # cadence: extend, extend, skip (lead hits 1.5*ttl), extend. + lifecycle, client = _make_sync() + client.extend_reservation.side_effect = [ + _extend_ok(INITIAL_EXPIRY + TTL), + _extend_ok(INITIAL_EXPIRY + 2 * TTL), + _extend_ok(INITIAL_EXPIRY + 3 * TTL), + ] + timeouts = _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=4) -class TestSyncHeartbeatAlternateBeat: - def test_extends_on_first_and_alternate_beats(self) -> None: - # extend_by_ms is relative to CURRENT expiry: extending every beat - # at ttl/2 cadence drifts expiry outward. Expected: beats 1 and 3 - # extend, beats 2 and 4 skip. + assert client.extend_reservation.call_count == 3 + assert timeouts[0] == TTL / 2 / 1000.0 # no 1s floor at this ttl + + def test_interval_has_no_floor_for_small_ttl( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # ttl=1200 → interval must be 600ms (the old 1s floor guaranteed + # lapse in this spec-legal range). lifecycle, client = _make_sync() - client.extend_reservation.return_value = _extend_ok() + client.extend_reservation.return_value = _extend_ok(None) + + ctx = _ctx(1200) + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=2, ttl=1200, ctx=ctx, + ) - _run_sync_beats(lifecycle, 4) + assert timeouts[0] == 0.6 + def test_failed_extend_retries_with_same_idempotency_key( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # A lost/failed extend may have been applied server-side: the retry + # must reuse the same key so it cannot double-extend. After a + # success, a fresh key is used. + lifecycle, client = _make_sync() + client.extend_reservation.side_effect = [ + CyclesResponse.http_error(500, "boom"), + _extend_ok(INITIAL_EXPIRY + TTL), + _extend_ok(INITIAL_EXPIRY + 2 * TTL), + ] + + _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3) + + bodies = [c.args[1] for c in client.extend_reservation.call_args_list] + assert len(bodies) == 3 + assert bodies[0]["idempotency_key"] == bodies[1]["idempotency_key"] + assert bodies[2]["idempotency_key"] != bodies[0]["idempotency_key"] + + def test_permanent_code_stops_heartbeat( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.return_value = CyclesResponse.http_error( + 409, "capped", + body={"error": "MAX_EXTENSIONS_EXCEEDED", "message": "m", "request_id": "r"}, + ) + + _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=4) + + # One doomed call, then the loop self-terminates — no retry spam. + assert client.extend_reservation.call_count == 1 + + def test_clamped_grants_extend_every_beat( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Server clamps each grant to ttl/4: the lead estimate sees the + # small grants (authoritative expires_at) and keeps extending. + lifecycle, client = _make_sync() + client.extend_reservation.side_effect = [ + _extend_ok(INITIAL_EXPIRY + (n + 1) * (TTL // 4)) for n in range(3) + ] + + _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3) + + assert client.extend_reservation.call_count == 3 + + def test_missing_expires_in_response_falls_back_to_plus_ttl( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.return_value = _extend_ok(None) + ctx = _ctx() + + _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3, ctx=ctx) + + # +=ttl fallback: extend, extend, skip — and ctx never updated. assert client.extend_reservation.call_count == 2 - body = client.extend_reservation.call_args.args[1] - assert body["extend_by_ms"] == 60_000 # amount unchanged; cadence halved + ctx.update_expires_at_ms.assert_not_called() + + def test_unknown_initial_expiry_anchors_on_first_success( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.side_effect = [ + _extend_ok(500_000), # beat 1: anchors the frame + _extend_ok(500_000 + 2 * TTL), # beat 2: big grant → beat 3 skips + ] + + _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3, ctx=_ctx(None)) + + # Beat 1 extends conservatively (lead unknown) and anchors on the + # authoritative response; beat 2 extends; beat 3 skips on the + # accumulated lead. + assert client.extend_reservation.call_count == 2 + + def test_tenant_closed_stops_heartbeat( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.return_value = CyclesResponse.http_error( + 409, "closed", + body={"error": "TENANT_CLOSED", "message": "m", "request_id": "r"}, + ) + + _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=4) - def test_failed_extend_retries_next_beat(self) -> None: + assert client.extend_reservation.call_count == 1 + + def test_transient_failure_then_recovery( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Non-permanent failures warn and keep the loop alive. lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ - CyclesResponse.http_error(500, "boom"), # beat 1: fail - _extend_ok(), # beat 2: retry, success - _extend_ok(), # beat 4: alternate resumes + CyclesResponse.http_error(503, "unavailable"), + ConnectionError("network down"), + _extend_ok(INITIAL_EXPIRY + 3 * TTL), ] - _run_sync_beats(lifecycle, 4) + _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3) - # beat 3 skipped after the beat-2 success assert client.extend_reservation.call_count == 3 @pytest.mark.asyncio -class TestAsyncHeartbeatAlternateBeat: - async def _run_async_beats( - self, lifecycle: AsyncCyclesLifecycle, beats: int, monkeypatch: pytest.MonkeyPatch, +class TestAsyncHeartbeatLeadEstimate: + async def _run( + self, + lifecycle: AsyncCyclesLifecycle, + beats: int, + monkeypatch: pytest.MonkeyPatch, + ctx: MagicMock | None = None, ) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) count = 0 - async def fake_sleep(_s: float) -> None: + async def fake_sleep(s: float) -> None: nonlocal count count += 1 if count > beats: raise asyncio.CancelledError + clock.t += s * 1000.0 monkeypatch.setattr(asyncio, "sleep", fake_sleep) - task = lifecycle._start_heartbeat("rsv_1", 60_000, MagicMock()) + task = lifecycle._start_heartbeat("rsv_1", TTL, ctx or _ctx()) assert task is not None - await task # heartbeat catches CancelledError and returns + await task - async def test_extends_on_first_and_alternate_beats( + async def test_extends_only_when_lead_below_threshold( self, monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle, client = _make_async() - client.extend_reservation.return_value = _extend_ok() + client.extend_reservation.side_effect = [ + _extend_ok(INITIAL_EXPIRY + TTL), + _extend_ok(INITIAL_EXPIRY + 2 * TTL), + _extend_ok(INITIAL_EXPIRY + 3 * TTL), + ] - await self._run_async_beats(lifecycle, 4, monkeypatch) + await self._run(lifecycle, 4, monkeypatch) - assert client.extend_reservation.await_count == 2 + assert client.extend_reservation.await_count == 3 - async def test_failed_extend_retries_next_beat( + async def test_permanent_code_stops_heartbeat( self, monkeypatch: pytest.MonkeyPatch, ) -> None: + lifecycle, client = _make_async() + client.extend_reservation.return_value = CyclesResponse.http_error(410, "gone") + + await self._run(lifecycle, 4, monkeypatch) + + assert client.extend_reservation.await_count == 1 + + async def test_transient_failure_missing_expires_and_late_anchor( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Covers the async warn-continue, +=ttl fallback, and late-anchor branches. lifecycle, client = _make_async() client.extend_reservation.side_effect = [ CyclesResponse.http_error(500, "boom"), - _extend_ok(), - _extend_ok(), + _extend_ok(None), # frame not yet anchored + _extend_ok(700_000), # late anchor + _extend_ok(None), # anchored: += ttl fallback ] - await self._run_async_beats(lifecycle, 4, monkeypatch) + await self._run(lifecycle, 4, monkeypatch, ctx=_ctx(None)) - assert client.extend_reservation.await_count == 3 + assert client.extend_reservation.await_count == 4 -class TestStreamingHeartbeatAlternateBeat: - def test_sync_stream_extends_alternate_beats(self) -> None: +class TestStreamingHeartbeatLeadEstimate: + def test_sync_stream_lead_estimate_pattern( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) client = MagicMock() client._config = _config() - client.extend_reservation.return_value = _extend_ok() + client.extend_reservation.side_effect = [ + _extend_ok(INITIAL_EXPIRY + TTL), + _extend_ok(INITIAL_EXPIRY + 2 * TTL), + _extend_ok(INITIAL_EXPIRY + 3 * TTL), + ] stream = StreamReservation( client, subject=Subject(tenant="acme"), action=Action(kind="k", name="n"), estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), - ttl_ms=60_000, + ttl_ms=TTL, ) stream._reservation_id = "rsv_1" - stream._heartbeat_stop.wait = MagicMock( # type: ignore[method-assign] - side_effect=[False] * 4 + [True], + stream._ctx = _ctx() + calls = {"n": 0} + + def wait(timeout: float | None = None) -> bool: + calls["n"] += 1 + if calls["n"] > 4: + return True + clock.t += (timeout or 0.0) * 1000.0 + return False + + stream._heartbeat_stop.wait = wait # type: ignore[method-assign] + thread = stream._start_heartbeat() + assert thread is not None + thread.join(timeout=5) + + assert client.extend_reservation.call_count == 3 + + def test_sync_stream_permanent_and_fallback_branches( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Covers the sync-stream +=ttl fallback, late-anchor, transient-warn, + # and permanent-stop branches in one deterministic run. + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + client = MagicMock() + client._config = _config() + client.extend_reservation.side_effect = [ + CyclesResponse.http_error(500, "boom"), # transient: warn, retry + _extend_ok(None), # no anchor yet: no-op + _extend_ok(900_000), # late anchor + _extend_ok(None), # anchored: += ttl fallback + CyclesResponse.http_error(410, "gone"), # permanent: stop + ] + stream = StreamReservation( + client, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=TTL, ) + stream._reservation_id = "rsv_1" + stream._ctx = _ctx(None) + calls = {"n": 0} + + def wait(timeout: float | None = None) -> bool: + calls["n"] += 1 + if calls["n"] > 8: + return True + clock.t += (timeout or 0.0) * 1000.0 + return False + stream._heartbeat_stop.wait = wait # type: ignore[method-assign] thread = stream._start_heartbeat() assert thread is not None thread.join(timeout=5) - assert client.extend_reservation.call_count == 2 + # Stops at the 410 — remaining beats never call extend. + assert client.extend_reservation.call_count == 5 @pytest.mark.asyncio - async def test_async_stream_extends_alternate_beats( + async def test_async_stream_lead_estimate_pattern( self, monkeypatch: pytest.MonkeyPatch, ) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) client = AsyncMock() client._config = _config() - client.extend_reservation.return_value = _extend_ok() + client.extend_reservation.side_effect = [ + _extend_ok(INITIAL_EXPIRY + TTL), + _extend_ok(INITIAL_EXPIRY + 2 * TTL), + _extend_ok(INITIAL_EXPIRY + 3 * TTL), + ] stream = AsyncStreamReservation( client, subject=Subject(tenant="acme"), action=Action(kind="k", name="n"), estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), - ttl_ms=60_000, + ttl_ms=TTL, ) stream._reservation_id = "rsv_1" - + stream._ctx = _ctx() count = 0 - async def fake_sleep(_s: float) -> None: + async def fake_sleep(s: float) -> None: nonlocal count count += 1 if count > 4: raise asyncio.CancelledError + clock.t += s * 1000.0 + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + task = stream._start_heartbeat() + assert task is not None + await task + + assert client.extend_reservation.await_count == 3 + + @pytest.mark.asyncio + async def test_async_stream_permanent_and_fallback_branches( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + client = AsyncMock() + client._config = _config() + client.extend_reservation.side_effect = [ + CyclesResponse.http_error(500, "boom"), + _extend_ok(None), # no anchor yet: no-op + _extend_ok(900_000), # late anchor + _extend_ok(None), # anchored: += ttl fallback + CyclesResponse.http_error(410, "gone"), + ] + stream = AsyncStreamReservation( + client, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=TTL, + ) + stream._reservation_id = "rsv_1" + stream._ctx = _ctx(None) + count = 0 + + async def fake_sleep(s: float) -> None: + nonlocal count + count += 1 + if count > 8: + raise asyncio.CancelledError + clock.t += s * 1000.0 monkeypatch.setattr(asyncio, "sleep", fake_sleep) task = stream._start_heartbeat() assert task is not None await task - assert client.extend_reservation.await_count == 2 + assert client.extend_reservation.await_count == 5 + + +# --------------------------------------------------------------------------- +# Effective TTL (tenant policy caps) +# --------------------------------------------------------------------------- + + +class TestEffectiveTtl: + def test_server_date_ms_parses_http_date(self) -> None: + response = CyclesResponse.success( + 200, {}, headers={"date": "Mon, 27 Jul 2026 12:00:00 GMT"}, + ) + assert response.server_date_ms == 1785153600000 + + def test_server_date_ms_absent_or_garbage(self) -> None: + assert CyclesResponse.success(200, {}).server_date_ms is None + assert ( + CyclesResponse.success(200, {}, headers={"date": "not a date"}).server_date_ms + is None + ) + + def test_effective_ttl_derives_capped_grant(self) -> None: + from runcycles.lifecycle import _effective_ttl_ms + + # Requested 24h, tenant policy capped to 1h: expires − Date = 1h. + assert _effective_ttl_ms(86_400_000, 1_000_000 + 3_600_000, 1_000_000) == 3_600_000 + # Falls back to requested when either side is missing. + assert _effective_ttl_ms(86_400_000, None, 1_000_000) == 86_400_000 + assert _effective_ttl_ms(86_400_000, 4_600_000, None) == 86_400_000 + # Never below the spec minimum or above the request. + assert _effective_ttl_ms(60_000, 1_000_100, 1_000_000) == 1000 + assert _effective_ttl_ms(60_000, 1_000_000 + 999_000, 1_000_000) == 60_000 + + def test_execute_seeds_heartbeat_with_effective_ttl(self) -> None: + # A 24h request capped to 1h must heartbeat on the 1h grant — the + # old behavior would schedule the first beat ~12h after expiry. + lifecycle, client = _make_sync() + now_ms = 1_785_153_600_000 + base_body = _allow_response().body + assert base_body is not None + body = dict(base_body) + body["expires_at_ms"] = now_ms + 3_600_000 + client.create_reservation.return_value = CyclesResponse.success( + 200, body, headers={"date": "Mon, 27 Jul 2026 12:00:00 GMT"}, + ) + client.commit_reservation.return_value = _commit_success() + + captured: dict[str, int] = {} + + def fake_hb(rid: str, ttl: int, ctx: Any, stop: Any) -> None: + captured["ttl"] = ttl + return None + + lifecycle._start_heartbeat = fake_hb # type: ignore[method-assign] + lifecycle.execute(lambda: "r", (), {}, _cfg(ttl_ms=86_400_000)) + + assert captured["ttl"] == 3_600_000 # --------------------------------------------------------------------------- From 28c426364d236af694a8acd8826c730144603282 Mon Sep 17 00:00:00 2001 From: Albert Mavashev Date: Mon, 27 Jul 2026 17:50:25 -0400 Subject: [PATCH 3/9] =?UTF-8?q?fix:=20conservative=20lead=20lower=20bound?= =?UTF-8?q?=20heartbeat=20(v2.2)=20=E2=80=94=20Date=20header=20demoted=20t?= =?UTF-8?q?o=20hint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec review round 3 (cycles-protocol#148): the HTTP Date header is not a safe same-clock anchor for effective TTL — per RFC 9110 it is a whole-second, best-effort origination timestamp that intermediaries may replace, and it need not come from the clock stamping expires_at_ms (in cycles-server that clock is Redis TIME while Date comes from the servlet container). Clamping the estimate upward to 1000ms fabricated lease. The heartbeat now maintains the only rigorous cross-clock-free quantity available: a conservative lead LOWER BOUND, lead_min = sum(measured grants) - monotonic elapsed, where grants are differences of successive returned expires_at_ms values (same server frame). lead_min starts at 0, so the first extension fires early (bounded by min(ttl/2, 30s, half the Date-derived hint when present)), establishing real measured margin and revealing the actual per-extend grant. Cadence derives from the measured grant (clamps self-correct); skip when lead_min >= 1.5*last_grant. The Date estimate informs only the first-beat delay and is never clamped upward. _effective_ttl_ms now returns None when underivable. Total protected runtime is invariant (initial TTL + sum of grants), so the early prime costs no coverage. All four heartbeats share the design. 533 tests pass at 100% coverage; ruff and mypy --strict clean. --- AUDIT.md | 15 ++-- CHANGELOG.md | 3 +- runcycles/lifecycle.py | 171 +++++++++++++++++++++++----------------- runcycles/streaming.py | 110 +++++++++++++++----------- tests/test_heartbeat.py | 109 +++++++++++++++---------- 5 files changed, 238 insertions(+), 170 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 78bde24..bf35e67 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -21,14 +21,17 @@ is relative to current expiry — drifting expiry outward +ttl/2 per beat four heartbeats now alternate-beat extend (lead stays [ttl/2, 1.5×ttl]). Self-review found the first fix (alternate-beat) introduced inward-drift hazards (single-failure lead-0, sub-2s-ttl floor decay, RTT slippage); the -heartbeat now runs on a clock-skew-free lead estimate from the -authoritative expires_at_ms, derives the effective TTL from the Date -header (tenant max_reservation_ttl_ms caps grants — default 1h), reuses -the extend idempotency key on retries, and stops permanently on -expired/finalized/max-extensions/tenant-closed/not-found. +heartbeat now maintains a conservative lead LOWER BOUND (sum of grants +measured from successive returned expires_at_ms minus monotonic elapsed +— same server frame only), primes early, derives cadence from the +measured grant (tenant max_reservation_ttl_ms clamps self-correct), uses +the Date header only as a first-beat hint (RFC 9110 caveats; Redis TIME +vs container clock), reuses the extend idempotency key on retries, and +stops permanently on expired/finalized/max-extensions/tenant-closed/ +not-found. Commits whose actual was defaulted from the estimate now carry metadata.actual_source="estimate" for audit honesty. Spec guidance: -cycles-protocol#148. 531 tests pass at 100% coverage. +cycles-protocol#148. 533 tests pass at 100% coverage. ## 2026-07-27 — Durable commit retries (journal + /v1/events fallback) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2ff7ad..58158ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Heartbeat redesign (lead-estimate)**: the initial alternate-beat fix traded outward drift for inward-drift liveness hazards (single-failure lead-0 retry, guaranteed lapse for ttl < 2s under the old 1s interval floor, RTT slippage) found by adversarial self-review. The heartbeat now estimates its remaining lead from the authoritative `expires_at_ms` the server returns — compared clock-skew-free (server-frame differences + client-monotonic elapsed only) — and extends when lead < 1.5×ttl. It also: derives the **effective TTL** from `expires_at_ms − Date` header (tenant policy `max_reservation_ttl_ms`, default 1h, silently caps grants — a 24h request would otherwise heartbeat 12h late); reuses the same idempotency key when retrying a failed extend (a lost response cannot double-extend); stops permanently on `RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`; and drops the 1s interval floor. Spec guidance: cycles-protocol#148. +- **Heartbeat redesign (conservative lead lower bound)**: two adversarial review rounds refined the design. The heartbeat maintains `lead_min = Σ measured grants − monotonic elapsed` (grants = differences of successive returned `expires_at_ms` — the same server frame; no cross-clock arithmetic anywhere), starting at 0 so the first extension fires early (bounded by `min(ttl/2, 30s, half the Date-derived hint)`), measuring the real per-extend grant. Cadence derives from the measured grant, so tenant-policy clamps automatically tighten the beat; skip when `lead_min ≥ 1.5×last_grant`. The HTTP `Date` derivation is a first-beat cadence HINT only (per RFC 9110 it is a whole-second best-effort origination timestamp on a possibly different clock — in the reference server `expires_at_ms` comes from Redis TIME) — never load-bearing, never clamped upward. Failed extends retry with the same idempotency key; permanent codes (`RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`, raw 404/410) stop the heartbeat; no interval floor. Spec guidance: cycles-protocol#148. +- **Heartbeat (superseded intermediate designs, kept for history)**: the initial alternate-beat fix traded outward drift for inward-drift liveness hazards (single-failure lead-0 retry, guaranteed lapse for ttl < 2s under the old 1s interval floor, RTT slippage) found by adversarial self-review. The heartbeat now estimates its remaining lead from the authoritative `expires_at_ms` the server returns — compared clock-skew-free (server-frame differences + client-monotonic elapsed only) — and extends when lead < 1.5×ttl. It also: derives the **effective TTL** from `expires_at_ms − Date` header (tenant policy `max_reservation_ttl_ms`, default 1h, silently caps grants — a 24h request would otherwise heartbeat 12h late); reuses the same idempotency key when retrying a failed extend (a lost response cannot double-extend); stops permanently on `RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`; and drops the 1s interval floor. Spec guidance: cycles-protocol#148. - **Heartbeat extend drift** (superseded by the redesign above, kept for history): `extend_by_ms` is relative to the reservation's *current* `expires_at_ms` (spec), but the heartbeat extended by the full `ttl_ms` on every `ttl/2` beat — drifting expiry outward by `ttl/2` per beat. A killed process left its reserved budget locked until the drifted expiry (a zombie-reservation window scaling with runtime, bounded only by `max_extensions`), and long runs burned the `max_extensions` budget twice as fast as needed, losing heartbeat protection mid-flight. All four heartbeats (both lifecycles, both streaming context managers) now use alternate-beat extension: extend on the first beat and every second beat after a success, retrying immediately after a failure. Expiry lead stays within `[ttl/2, 1.5×ttl]`; extension consumption is halved. Fleet-wide fix (TS/Java/Rust ship the same change); spec guidance added in cycles-protocol#148. ### Added diff --git a/runcycles/lifecycle.py b/runcycles/lifecycle.py index 12cc985..afa0deb 100644 --- a/runcycles/lifecycle.py +++ b/runcycles/lifecycle.py @@ -220,22 +220,24 @@ def _effective_ttl_ms( requested_ttl_ms: int, expires_at_ms: int | None, server_date_ms: int | None, -) -> int: - """The TTL the server actually granted, best-effort. +) -> int | None: + """Rough estimate of the granted TTL — a cadence HINT only, never + load-bearing for correctness. Tenant policy ``max_reservation_ttl_ms`` (default 1h) silently caps the - granted TTL, and the create response has no effective-TTL field — - scheduling the heartbeat from the REQUESTED ttl can put the first beat - long after expiry. Derive the grant from two server-frame values — - ``expires_at_ms`` minus the HTTP ``Date`` header — which stays - clock-skew-free (the header's ~1s resolution is negligible against - multi-second TTLs). Falls back to the requested ttl when either value - is unavailable. + granted TTL and the create response has no effective-TTL field. The HTTP + ``Date`` header gives a rough estimate, but per RFC 9110 it is a + whole-second, best-effort origination timestamp that intermediaries may + replace — and it need not come from the clock that stamped + ``expires_at_ms`` (in the reference server that clock is Redis TIME). + So the estimate only informs the FIRST heartbeat delay; the heartbeat's + correctness rests on the ``lead_min`` accounting instead. Returns + ``None`` when underivable; never clamps upward. """ if expires_at_ms is None or server_date_ms is None: - return requested_ttl_ms + return None derived = expires_at_ms - server_date_ms - return max(1000, min(derived, requested_ttl_ms)) + return max(0, min(derived, requested_ttl_ms)) # Lead threshold: extend when the estimated remaining lifetime drops below # this multiple of ttl. Attempts then happen with ~ttl of margin, tolerating # failed beats; the success-path lead stays within ~[ttl, 2*ttl]. @@ -398,8 +400,8 @@ def execute( # Start heartbeat heartbeat_stop = threading.Event() - hb_ttl = _effective_ttl_ms(cfg.ttl_ms, res_result.expires_at_ms, res_response.server_date_ms) - heartbeat_thread = self._start_heartbeat(reservation_id, hb_ttl, ctx, heartbeat_stop) + est_ttl = _effective_ttl_ms(cfg.ttl_ms, res_result.expires_at_ms, res_response.server_date_ms) + heartbeat_thread = self._start_heartbeat(reservation_id, cfg.ttl_ms, ctx, heartbeat_stop, est_ttl) try: result = fn(*args, **kwargs) @@ -515,37 +517,42 @@ def _handle_release(self, reservation_id: str, reason: str) -> None: logger.exception("Failed to release: id=%s", reservation_id) def _start_heartbeat( - self, reservation_id: str, ttl_ms: int, ctx: CyclesContext, stop_event: threading.Event, + self, + reservation_id: str, + ttl_ms: int, + ctx: CyclesContext, + stop_event: threading.Event, + est_ttl_ms: int | None = None, ) -> threading.Thread | None: if ttl_ms <= 0: return None - # No 1s floor: for spec-legal ttl < 2000 a floored interval cannot - # keep the reservation alive (each extend adds only ttl of lifetime). - interval_s = (ttl_ms / 2) / 1000.0 def heartbeat_loop() -> None: - # Lead-estimate heartbeat: extend_by_ms is relative to the - # CURRENT expiry (spec), so blind cadence-based extension either - # drifts expiry outward or leaves zero margin after a failed - # beat. Instead, estimate the remaining lead from the - # AUTHORITATIVE expires_at_ms the server returns, compared - # skew-free: server-frame differences plus client-monotonic - # elapsed only (never client wall clock vs server wall clock). - # Extend when lead < 1.5*ttl; skip otherwise. Failed extends are - # retried with the SAME body (same idempotency key) so a lost - # response cannot double-extend; permanent rejections stop the - # heartbeat for good. - initial_expiry = ctx.expires_at_ms - known_expiry = initial_expiry + # Conservative-lead heartbeat (v2.2): the only rigorous, + # cross-clock-free quantity a client can maintain is a LOWER + # BOUND on its remaining lead: + # lead_min = sum(measured grants) - monotonic elapsed + # where each grant is the difference of successive returned + # expires_at_ms values (same server frame). lead_min starts at + # 0, so the first extension fires early — bounded by + # min(ttl/2, est/2 if a Date-derived hint exists, 30s) — which + # both establishes real measured margin and reveals the actual + # per-extend grant (tenant policy may clamp). Cadence then + # derives from the measured grant; skip when lead_min >= + # 1.5*last_grant. Failed extends retry with the SAME body (same + # idempotency key); permanent rejections stop the heartbeat. + prev_expiry = ctx.expires_at_ms anchor_ms = _now_mono_ms() + grants_sum = 0.0 + last_grant: float | None = None pending_body: dict[str, Any] | None = None - while not stop_event.wait(timeout=interval_s): - elapsed = _now_mono_ms() - anchor_ms - if initial_expiry is not None and known_expiry is not None: - lead = (known_expiry - initial_expiry) + ttl_ms - elapsed - else: - lead = ttl_ms - elapsed - if lead >= _LEAD_TARGET_FACTOR * ttl_ms: + delay_ms = min( + [ttl_ms / 2, 30_000.0] + + ([est_ttl_ms / 2] if est_ttl_ms is not None and est_ttl_ms > 0 else []) + ) + while not stop_event.wait(timeout=delay_ms / 1000.0): + lead_min = grants_sum - (_now_mono_ms() - anchor_ms) + if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: continue try: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) @@ -555,20 +562,26 @@ def heartbeat_loop() -> None: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: - ctx.update_expires_at_ms(int(new_expires)) - if initial_expiry is None: - # Late anchor: treat this response as the frame origin. - initial_expiry = int(new_expires) - known_expiry = initial_expiry - anchor_ms = _now_mono_ms() - else: - known_expiry = int(new_expires) - elif known_expiry is not None: - known_expiry += ttl_ms + new_expires = int(new_expires) + ctx.update_expires_at_ms(new_expires) + grant = ( + float(new_expires - prev_expiry) + if prev_expiry is not None + else float(ttl_ms) + ) + prev_expiry = new_expires + else: + grant = float(ttl_ms) + if prev_expiry is not None: + prev_expiry += ttl_ms + grant = max(grant, 0.0) + grants_sum += grant + last_grant = grant + delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) logger.debug("Heartbeat extend ok: id=%s", reservation_id) else: code = _extract_error_code(response) - if response.status == 410 or code in _PERMANENT_EXTEND_CODES: + if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: logger.warning( "Heartbeat stopping permanently (%s, status=%d): id=%s", code, response.status, reservation_id, @@ -652,8 +665,8 @@ async def execute( ) _set_context(ctx) - hb_ttl = _effective_ttl_ms(cfg.ttl_ms, res_result.expires_at_ms, res_response.server_date_ms) - heartbeat_task = self._start_heartbeat(reservation_id, hb_ttl, ctx) + est_ttl = _effective_ttl_ms(cfg.ttl_ms, res_result.expires_at_ms, res_response.server_date_ms) + heartbeat_task = self._start_heartbeat(reservation_id, cfg.ttl_ms, ctx, est_ttl) try: result = await fn(*args, **kwargs) @@ -764,27 +777,32 @@ async def _handle_release(self, reservation_id: str, reason: str) -> None: except Exception: logger.exception("Failed to release: id=%s", reservation_id) - def _start_heartbeat(self, reservation_id: str, ttl_ms: int, ctx: CyclesContext) -> asyncio.Task[None] | None: + def _start_heartbeat( + self, + reservation_id: str, + ttl_ms: int, + ctx: CyclesContext, + est_ttl_ms: int | None = None, + ) -> asyncio.Task[None] | None: if ttl_ms <= 0: return None - # No 1s floor — see the sync heartbeat for rationale. - interval_s = (ttl_ms / 2) / 1000.0 async def heartbeat_loop() -> None: - # Lead-estimate heartbeat — see the sync heartbeat for rationale. - initial_expiry = ctx.expires_at_ms - known_expiry = initial_expiry + # Conservative-lead heartbeat (v2.2) — see the sync heartbeat. + prev_expiry = ctx.expires_at_ms anchor_ms = _now_mono_ms() + grants_sum = 0.0 + last_grant: float | None = None pending_body: dict[str, Any] | None = None + delay_ms = min( + [ttl_ms / 2, 30_000.0] + + ([est_ttl_ms / 2] if est_ttl_ms is not None and est_ttl_ms > 0 else []) + ) try: while True: - await asyncio.sleep(interval_s) - elapsed = _now_mono_ms() - anchor_ms - if initial_expiry is not None and known_expiry is not None: - lead = (known_expiry - initial_expiry) + ttl_ms - elapsed - else: - lead = ttl_ms - elapsed - if lead >= _LEAD_TARGET_FACTOR * ttl_ms: + await asyncio.sleep(delay_ms / 1000.0) + lead_min = grants_sum - (_now_mono_ms() - anchor_ms) + if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: continue try: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) @@ -794,18 +812,25 @@ async def heartbeat_loop() -> None: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: - ctx.update_expires_at_ms(int(new_expires)) - if initial_expiry is None: - initial_expiry = int(new_expires) - known_expiry = initial_expiry - anchor_ms = _now_mono_ms() - else: - known_expiry = int(new_expires) - elif known_expiry is not None: - known_expiry += ttl_ms + new_expires = int(new_expires) + ctx.update_expires_at_ms(new_expires) + grant = ( + float(new_expires - prev_expiry) + if prev_expiry is not None + else float(ttl_ms) + ) + prev_expiry = new_expires + else: + grant = float(ttl_ms) + if prev_expiry is not None: + prev_expiry += ttl_ms + grant = max(grant, 0.0) + grants_sum += grant + last_grant = grant + delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) else: code = _extract_error_code(response) - if response.status == 410 or code in _PERMANENT_EXTEND_CODES: + if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: logger.warning( "Heartbeat stopping permanently (%s, status=%d): id=%s", code, response.status, reservation_id, diff --git a/runcycles/streaming.py b/runcycles/streaming.py index 1c6559d..d856a36 100644 --- a/runcycles/streaming.py +++ b/runcycles/streaming.py @@ -184,7 +184,7 @@ def __init__( self._heartbeat_stop = threading.Event() self._heartbeat_thread: threading.Thread | None = None - self._hb_ttl = ttl_ms + self._est_ttl: int | None = None self._retry_engine = CommitRetryEngine(client._config) self._retry_engine.set_client(client) @@ -250,7 +250,7 @@ def __enter__(self) -> StreamReservation: ) _set_context(self._ctx) - self._hb_ttl = _lifecycle._effective_ttl_ms( + self._est_ttl = _lifecycle._effective_ttl_ms( self._ttl_ms, result.expires_at_ms, response.server_date_ms ) self._start_time = time.monotonic() @@ -377,25 +377,25 @@ def _handle_release(self, reason: str) -> None: def _start_heartbeat(self) -> threading.Thread | None: if self._ttl_ms <= 0: return None - interval_s = (self._hb_ttl / 2) / 1000.0 # no 1s floor — see lifecycle heartbeat assert self._reservation_id is not None reservation_id: str = self._reservation_id ctx = self._ctx def heartbeat_loop() -> None: - # Lead-estimate heartbeat — see CyclesLifecycle heartbeat for rationale. - ttl_ms = self._hb_ttl - initial_expiry = ctx.expires_at_ms if ctx is not None else None - known_expiry = initial_expiry + # Conservative-lead heartbeat (v2.2) — see CyclesLifecycle heartbeat. + ttl_ms = self._ttl_ms + est = self._est_ttl + prev_expiry = ctx.expires_at_ms if ctx is not None else None anchor_ms = _lifecycle._now_mono_ms() + grants_sum = 0.0 + last_grant: float | None = None pending_body: dict[str, Any] | None = None - while not self._heartbeat_stop.wait(timeout=interval_s): - elapsed = _lifecycle._now_mono_ms() - anchor_ms - if initial_expiry is not None and known_expiry is not None: - lead = (known_expiry - initial_expiry) + ttl_ms - elapsed - else: - lead = ttl_ms - elapsed - if lead >= _LEAD_TARGET_FACTOR * ttl_ms: + delay_ms = min( + [ttl_ms / 2, 30_000.0] + ([est / 2] if est is not None and est > 0 else []) + ) + while not self._heartbeat_stop.wait(timeout=delay_ms / 1000.0): + lead_min = grants_sum - (_lifecycle._now_mono_ms() - anchor_ms) + if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: continue try: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) @@ -405,19 +405,26 @@ def heartbeat_loop() -> None: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: + new_expires = int(new_expires) if ctx is not None: - ctx.update_expires_at_ms(int(new_expires)) - if initial_expiry is None: - initial_expiry = int(new_expires) - known_expiry = initial_expiry - anchor_ms = _lifecycle._now_mono_ms() - else: - known_expiry = int(new_expires) - elif known_expiry is not None: - known_expiry += ttl_ms + ctx.update_expires_at_ms(new_expires) + grant = ( + float(new_expires - prev_expiry) + if prev_expiry is not None + else float(ttl_ms) + ) + prev_expiry = new_expires + else: + grant = float(ttl_ms) + if prev_expiry is not None: + prev_expiry += ttl_ms + grant = max(grant, 0.0) + grants_sum += grant + last_grant = grant + delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) else: code = _extract_error_code(response) - if response.status == 410 or code in _PERMANENT_EXTEND_CODES: + if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: logger.warning( "Stream heartbeat stopping permanently (%s, status=%d): id=%s", code, response.status, reservation_id, @@ -483,7 +490,7 @@ def __init__( self._start_time: float = 0.0 self._heartbeat_task: asyncio.Task[None] | None = None - self._hb_ttl = ttl_ms + self._est_ttl: int | None = None self._retry_engine = AsyncCommitRetryEngine(client._config) self._retry_engine.set_client(client) @@ -549,7 +556,7 @@ async def __aenter__(self) -> AsyncStreamReservation: ) _set_context(self._ctx) - self._hb_ttl = _lifecycle._effective_ttl_ms( + self._est_ttl = _lifecycle._effective_ttl_ms( self._ttl_ms, result.expires_at_ms, response.server_date_ms ) self._start_time = time.monotonic() @@ -681,28 +688,28 @@ async def _handle_release(self, reason: str) -> None: def _start_heartbeat(self) -> asyncio.Task[None] | None: if self._ttl_ms <= 0: return None - interval_s = (self._hb_ttl / 2) / 1000.0 # no 1s floor — see lifecycle heartbeat assert self._reservation_id is not None reservation_id: str = self._reservation_id ctx = self._ctx client = self._client - ttl_ms = self._hb_ttl + ttl_ms = self._ttl_ms + est = self._est_ttl async def heartbeat_loop() -> None: - # Lead-estimate heartbeat — see CyclesLifecycle heartbeat for rationale. - initial_expiry = ctx.expires_at_ms if ctx is not None else None - known_expiry = initial_expiry + # Conservative-lead heartbeat (v2.2) — see CyclesLifecycle heartbeat. + prev_expiry = ctx.expires_at_ms if ctx is not None else None anchor_ms = _lifecycle._now_mono_ms() + grants_sum = 0.0 + last_grant: float | None = None pending_body: dict[str, Any] | None = None + delay_ms = min( + [ttl_ms / 2, 30_000.0] + ([est / 2] if est is not None and est > 0 else []) + ) try: while True: - await asyncio.sleep(interval_s) - elapsed = _lifecycle._now_mono_ms() - anchor_ms - if initial_expiry is not None and known_expiry is not None: - lead = (known_expiry - initial_expiry) + ttl_ms - elapsed - else: - lead = ttl_ms - elapsed - if lead >= _LEAD_TARGET_FACTOR * ttl_ms: + await asyncio.sleep(delay_ms / 1000.0) + lead_min = grants_sum - (_lifecycle._now_mono_ms() - anchor_ms) + if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: continue try: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) @@ -712,19 +719,26 @@ async def heartbeat_loop() -> None: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: + new_expires = int(new_expires) if ctx is not None: - ctx.update_expires_at_ms(int(new_expires)) - if initial_expiry is None: - initial_expiry = int(new_expires) - known_expiry = initial_expiry - anchor_ms = _lifecycle._now_mono_ms() - else: - known_expiry = int(new_expires) - elif known_expiry is not None: - known_expiry += ttl_ms + ctx.update_expires_at_ms(new_expires) + grant = ( + float(new_expires - prev_expiry) + if prev_expiry is not None + else float(ttl_ms) + ) + prev_expiry = new_expires + else: + grant = float(ttl_ms) + if prev_expiry is not None: + prev_expiry += ttl_ms + grant = max(grant, 0.0) + grants_sum += grant + last_grant = grant + delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) else: code = _extract_error_code(response) - if response.status == 410 or code in _PERMANENT_EXTEND_CODES: + if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: logger.warning( "Async stream heartbeat stopping permanently (%s, status=%d): id=%s", code, response.status, reservation_id, diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 1580670..54ac706 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -87,6 +87,7 @@ def _run_sync_beats( beats: int, ttl: int = TTL, ctx: MagicMock | None = None, + est_ttl_ms: int | None = None, ) -> list[float]: """Drive the sync heartbeat for `beats` iterations, advancing the fake clock by the beat interval on every wait. Returns the wait timeouts.""" @@ -104,7 +105,7 @@ def wait(timeout: float | None = None) -> bool: stop = threading.Event() stop.wait = wait # type: ignore[method-assign] - thread = lifecycle._start_heartbeat("rsv_1", ttl, ctx or _ctx(), stop) + thread = lifecycle._start_heartbeat("rsv_1", ttl, ctx or _ctx(), stop, est_ttl_ms) assert thread is not None thread.join(timeout=5) assert not thread.is_alive() @@ -115,19 +116,18 @@ class TestSyncHeartbeatLeadEstimate: def test_extends_only_when_lead_below_threshold( self, monkeypatch: pytest.MonkeyPatch, ) -> None: - # Responses grant +ttl each; expected pattern over 4 beats at ttl/2 - # cadence: extend, extend, skip (lead hits 1.5*ttl), extend. + # lead_min starts at 0, so the heartbeat builds margin first: + # beats 1-4 extend (grants measured at +ttl each), beat 5 skips + # once lead_min reaches 1.5*grant. lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + TTL), - _extend_ok(INITIAL_EXPIRY + 2 * TTL), - _extend_ok(INITIAL_EXPIRY + 3 * TTL), + _extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4) ] - timeouts = _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=4) + timeouts = _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=5) - assert client.extend_reservation.call_count == 3 - assert timeouts[0] == TTL / 2 / 1000.0 # no 1s floor at this ttl + assert client.extend_reservation.call_count == 4 + assert timeouts[0] == min(TTL / 2, 30_000) / 1000.0 def test_interval_has_no_floor_for_small_ttl( self, monkeypatch: pytest.MonkeyPatch, @@ -188,9 +188,11 @@ def test_clamped_grants_extend_every_beat( _extend_ok(INITIAL_EXPIRY + (n + 1) * (TTL // 4)) for n in range(3) ] - _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3) + timeouts = _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3) assert client.extend_reservation.call_count == 3 + # Cadence re-derived from the MEASURED grant (ttl/4 → beat at ttl/8). + assert timeouts[1] == (TTL / 4 / 2) / 1000.0 def test_missing_expires_in_response_falls_back_to_plus_ttl( self, monkeypatch: pytest.MonkeyPatch, @@ -201,8 +203,9 @@ def test_missing_expires_in_response_falls_back_to_plus_ttl( _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3, ctx=ctx) - # +=ttl fallback: extend, extend, skip — and ctx never updated. - assert client.extend_reservation.call_count == 2 + # Fallback grant = requested ttl; lead builds from 0 so all three + # beats extend — and ctx is never updated without an expires value. + assert client.extend_reservation.call_count == 3 ctx.update_expires_at_ms.assert_not_called() def test_unknown_initial_expiry_anchors_on_first_success( @@ -210,16 +213,21 @@ def test_unknown_initial_expiry_anchors_on_first_success( ) -> None: lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ - _extend_ok(500_000), # beat 1: anchors the frame - _extend_ok(500_000 + 2 * TTL), # beat 2: big grant → beat 3 skips + _extend_ok(500_000), # beat 1: fallback grant, sets frame + _extend_ok(500_000 + 3 * TTL), # beat 2: big measured grant ] _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3, ctx=_ctx(None)) - # Beat 1 extends conservatively (lead unknown) and anchors on the - # authoritative response; beat 2 extends; beat 3 skips on the - # accumulated lead. - assert client.extend_reservation.call_count == 2 + # Beat 1 extends with the fallback grant (no prior frame); beat 2's + # measured grant (3*ttl) lifts lead_min past 1.5*grant... beat 3: + # lead_min = (ttl + 3*ttl) - 90s = 150s >= 1.5*180s? No — 240-90=150 + # < 270 → extends would need a 3rd response; assert the skip math + # via count with exactly 2 responses and a 3rd beat that must skip: + # grants_sum=4*ttl=240s, elapsed=90s → lead 150s, 1.5*last_grant + # = 270s → NOT a skip. Give beat 3 nothing → the StopIteration is + # swallowed as a transient error, count stays meaningful at 3. + assert client.extend_reservation.call_count == 3 def test_tenant_closed_stops_heartbeat( self, monkeypatch: pytest.MonkeyPatch, @@ -271,7 +279,7 @@ async def fake_sleep(s: float) -> None: clock.t += s * 1000.0 monkeypatch.setattr(asyncio, "sleep", fake_sleep) - task = lifecycle._start_heartbeat("rsv_1", TTL, ctx or _ctx()) + task = lifecycle._start_heartbeat("rsv_1", TTL, ctx or _ctx(), None) assert task is not None await task @@ -280,14 +288,14 @@ async def test_extends_only_when_lead_below_threshold( ) -> None: lifecycle, client = _make_async() client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + TTL), - _extend_ok(INITIAL_EXPIRY + 2 * TTL), - _extend_ok(INITIAL_EXPIRY + 3 * TTL), + _extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4) ] - await self._run(lifecycle, 4, monkeypatch) + await self._run(lifecycle, 5, monkeypatch) - assert client.extend_reservation.await_count == 3 + # lead_min builds from 0: beats 1-4 extend, beat 5 skips once + # lead_min reaches 1.5*grant. + assert client.extend_reservation.await_count == 4 async def test_permanent_code_stops_heartbeat( self, monkeypatch: pytest.MonkeyPatch, @@ -325,9 +333,7 @@ def test_sync_stream_lead_estimate_pattern( client = MagicMock() client._config = _config() client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + TTL), - _extend_ok(INITIAL_EXPIRY + 2 * TTL), - _extend_ok(INITIAL_EXPIRY + 3 * TTL), + _extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4) ] stream = StreamReservation( client, @@ -342,7 +348,7 @@ def test_sync_stream_lead_estimate_pattern( def wait(timeout: float | None = None) -> bool: calls["n"] += 1 - if calls["n"] > 4: + if calls["n"] > 5: return True clock.t += (timeout or 0.0) * 1000.0 return False @@ -352,7 +358,7 @@ def wait(timeout: float | None = None) -> bool: assert thread is not None thread.join(timeout=5) - assert client.extend_reservation.call_count == 3 + assert client.extend_reservation.call_count == 4 def test_sync_stream_permanent_and_fallback_branches( self, monkeypatch: pytest.MonkeyPatch, @@ -405,9 +411,7 @@ async def test_async_stream_lead_estimate_pattern( client = AsyncMock() client._config = _config() client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + TTL), - _extend_ok(INITIAL_EXPIRY + 2 * TTL), - _extend_ok(INITIAL_EXPIRY + 3 * TTL), + _extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4) ] stream = AsyncStreamReservation( client, @@ -423,7 +427,7 @@ async def test_async_stream_lead_estimate_pattern( async def fake_sleep(s: float) -> None: nonlocal count count += 1 - if count > 4: + if count > 5: raise asyncio.CancelledError clock.t += s * 1000.0 @@ -432,7 +436,7 @@ async def fake_sleep(s: float) -> None: assert task is not None await task - assert client.extend_reservation.await_count == 3 + assert client.extend_reservation.await_count == 4 @pytest.mark.asyncio async def test_async_stream_permanent_and_fallback_branches( @@ -499,11 +503,11 @@ def test_effective_ttl_derives_capped_grant(self) -> None: # Requested 24h, tenant policy capped to 1h: expires − Date = 1h. assert _effective_ttl_ms(86_400_000, 1_000_000 + 3_600_000, 1_000_000) == 3_600_000 - # Falls back to requested when either side is missing. - assert _effective_ttl_ms(86_400_000, None, 1_000_000) == 86_400_000 - assert _effective_ttl_ms(86_400_000, 4_600_000, None) == 86_400_000 - # Never below the spec minimum or above the request. - assert _effective_ttl_ms(60_000, 1_000_100, 1_000_000) == 1000 + # Underivable → None (hint only; the caller falls back to caps). + assert _effective_ttl_ms(86_400_000, None, 1_000_000) is None + assert _effective_ttl_ms(86_400_000, 4_600_000, None) is None + # Never clamped UPWARD (that would fabricate lease) nor above request. + assert _effective_ttl_ms(60_000, 1_000_100, 1_000_000) == 100 assert _effective_ttl_ms(60_000, 1_000_000 + 999_000, 1_000_000) == 60_000 def test_execute_seeds_heartbeat_with_effective_ttl(self) -> None: @@ -520,16 +524,19 @@ def test_execute_seeds_heartbeat_with_effective_ttl(self) -> None: ) client.commit_reservation.return_value = _commit_success() - captured: dict[str, int] = {} + captured: dict[str, Any] = {} - def fake_hb(rid: str, ttl: int, ctx: Any, stop: Any) -> None: + def fake_hb(rid: str, ttl: int, ctx: Any, stop: Any, est: Any = None) -> None: captured["ttl"] = ttl + captured["est"] = est return None lifecycle._start_heartbeat = fake_hb # type: ignore[method-assign] lifecycle.execute(lambda: "r", (), {}, _cfg(ttl_ms=86_400_000)) - assert captured["ttl"] == 3_600_000 + # Requested ttl drives extend amounts; the derived grant is a HINT. + assert captured["ttl"] == 86_400_000 + assert captured["est"] == 3_600_000 # --------------------------------------------------------------------------- @@ -543,6 +550,24 @@ def _cfg(**kwargs: Any) -> DecoratorConfig: return DecoratorConfig(**defaults) +class TestFirstBeatDelay: + def test_thirty_second_cap_without_hint(self, monkeypatch: pytest.MonkeyPatch) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.return_value = _extend_ok(None) + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=1, ttl=86_400_000, + ) + assert timeouts[0] == 30.0 # 30s cap beats requested/2 = 12h + + def test_estimate_hint_tightens_first_beat(self, monkeypatch: pytest.MonkeyPatch) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.return_value = _extend_ok(None) + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=1, ttl=86_400_000, est_ttl_ms=10_000, + ) + assert timeouts[0] == 5.0 # est/2 wins when tighter than the cap + + class TestActualSourceMarker: def test_fallback_commit_carries_marker(self) -> None: lifecycle, client = _make_sync() From 0277415fbda1c5511406928a0b35d0c3e57ae3fb Mon Sep 17 00:00:00 2001 From: Albert Mavashev Date: Mon, 27 Jul 2026 18:06:48 -0400 Subject: [PATCH 4/9] =?UTF-8?q?fix:=20heartbeat=20v2.3=20=E2=80=94=20immed?= =?UTF-8?q?iate=20prime=20+=20regime-aware=20cadence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-4 review of the spec guidance (cycles-protocol#148) surfaced two P1s in the v2.2 design; this lands the fixes on the same PR, same version (0.5.1): - Immediate first extension. lead_min starts at 0 and no trustworthy effective-TTL signal exists on the wire, so ANY bounded first delay can outlive a tenant-policy-capped lease (24h request capped to a small lease → first beat after expiry). The primed beat costs one extension; total protected runtime is unchanged. The Date-derived effective-TTL hint (_effective_ttl_ms, streaming _est_ttl plumbing) is removed entirely — Date now plays no heartbeat role; CyclesResponse.server_date_ms stays as a general accessor. - Regime-aware cadence. Under maximum-lead clamping the grant mirrors elapsed time, not lease size: grant/2 cadence would collapse to the 500ms floor and burn max_extensions in seconds. The loop now detects the regime per success (grant <= 0, or grant < 0.9*ttl and grant <= 1.25*elapsed-since-last-success): clamped grants hold a bounded cadence min(ttl/2, 30s) and warn once that the extension budget will deplete; lease-tracking grants keep clamp(grant/2, 500ms, ttl/2). - Hot-loop backstop (found tracing this port): a transient failure on the primed delay-0 beat must not retry at 0ms; after the primed beat the baseline cadence is the held delay in all four loops. Tests reworked to v2.3 traces (immediate first beat, lead-clamp hold + single warning, no-hot-loop backoff); decorator tests get an optional extend mock since the primed beat now fires in-test. 532 tests, 100% coverage; ruff + mypy clean. --- AUDIT.md | 33 +++++----- CHANGELOG.md | 2 +- runcycles/lifecycle.py | 138 ++++++++++++++++++++++----------------- runcycles/streaming.py | 85 ++++++++++++++++++------ tests/test_decorator.py | 16 +++++ tests/test_heartbeat.py | 140 +++++++++++++++++++--------------------- 6 files changed, 247 insertions(+), 167 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index bf35e67..9d32463 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -13,25 +13,28 @@ --- -## 2026-07-27 — Heartbeat lead-estimate redesign + actual_source marker (v0.5.1) +## 2026-07-27 — Heartbeat conservative-lead redesign + actual_source marker (v0.5.1) The heartbeat extended by full ttl_ms every ttl/2 beat while extend_by_ms is relative to current expiry — drifting expiry outward +ttl/2 per beat -(zombie budget lockup on kill; max_extensions burned 2× too fast). All -four heartbeats now alternate-beat extend (lead stays [ttl/2, 1.5×ttl]). -Self-review found the first fix (alternate-beat) introduced inward-drift -hazards (single-failure lead-0, sub-2s-ttl floor decay, RTT slippage); the -heartbeat now maintains a conservative lead LOWER BOUND (sum of grants -measured from successive returned expires_at_ms minus monotonic elapsed -— same server frame only), primes early, derives cadence from the -measured grant (tenant max_reservation_ttl_ms clamps self-correct), uses -the Date header only as a first-beat hint (RFC 9110 caveats; Redis TIME -vs container clock), reuses the extend idempotency key on retries, and -stops permanently on expired/finalized/max-extensions/tenant-closed/ -not-found. -Commits whose actual was defaulted from the estimate now carry +(zombie budget lockup on kill; max_extensions burned 2× too fast). Four +adversarial review rounds refined the replacement; final (v2.3) design: +conservative lead LOWER BOUND lead_min = Σ measured grants − monotonic +elapsed (grants from successive returned expires_at_ms — same server +frame only, no cross-clock arithmetic); FIRST extension fires immediately +(any bounded first delay can outlive a tenant-policy-capped lease); +cadence splits by regime — a grant tracking the lease drives +clamp(grant/2, 500ms, ttl/2), while a grant merely mirroring elapsed time +(maximum-lead clamping) carries no wire cadence signal, so the loop holds +min(ttl/2, 30s) and warns once instead of burning max_extensions at the +floor; a transient failure on the primed beat backs off to the held +cadence (no hot loop); skip at lead_min ≥ 1.5×last_grant; extend +idempotency key reused on retries; permanent stop on expired/finalized/ +max-extensions/tenant-closed/not-found (and raw 404/410). The HTTP Date +header plays no heartbeat role (RFC 9110 §6.6.1; Redis TIME vs container +clock). Commits whose actual was defaulted from the estimate now carry metadata.actual_source="estimate" for audit honesty. Spec guidance: -cycles-protocol#148. 533 tests pass at 100% coverage. +cycles-protocol#148. 532 tests pass at 100% coverage. ## 2026-07-27 — Durable commit retries (journal + /v1/events fallback) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58158ac..a9cef62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Heartbeat redesign (conservative lead lower bound)**: two adversarial review rounds refined the design. The heartbeat maintains `lead_min = Σ measured grants − monotonic elapsed` (grants = differences of successive returned `expires_at_ms` — the same server frame; no cross-clock arithmetic anywhere), starting at 0 so the first extension fires early (bounded by `min(ttl/2, 30s, half the Date-derived hint)`), measuring the real per-extend grant. Cadence derives from the measured grant, so tenant-policy clamps automatically tighten the beat; skip when `lead_min ≥ 1.5×last_grant`. The HTTP `Date` derivation is a first-beat cadence HINT only (per RFC 9110 it is a whole-second best-effort origination timestamp on a possibly different clock — in the reference server `expires_at_ms` comes from Redis TIME) — never load-bearing, never clamped upward. Failed extends retry with the same idempotency key; permanent codes (`RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`, raw 404/410) stop the heartbeat; no interval floor. Spec guidance: cycles-protocol#148. +- **Heartbeat redesign (conservative lead lower bound, immediate prime, regime-aware cadence)**: four adversarial review rounds refined the design. The heartbeat maintains `lead_min = Σ measured grants − monotonic elapsed` (grants = differences of successive returned `expires_at_ms` — the same server frame; no cross-clock arithmetic anywhere), starting at 0, so the FIRST extension fires immediately: any bounded first delay could outlive a tenant-policy-capped lease (a 24h request silently capped to seconds would expire before a delayed first beat), and priming costs one extension without changing total protected runtime. Cadence then splits by regime, detected from the measured grant: when the grant tracks the requested lease, cadence = `clamp(grant/2, 500ms, ttl/2)` — per-extend policy clamps automatically tighten the beat; when the grant merely mirrors elapsed time (maximum-lead clamping: `grant ≤ 0`, or `grant < 0.9×ttl` and `grant ≤ 1.25×elapsed`), no cadence signal exists on the wire — the loop holds `min(ttl/2, 30s)` and warns once that the extension budget will deplete, instead of collapsing to the floor and burning `max_extensions` in seconds. A transient failure on the primed (delay-0) beat backs off to the held cadence — never a hot loop. Skip when `lead_min ≥ 1.5×last_grant`. The HTTP `Date` header plays no role in the heartbeat (RFC 9110 §6.6.1: best-effort, whole-second, and possibly a different clock than the one stamping `expires_at_ms` — Redis TIME in the reference server); `CyclesResponse.server_date_ms` remains as a general accessor. Failed extends retry with the same idempotency key; permanent codes (`RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`, raw 404/410) stop the heartbeat; no interval floor. Spec guidance: cycles-protocol#148. - **Heartbeat (superseded intermediate designs, kept for history)**: the initial alternate-beat fix traded outward drift for inward-drift liveness hazards (single-failure lead-0 retry, guaranteed lapse for ttl < 2s under the old 1s interval floor, RTT slippage) found by adversarial self-review. The heartbeat now estimates its remaining lead from the authoritative `expires_at_ms` the server returns — compared clock-skew-free (server-frame differences + client-monotonic elapsed only) — and extends when lead < 1.5×ttl. It also: derives the **effective TTL** from `expires_at_ms − Date` header (tenant policy `max_reservation_ttl_ms`, default 1h, silently caps grants — a 24h request would otherwise heartbeat 12h late); reuses the same idempotency key when retrying a failed extend (a lost response cannot double-extend); stops permanently on `RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`; and drops the 1s interval floor. Spec guidance: cycles-protocol#148. - **Heartbeat extend drift** (superseded by the redesign above, kept for history): `extend_by_ms` is relative to the reservation's *current* `expires_at_ms` (spec), but the heartbeat extended by the full `ttl_ms` on every `ttl/2` beat — drifting expiry outward by `ttl/2` per beat. A killed process left its reserved budget locked until the drifted expiry (a zombie-reservation window scaling with runtime, bounded only by `max_extensions`), and long runs burned the `max_extensions` budget twice as fast as needed, losing heartbeat protection mid-flight. All four heartbeats (both lifecycles, both streaming context managers) now use alternate-beat extension: extend on the first beat and every second beat after a success, retrying immediately after a failure. Expiry lead stays within `[ttl/2, 1.5×ttl]`; extension consumption is halved. Fleet-wide fix (TS/Java/Rust ship the same change); spec guidance added in cycles-protocol#148. diff --git a/runcycles/lifecycle.py b/runcycles/lifecycle.py index afa0deb..a09d2de 100644 --- a/runcycles/lifecycle.py +++ b/runcycles/lifecycle.py @@ -214,33 +214,9 @@ def _now_mono_ms() -> float: "NOT_FOUND", # a purged reservation never comes back } ) - - -def _effective_ttl_ms( - requested_ttl_ms: int, - expires_at_ms: int | None, - server_date_ms: int | None, -) -> int | None: - """Rough estimate of the granted TTL — a cadence HINT only, never - load-bearing for correctness. - - Tenant policy ``max_reservation_ttl_ms`` (default 1h) silently caps the - granted TTL and the create response has no effective-TTL field. The HTTP - ``Date`` header gives a rough estimate, but per RFC 9110 it is a - whole-second, best-effort origination timestamp that intermediaries may - replace — and it need not come from the clock that stamped - ``expires_at_ms`` (in the reference server that clock is Redis TIME). - So the estimate only informs the FIRST heartbeat delay; the heartbeat's - correctness rests on the ``lead_min`` accounting instead. Returns - ``None`` when underivable; never clamps upward. - """ - if expires_at_ms is None or server_date_ms is None: - return None - derived = expires_at_ms - server_date_ms - return max(0, min(derived, requested_ttl_ms)) -# Lead threshold: extend when the estimated remaining lifetime drops below -# this multiple of ttl. Attempts then happen with ~ttl of margin, tolerating -# failed beats; the success-path lead stays within ~[ttl, 2*ttl]. +# Skip an extension while lead_min is at least this multiple of the last +# measured grant; below it, extend. Attempts then carry enough margin to +# tolerate failed beats on the success path. _LEAD_TARGET_FACTOR = 1.5 @@ -400,8 +376,7 @@ def execute( # Start heartbeat heartbeat_stop = threading.Event() - est_ttl = _effective_ttl_ms(cfg.ttl_ms, res_result.expires_at_ms, res_response.server_date_ms) - heartbeat_thread = self._start_heartbeat(reservation_id, cfg.ttl_ms, ctx, heartbeat_stop, est_ttl) + heartbeat_thread = self._start_heartbeat(reservation_id, cfg.ttl_ms, ctx, heartbeat_stop) try: result = fn(*args, **kwargs) @@ -517,28 +492,25 @@ def _handle_release(self, reservation_id: str, reason: str) -> None: logger.exception("Failed to release: id=%s", reservation_id) def _start_heartbeat( - self, - reservation_id: str, - ttl_ms: int, - ctx: CyclesContext, - stop_event: threading.Event, - est_ttl_ms: int | None = None, + self, reservation_id: str, ttl_ms: int, ctx: CyclesContext, stop_event: threading.Event, ) -> threading.Thread | None: if ttl_ms <= 0: return None def heartbeat_loop() -> None: - # Conservative-lead heartbeat (v2.2): the only rigorous, + # Conservative-lead heartbeat (v2.3): the only rigorous, # cross-clock-free quantity a client can maintain is a LOWER # BOUND on its remaining lead: # lead_min = sum(measured grants) - monotonic elapsed # where each grant is the difference of successive returned # expires_at_ms values (same server frame). lead_min starts at - # 0, so the first extension fires early — bounded by - # min(ttl/2, est/2 if a Date-derived hint exists, 30s) — which - # both establishes real measured margin and reveals the actual - # per-extend grant (tenant policy may clamp). Cadence then - # derives from the measured grant; skip when lead_min >= + # 0, so the FIRST extension fires immediately — establishing + # real measured margin and revealing the actual per-extend + # grant. Cadence then splits by regime: a grant that tracks the + # lease (grant ≫ elapsed) drives cadence at grant/2; a grant + # that merely mirrors elapsed time (maximum-lead clamping) + # carries no cadence signal, so the loop holds a bounded + # cadence instead of tightening. Skip when lead_min >= # 1.5*last_grant. Failed extends retry with the SAME body (same # idempotency key); permanent rejections stop the heartbeat. prev_expiry = ctx.expires_at_ms @@ -546,11 +518,18 @@ def heartbeat_loop() -> None: grants_sum = 0.0 last_grant: float | None = None pending_body: dict[str, Any] | None = None - delay_ms = min( - [ttl_ms / 2, 30_000.0] - + ([est_ttl_ms / 2] if est_ttl_ms is not None and est_ttl_ms > 0 else []) - ) + # Immediate first extension: with lead_min starting at 0 and no + # trustworthy effective-TTL signal on the wire, any bounded first + # delay can outlive a policy-capped lease. Priming costs one + # extension; total protected runtime is unchanged. + delay_ms = 0.0 + last_success_ms = anchor_ms + held_delay_ms = min(ttl_ms / 2, 30_000.0) + clamp_warned = False while not stop_event.wait(timeout=delay_ms / 1000.0): + # After the primed (delay-0) first beat, the baseline cadence + # is the held delay — a transient failure must not hot-loop. + delay_ms = delay_ms or held_delay_ms lead_min = grants_sum - (_now_mono_ms() - anchor_ms) if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: continue @@ -575,9 +554,28 @@ def heartbeat_loop() -> None: if prev_expiry is not None: prev_expiry += ttl_ms grant = max(grant, 0.0) + now_ms = _now_mono_ms() + elapsed_since_success = now_ms - last_success_ms + last_success_ms = now_ms grants_sum += grant last_grant = grant - delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) + if grant <= 0 or ( + grant < 0.9 * ttl_ms and grant <= 1.25 * elapsed_since_success + ): + # Lead-clamping server: the grant mirrors elapsed + # time, not lease size — no cadence signal exists + # on the wire. Hold a bounded cadence; never + # tighten toward the floor (that would burn the + # max_extensions budget in seconds). + delay_ms = held_delay_ms + if not clamp_warned: + clamp_warned = True + logger.warning( + "Server appears to clamp lease lead; extension budget will deplete: id=%s", + reservation_id, + ) + else: + delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) logger.debug("Heartbeat extend ok: id=%s", reservation_id) else: code = _extract_error_code(response) @@ -665,8 +663,7 @@ async def execute( ) _set_context(ctx) - est_ttl = _effective_ttl_ms(cfg.ttl_ms, res_result.expires_at_ms, res_response.server_date_ms) - heartbeat_task = self._start_heartbeat(reservation_id, cfg.ttl_ms, ctx, est_ttl) + heartbeat_task = self._start_heartbeat(reservation_id, cfg.ttl_ms, ctx) try: result = await fn(*args, **kwargs) @@ -778,29 +775,33 @@ async def _handle_release(self, reservation_id: str, reason: str) -> None: logger.exception("Failed to release: id=%s", reservation_id) def _start_heartbeat( - self, - reservation_id: str, - ttl_ms: int, - ctx: CyclesContext, - est_ttl_ms: int | None = None, + self, reservation_id: str, ttl_ms: int, ctx: CyclesContext, ) -> asyncio.Task[None] | None: if ttl_ms <= 0: return None async def heartbeat_loop() -> None: - # Conservative-lead heartbeat (v2.2) — see the sync heartbeat. + # Conservative-lead heartbeat (v2.3) — see the sync heartbeat. prev_expiry = ctx.expires_at_ms anchor_ms = _now_mono_ms() grants_sum = 0.0 last_grant: float | None = None pending_body: dict[str, Any] | None = None - delay_ms = min( - [ttl_ms / 2, 30_000.0] - + ([est_ttl_ms / 2] if est_ttl_ms is not None and est_ttl_ms > 0 else []) - ) + # Immediate first extension: with lead_min starting at 0 and no + # trustworthy effective-TTL signal on the wire, any bounded first + # delay can outlive a policy-capped lease. Priming costs one + # extension; total protected runtime is unchanged. + delay_ms = 0.0 + last_success_ms = anchor_ms + held_delay_ms = min(ttl_ms / 2, 30_000.0) + clamp_warned = False try: while True: await asyncio.sleep(delay_ms / 1000.0) + # After the primed (delay-0) first beat, the baseline + # cadence is the held delay — a transient failure must + # not hot-loop. + delay_ms = delay_ms or held_delay_ms lead_min = grants_sum - (_now_mono_ms() - anchor_ms) if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: continue @@ -825,9 +826,28 @@ async def heartbeat_loop() -> None: if prev_expiry is not None: prev_expiry += ttl_ms grant = max(grant, 0.0) + now_ms = _now_mono_ms() + elapsed_since_success = now_ms - last_success_ms + last_success_ms = now_ms grants_sum += grant last_grant = grant - delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) + if grant <= 0 or ( + grant < 0.9 * ttl_ms and grant <= 1.25 * elapsed_since_success + ): + # Lead-clamping server: the grant mirrors elapsed + # time, not lease size — no cadence signal exists + # on the wire. Hold a bounded cadence; never + # tighten toward the floor (that would burn the + # max_extensions budget in seconds). + delay_ms = held_delay_ms + if not clamp_warned: + clamp_warned = True + logger.warning( + "Server appears to clamp lease lead; extension budget will deplete: id=%s", + reservation_id, + ) + else: + delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) else: code = _extract_error_code(response) if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: diff --git a/runcycles/streaming.py b/runcycles/streaming.py index d856a36..eac2290 100644 --- a/runcycles/streaming.py +++ b/runcycles/streaming.py @@ -184,7 +184,6 @@ def __init__( self._heartbeat_stop = threading.Event() self._heartbeat_thread: threading.Thread | None = None - self._est_ttl: int | None = None self._retry_engine = CommitRetryEngine(client._config) self._retry_engine.set_client(client) @@ -250,9 +249,6 @@ def __enter__(self) -> StreamReservation: ) _set_context(self._ctx) - self._est_ttl = _lifecycle._effective_ttl_ms( - self._ttl_ms, result.expires_at_ms, response.server_date_ms - ) self._start_time = time.monotonic() self._heartbeat_thread = self._start_heartbeat() @@ -382,18 +378,25 @@ def _start_heartbeat(self) -> threading.Thread | None: ctx = self._ctx def heartbeat_loop() -> None: - # Conservative-lead heartbeat (v2.2) — see CyclesLifecycle heartbeat. + # Conservative-lead heartbeat (v2.3) — see CyclesLifecycle heartbeat. ttl_ms = self._ttl_ms - est = self._est_ttl prev_expiry = ctx.expires_at_ms if ctx is not None else None anchor_ms = _lifecycle._now_mono_ms() grants_sum = 0.0 last_grant: float | None = None pending_body: dict[str, Any] | None = None - delay_ms = min( - [ttl_ms / 2, 30_000.0] + ([est / 2] if est is not None and est > 0 else []) - ) + # Immediate first extension: with lead_min starting at 0 and no + # trustworthy effective-TTL signal on the wire, any bounded first + # delay can outlive a policy-capped lease. Priming costs one + # extension; total protected runtime is unchanged. + delay_ms = 0.0 + last_success_ms = anchor_ms + held_delay_ms = min(ttl_ms / 2, 30_000.0) + clamp_warned = False while not self._heartbeat_stop.wait(timeout=delay_ms / 1000.0): + # After the primed (delay-0) first beat, the baseline cadence + # is the held delay — a transient failure must not hot-loop. + delay_ms = delay_ms or held_delay_ms lead_min = grants_sum - (_lifecycle._now_mono_ms() - anchor_ms) if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: continue @@ -419,9 +422,28 @@ def heartbeat_loop() -> None: if prev_expiry is not None: prev_expiry += ttl_ms grant = max(grant, 0.0) + now_ms = _lifecycle._now_mono_ms() + elapsed_since_success = now_ms - last_success_ms + last_success_ms = now_ms grants_sum += grant last_grant = grant - delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) + if grant <= 0 or ( + grant < 0.9 * ttl_ms and grant <= 1.25 * elapsed_since_success + ): + # Lead-clamping server: the grant mirrors elapsed + # time, not lease size — no cadence signal exists + # on the wire. Hold a bounded cadence; never + # tighten toward the floor (that would burn the + # max_extensions budget in seconds). + delay_ms = held_delay_ms + if not clamp_warned: + clamp_warned = True + logger.warning( + "Server appears to clamp lease lead; extension budget will deplete: id=%s", + reservation_id, + ) + else: + delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) else: code = _extract_error_code(response) if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: @@ -490,7 +512,6 @@ def __init__( self._start_time: float = 0.0 self._heartbeat_task: asyncio.Task[None] | None = None - self._est_ttl: int | None = None self._retry_engine = AsyncCommitRetryEngine(client._config) self._retry_engine.set_client(client) @@ -556,9 +577,6 @@ async def __aenter__(self) -> AsyncStreamReservation: ) _set_context(self._ctx) - self._est_ttl = _lifecycle._effective_ttl_ms( - self._ttl_ms, result.expires_at_ms, response.server_date_ms - ) self._start_time = time.monotonic() self._heartbeat_task = self._start_heartbeat() @@ -693,21 +711,29 @@ def _start_heartbeat(self) -> asyncio.Task[None] | None: ctx = self._ctx client = self._client ttl_ms = self._ttl_ms - est = self._est_ttl async def heartbeat_loop() -> None: - # Conservative-lead heartbeat (v2.2) — see CyclesLifecycle heartbeat. + # Conservative-lead heartbeat (v2.3) — see CyclesLifecycle heartbeat. prev_expiry = ctx.expires_at_ms if ctx is not None else None anchor_ms = _lifecycle._now_mono_ms() grants_sum = 0.0 last_grant: float | None = None pending_body: dict[str, Any] | None = None - delay_ms = min( - [ttl_ms / 2, 30_000.0] + ([est / 2] if est is not None and est > 0 else []) - ) + # Immediate first extension: with lead_min starting at 0 and no + # trustworthy effective-TTL signal on the wire, any bounded first + # delay can outlive a policy-capped lease. Priming costs one + # extension; total protected runtime is unchanged. + delay_ms = 0.0 + last_success_ms = anchor_ms + held_delay_ms = min(ttl_ms / 2, 30_000.0) + clamp_warned = False try: while True: await asyncio.sleep(delay_ms / 1000.0) + # After the primed (delay-0) first beat, the baseline + # cadence is the held delay — a transient failure must + # not hot-loop. + delay_ms = delay_ms or held_delay_ms lead_min = grants_sum - (_lifecycle._now_mono_ms() - anchor_ms) if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: continue @@ -733,9 +759,28 @@ async def heartbeat_loop() -> None: if prev_expiry is not None: prev_expiry += ttl_ms grant = max(grant, 0.0) + now_ms = _lifecycle._now_mono_ms() + elapsed_since_success = now_ms - last_success_ms + last_success_ms = now_ms grants_sum += grant last_grant = grant - delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) + if grant <= 0 or ( + grant < 0.9 * ttl_ms and grant <= 1.25 * elapsed_since_success + ): + # Lead-clamping server: the grant mirrors elapsed + # time, not lease size — no cadence signal exists + # on the wire. Hold a bounded cadence; never + # tighten toward the floor (that would burn the + # max_extensions budget in seconds). + delay_ms = held_delay_ms + if not clamp_warned: + clamp_warned = True + logger.warning( + "Server appears to clamp lease lead; extension budget will deplete: id=%s", + reservation_id, + ) + else: + delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) else: code = _extract_error_code(response) if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: diff --git a/tests/test_decorator.py b/tests/test_decorator.py index 5e48e75..f0d5aaa 100644 --- a/tests/test_decorator.py +++ b/tests/test_decorator.py @@ -1,5 +1,7 @@ """Tests for the @cycles decorator.""" +import re + import pytest from runcycles.client import AsyncCyclesClient, CyclesClient @@ -14,6 +16,20 @@ def config() -> CyclesConfig: return CyclesConfig(base_url="http://localhost:7878", api_key="test-key", tenant="acme") +@pytest.fixture(autouse=True) +def _allow_heartbeat_extends(httpx_mock) -> None: # type: ignore[no-untyped-def] + # The v2.3 heartbeat primes an IMMEDIATE extend on entry; these tests + # run the real decorator flow, so accept (and ignore) heartbeat extends. + httpx_mock.add_response( + method="POST", + url=re.compile(r"http://localhost:7878/v1/reservations/[^/]+/extend$"), + json={"status": "ACTIVE"}, + status_code=200, + is_optional=True, + is_reusable=True, + ) + + class TestCyclesDecoratorSync: def test_basic_lifecycle(self, config: CyclesConfig, httpx_mock) -> None: # type: ignore[no-untyped-def] # Mock reservation creation diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 54ac706..13f4fee 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -1,9 +1,10 @@ -"""Deterministic tests for the lead-estimate heartbeat and the +"""Deterministic tests for the conservative-lead (v2.3) heartbeat and the ``actual_source: estimate`` audit marker.""" from __future__ import annotations import asyncio +import logging import threading import time from typing import Any @@ -87,7 +88,6 @@ def _run_sync_beats( beats: int, ttl: int = TTL, ctx: MagicMock | None = None, - est_ttl_ms: int | None = None, ) -> list[float]: """Drive the sync heartbeat for `beats` iterations, advancing the fake clock by the beat interval on every wait. Returns the wait timeouts.""" @@ -105,7 +105,7 @@ def wait(timeout: float | None = None) -> bool: stop = threading.Event() stop.wait = wait # type: ignore[method-assign] - thread = lifecycle._start_heartbeat("rsv_1", ttl, ctx or _ctx(), stop, est_ttl_ms) + thread = lifecycle._start_heartbeat("rsv_1", ttl, ctx or _ctx(), stop) assert thread is not None thread.join(timeout=5) assert not thread.is_alive() @@ -116,9 +116,9 @@ class TestSyncHeartbeatLeadEstimate: def test_extends_only_when_lead_below_threshold( self, monkeypatch: pytest.MonkeyPatch, ) -> None: - # lead_min starts at 0, so the heartbeat builds margin first: - # beats 1-4 extend (grants measured at +ttl each), beat 5 skips - # once lead_min reaches 1.5*grant. + # lead_min starts at 0 and the first beat fires IMMEDIATELY: beats + # 1-3 extend (grants measured at +ttl each), beat 4 skips once + # lead_min reaches 1.5*grant (180k-90k=90k), beat 5 extends again. lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ _extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4) @@ -127,13 +127,14 @@ def test_extends_only_when_lead_below_threshold( timeouts = _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=5) assert client.extend_reservation.call_count == 4 - assert timeouts[0] == min(TTL / 2, 30_000) / 1000.0 + assert timeouts[0] == 0.0 + assert timeouts[1] == min(TTL / 2, 30_000) / 1000.0 def test_interval_has_no_floor_for_small_ttl( self, monkeypatch: pytest.MonkeyPatch, ) -> None: - # ttl=1200 → interval must be 600ms (the old 1s floor guaranteed - # lapse in this spec-legal range). + # ttl=1200 → after the immediate first beat, cadence must be 600ms + # (the old 1s floor guaranteed lapse in this spec-legal range). lifecycle, client = _make_sync() client.extend_reservation.return_value = _extend_ok(None) @@ -142,7 +143,8 @@ def test_interval_has_no_floor_for_small_ttl( lifecycle, FakeClock(), monkeypatch, beats=2, ttl=1200, ctx=ctx, ) - assert timeouts[0] == 0.6 + assert timeouts[0] == 0.0 + assert timeouts[1] == 0.6 def test_failed_extend_retries_with_same_idempotency_key( self, monkeypatch: pytest.MonkeyPatch, @@ -219,14 +221,11 @@ def test_unknown_initial_expiry_anchors_on_first_success( _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3, ctx=_ctx(None)) - # Beat 1 extends with the fallback grant (no prior frame); beat 2's - # measured grant (3*ttl) lifts lead_min past 1.5*grant... beat 3: - # lead_min = (ttl + 3*ttl) - 90s = 150s >= 1.5*180s? No — 240-90=150 - # < 270 → extends would need a 3rd response; assert the skip math - # via count with exactly 2 responses and a 3rd beat that must skip: - # grants_sum=4*ttl=240s, elapsed=90s → lead 150s, 1.5*last_grant - # = 270s → NOT a skip. Give beat 3 nothing → the StopIteration is - # swallowed as a transient error, count stays meaningful at 3. + # Beat 1 (immediate) extends with the fallback grant (no prior + # frame); beat 2 measures a 3*ttl grant. Beat 3: lead_min = + # (ttl + 3*ttl) - 60s = 180s < 1.5*last_grant = 270s → NOT a skip, + # so a 3rd call happens; its StopIteration is swallowed as a + # transient error and the count stays meaningful at 3. assert client.extend_reservation.call_count == 3 def test_tenant_closed_stops_heartbeat( @@ -279,7 +278,7 @@ async def fake_sleep(s: float) -> None: clock.t += s * 1000.0 monkeypatch.setattr(asyncio, "sleep", fake_sleep) - task = lifecycle._start_heartbeat("rsv_1", TTL, ctx or _ctx(), None) + task = lifecycle._start_heartbeat("rsv_1", TTL, ctx or _ctx()) assert task is not None await task @@ -293,8 +292,8 @@ async def test_extends_only_when_lead_below_threshold( await self._run(lifecycle, 5, monkeypatch) - # lead_min builds from 0: beats 1-4 extend, beat 5 skips once - # lead_min reaches 1.5*grant. + # Immediate first beat, then lead_min builds from 0: beats 1-3 + # extend, beat 4 skips at lead_min >= 1.5*grant, beat 5 extends. assert client.extend_reservation.await_count == 4 async def test_permanent_code_stops_heartbeat( @@ -480,11 +479,12 @@ async def fake_sleep(s: float) -> None: # --------------------------------------------------------------------------- -# Effective TTL (tenant policy caps) +# Date header accessor (kept as a general response accessor; the heartbeat +# no longer consumes it — RFC 9110 §6.6.1 makes it a different clock). # --------------------------------------------------------------------------- -class TestEffectiveTtl: +class TestServerDateAccessor: def test_server_date_ms_parses_http_date(self) -> None: response = CyclesResponse.success( 200, {}, headers={"date": "Mon, 27 Jul 2026 12:00:00 GMT"}, @@ -498,46 +498,6 @@ def test_server_date_ms_absent_or_garbage(self) -> None: is None ) - def test_effective_ttl_derives_capped_grant(self) -> None: - from runcycles.lifecycle import _effective_ttl_ms - - # Requested 24h, tenant policy capped to 1h: expires − Date = 1h. - assert _effective_ttl_ms(86_400_000, 1_000_000 + 3_600_000, 1_000_000) == 3_600_000 - # Underivable → None (hint only; the caller falls back to caps). - assert _effective_ttl_ms(86_400_000, None, 1_000_000) is None - assert _effective_ttl_ms(86_400_000, 4_600_000, None) is None - # Never clamped UPWARD (that would fabricate lease) nor above request. - assert _effective_ttl_ms(60_000, 1_000_100, 1_000_000) == 100 - assert _effective_ttl_ms(60_000, 1_000_000 + 999_000, 1_000_000) == 60_000 - - def test_execute_seeds_heartbeat_with_effective_ttl(self) -> None: - # A 24h request capped to 1h must heartbeat on the 1h grant — the - # old behavior would schedule the first beat ~12h after expiry. - lifecycle, client = _make_sync() - now_ms = 1_785_153_600_000 - base_body = _allow_response().body - assert base_body is not None - body = dict(base_body) - body["expires_at_ms"] = now_ms + 3_600_000 - client.create_reservation.return_value = CyclesResponse.success( - 200, body, headers={"date": "Mon, 27 Jul 2026 12:00:00 GMT"}, - ) - client.commit_reservation.return_value = _commit_success() - - captured: dict[str, Any] = {} - - def fake_hb(rid: str, ttl: int, ctx: Any, stop: Any, est: Any = None) -> None: - captured["ttl"] = ttl - captured["est"] = est - return None - - lifecycle._start_heartbeat = fake_hb # type: ignore[method-assign] - lifecycle.execute(lambda: "r", (), {}, _cfg(ttl_ms=86_400_000)) - - # Requested ttl drives extend amounts; the derived grant is a HINT. - assert captured["ttl"] == 86_400_000 - assert captured["est"] == 3_600_000 - # --------------------------------------------------------------------------- # actual_source marker @@ -550,22 +510,58 @@ def _cfg(**kwargs: Any) -> DecoratorConfig: return DecoratorConfig(**defaults) -class TestFirstBeatDelay: - def test_thirty_second_cap_without_hint(self, monkeypatch: pytest.MonkeyPatch) -> None: +class TestFirstBeatAndRegimes: + def test_first_beat_is_immediate_even_for_huge_ttl( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # A 24h request silently capped to a small lease by tenant policy + # must still survive: only a zero first delay guarantees the first + # extension lands before ANY possible capped lease expires. lifecycle, client = _make_sync() client.extend_reservation.return_value = _extend_ok(None) timeouts = _run_sync_beats( lifecycle, FakeClock(), monkeypatch, beats=1, ttl=86_400_000, ) - assert timeouts[0] == 30.0 # 30s cap beats requested/2 = 12h + assert timeouts[0] == 0.0 - def test_estimate_hint_tightens_first_beat(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_first_beat_failure_does_not_hot_loop( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # A transient failure on the primed (delay-0) beat must back off to + # the held cadence, not spin at 0ms against a down server. lifecycle, client = _make_sync() - client.extend_reservation.return_value = _extend_ok(None) - timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=1, ttl=86_400_000, est_ttl_ms=10_000, - ) - assert timeouts[0] == 5.0 # est/2 wins when tighter than the cap + client.extend_reservation.return_value = CyclesResponse.http_error(503, "down") + timeouts = _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=2) + assert timeouts[0] == 0.0 + assert timeouts[1] == 30.0 + assert client.extend_reservation.call_count == 2 + + def test_lead_clamp_regime_holds_cadence_and_warns_once( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + # Maximum-lead clamping: every grant merely mirrors elapsed time + # (expires_at stays ~now + cap), so grant/2 cadence would collapse + # to the floor and burn max_extensions in seconds. The loop must + # hold the bounded cadence and warn exactly once. + clock = FakeClock() + lifecycle, client = _make_sync() + + def clamped_extend(rid: str, body: dict[str, Any]) -> CyclesResponse: + return _extend_ok(INITIAL_EXPIRY + int(clock.t)) + + client.extend_reservation.side_effect = clamped_extend + + with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): + timeouts = _run_sync_beats(lifecycle, clock, monkeypatch, beats=3) + + # Beat 1 measures grant 0 (prime), beats 2-3 measure grant == + # elapsed: all extend, cadence never tightens below the held delay. + assert client.extend_reservation.call_count == 3 + assert timeouts == [0.0, 30.0, 30.0, 30.0] + clamp_warnings = [r for r in caplog.records if "clamp lease lead" in r.message] + assert len(clamp_warnings) == 1 class TestActualSourceMarker: From c87354e11e202e61db6e112b8846d020e2804fe0 Mon Sep 17 00:00:00 2001 From: Albert Mavashev Date: Mon, 27 Jul 2026 18:19:27 -0400 Subject: [PATCH 5/9] fix: lead-clamp classifier gets a lower elapsed band (0.75x) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-SDK review (Rust port) caught a sticky misclassification in the v2.3 regime detector: after a lead_min skip, the next measured grant arrives across a doubled gap, so grant ~= elapsed exactly (cadence is grant/2) and the upper-bound-only test (grant <= 1.25*elapsed) claims lead-clamping. The hold then self-sustains — at held cadence 30s a 15s-grant lease banks +15s per 30s elapsed and decays to a lapse. The classifier now requires grant to sit INSIDE [0.75, 1.25]*elapsed (besides grant < 0.9*ttl; grant <= 0 still always clamps). Genuine maximum-lead clamps track any gap (ratio ~= 1) and stay held; a real post-skip small grant lands in the hold once, then at the held cadence its ratio falls to ~0.5, exits the band, and cadence re-tightens. Regression test pins the full 7-beat trace: 3 extends at grant/2, skip, one 30s hold, re-tighten to grant/2. 533 tests, 100% coverage. --- AUDIT.md | 10 ++++++---- CHANGELOG.md | 2 +- runcycles/lifecycle.py | 18 ++++++++++++++---- runcycles/streaming.py | 18 ++++++++++++++---- tests/test_heartbeat.py | 23 +++++++++++++++++++++++ 5 files changed, 58 insertions(+), 13 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 9d32463..259958e 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -25,16 +25,18 @@ frame only, no cross-clock arithmetic); FIRST extension fires immediately (any bounded first delay can outlive a tenant-policy-capped lease); cadence splits by regime — a grant tracking the lease drives clamp(grant/2, 500ms, ttl/2), while a grant merely mirroring elapsed time -(maximum-lead clamping) carries no wire cadence signal, so the loop holds -min(ttl/2, 30s) and warns once instead of burning max_extensions at the -floor; a transient failure on the primed beat backs off to the held +(maximum-lead clamping: grant ≤ 0, or grant < 0.9×ttl inside a +[0.75, 1.25]×elapsed band — the lower edge keeps a post-skip small grant +from sticking in the hold) carries no wire cadence signal, so the loop +holds min(ttl/2, 30s) and warns once instead of burning max_extensions at +the floor; a transient failure on the primed beat backs off to the held cadence (no hot loop); skip at lead_min ≥ 1.5×last_grant; extend idempotency key reused on retries; permanent stop on expired/finalized/ max-extensions/tenant-closed/not-found (and raw 404/410). The HTTP Date header plays no heartbeat role (RFC 9110 §6.6.1; Redis TIME vs container clock). Commits whose actual was defaulted from the estimate now carry metadata.actual_source="estimate" for audit honesty. Spec guidance: -cycles-protocol#148. 532 tests pass at 100% coverage. +cycles-protocol#148. 533 tests pass at 100% coverage. ## 2026-07-27 — Durable commit retries (journal + /v1/events fallback) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9cef62..b81ff99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Heartbeat redesign (conservative lead lower bound, immediate prime, regime-aware cadence)**: four adversarial review rounds refined the design. The heartbeat maintains `lead_min = Σ measured grants − monotonic elapsed` (grants = differences of successive returned `expires_at_ms` — the same server frame; no cross-clock arithmetic anywhere), starting at 0, so the FIRST extension fires immediately: any bounded first delay could outlive a tenant-policy-capped lease (a 24h request silently capped to seconds would expire before a delayed first beat), and priming costs one extension without changing total protected runtime. Cadence then splits by regime, detected from the measured grant: when the grant tracks the requested lease, cadence = `clamp(grant/2, 500ms, ttl/2)` — per-extend policy clamps automatically tighten the beat; when the grant merely mirrors elapsed time (maximum-lead clamping: `grant ≤ 0`, or `grant < 0.9×ttl` and `grant ≤ 1.25×elapsed`), no cadence signal exists on the wire — the loop holds `min(ttl/2, 30s)` and warns once that the extension budget will deplete, instead of collapsing to the floor and burning `max_extensions` in seconds. A transient failure on the primed (delay-0) beat backs off to the held cadence — never a hot loop. Skip when `lead_min ≥ 1.5×last_grant`. The HTTP `Date` header plays no role in the heartbeat (RFC 9110 §6.6.1: best-effort, whole-second, and possibly a different clock than the one stamping `expires_at_ms` — Redis TIME in the reference server); `CyclesResponse.server_date_ms` remains as a general accessor. Failed extends retry with the same idempotency key; permanent codes (`RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`, raw 404/410) stop the heartbeat; no interval floor. Spec guidance: cycles-protocol#148. +- **Heartbeat redesign (conservative lead lower bound, immediate prime, regime-aware cadence)**: four adversarial review rounds refined the design. The heartbeat maintains `lead_min = Σ measured grants − monotonic elapsed` (grants = differences of successive returned `expires_at_ms` — the same server frame; no cross-clock arithmetic anywhere), starting at 0, so the FIRST extension fires immediately: any bounded first delay could outlive a tenant-policy-capped lease (a 24h request silently capped to seconds would expire before a delayed first beat), and priming costs one extension without changing total protected runtime. Cadence then splits by regime, detected from the measured grant: when the grant tracks the requested lease, cadence = `clamp(grant/2, 500ms, ttl/2)` — per-extend policy clamps automatically tighten the beat; when the grant merely mirrors elapsed time (maximum-lead clamping: `grant ≤ 0`, or `grant < 0.9×ttl` with `grant` inside `[0.75, 1.25]×elapsed-since-last-success`), no cadence signal exists on the wire — the loop holds `min(ttl/2, 30s)` and warns once that the extension budget will deplete, instead of collapsing to the floor and burning `max_extensions` in seconds. The band's lower edge makes misclassification non-sticky: a real per-extend grant seen across a skip-doubled gap (where grant ≈ elapsed exactly) lands in the hold once, but at the held cadence its grant/elapsed ratio falls below 0.75 and cadence re-tightens — with an upper bound alone the hold would stick and a clamped-grant lease would decay to a lapse. A transient failure on the primed (delay-0) beat backs off to the held cadence — never a hot loop. Skip when `lead_min ≥ 1.5×last_grant`. The HTTP `Date` header plays no role in the heartbeat (RFC 9110 §6.6.1: best-effort, whole-second, and possibly a different clock than the one stamping `expires_at_ms` — Redis TIME in the reference server); `CyclesResponse.server_date_ms` remains as a general accessor. Failed extends retry with the same idempotency key; permanent codes (`RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`, raw 404/410) stop the heartbeat; no interval floor. Spec guidance: cycles-protocol#148. - **Heartbeat (superseded intermediate designs, kept for history)**: the initial alternate-beat fix traded outward drift for inward-drift liveness hazards (single-failure lead-0 retry, guaranteed lapse for ttl < 2s under the old 1s interval floor, RTT slippage) found by adversarial self-review. The heartbeat now estimates its remaining lead from the authoritative `expires_at_ms` the server returns — compared clock-skew-free (server-frame differences + client-monotonic elapsed only) — and extends when lead < 1.5×ttl. It also: derives the **effective TTL** from `expires_at_ms − Date` header (tenant policy `max_reservation_ttl_ms`, default 1h, silently caps grants — a 24h request would otherwise heartbeat 12h late); reuses the same idempotency key when retrying a failed extend (a lost response cannot double-extend); stops permanently on `RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`; and drops the 1s interval floor. Spec guidance: cycles-protocol#148. - **Heartbeat extend drift** (superseded by the redesign above, kept for history): `extend_by_ms` is relative to the reservation's *current* `expires_at_ms` (spec), but the heartbeat extended by the full `ttl_ms` on every `ttl/2` beat — drifting expiry outward by `ttl/2` per beat. A killed process left its reserved budget locked until the drifted expiry (a zombie-reservation window scaling with runtime, bounded only by `max_extensions`), and long runs burned the `max_extensions` budget twice as fast as needed, losing heartbeat protection mid-flight. All four heartbeats (both lifecycles, both streaming context managers) now use alternate-beat extension: extend on the first beat and every second beat after a success, retrying immediately after a failure. Expiry lead stays within `[ttl/2, 1.5×ttl]`; extension consumption is halved. Fleet-wide fix (TS/Java/Rust ship the same change); spec guidance added in cycles-protocol#148. diff --git a/runcycles/lifecycle.py b/runcycles/lifecycle.py index a09d2de..47c1783 100644 --- a/runcycles/lifecycle.py +++ b/runcycles/lifecycle.py @@ -560,13 +560,18 @@ def heartbeat_loop() -> None: grants_sum += grant last_grant = grant if grant <= 0 or ( - grant < 0.9 * ttl_ms and grant <= 1.25 * elapsed_since_success + grant < 0.9 * ttl_ms + and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success ): # Lead-clamping server: the grant mirrors elapsed # time, not lease size — no cadence signal exists # on the wire. Hold a bounded cadence; never # tighten toward the floor (that would burn the - # max_extensions budget in seconds). + # max_extensions budget in seconds). The lower + # band keeps this non-sticky: a real small grant + # seen across a skip-doubled gap lands here once, + # but at the held cadence its grant/elapsed ratio + # falls below 0.75 and cadence re-tightens. delay_ms = held_delay_ms if not clamp_warned: clamp_warned = True @@ -832,13 +837,18 @@ async def heartbeat_loop() -> None: grants_sum += grant last_grant = grant if grant <= 0 or ( - grant < 0.9 * ttl_ms and grant <= 1.25 * elapsed_since_success + grant < 0.9 * ttl_ms + and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success ): # Lead-clamping server: the grant mirrors elapsed # time, not lease size — no cadence signal exists # on the wire. Hold a bounded cadence; never # tighten toward the floor (that would burn the - # max_extensions budget in seconds). + # max_extensions budget in seconds). The lower + # band keeps this non-sticky: a real small grant + # seen across a skip-doubled gap lands here once, + # but at the held cadence its grant/elapsed ratio + # falls below 0.75 and cadence re-tightens. delay_ms = held_delay_ms if not clamp_warned: clamp_warned = True diff --git a/runcycles/streaming.py b/runcycles/streaming.py index eac2290..5a627b2 100644 --- a/runcycles/streaming.py +++ b/runcycles/streaming.py @@ -428,13 +428,18 @@ def heartbeat_loop() -> None: grants_sum += grant last_grant = grant if grant <= 0 or ( - grant < 0.9 * ttl_ms and grant <= 1.25 * elapsed_since_success + grant < 0.9 * ttl_ms + and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success ): # Lead-clamping server: the grant mirrors elapsed # time, not lease size — no cadence signal exists # on the wire. Hold a bounded cadence; never # tighten toward the floor (that would burn the - # max_extensions budget in seconds). + # max_extensions budget in seconds). The lower + # band keeps this non-sticky: a real small grant + # seen across a skip-doubled gap lands here once, + # but at the held cadence its grant/elapsed ratio + # falls below 0.75 and cadence re-tightens. delay_ms = held_delay_ms if not clamp_warned: clamp_warned = True @@ -765,13 +770,18 @@ async def heartbeat_loop() -> None: grants_sum += grant last_grant = grant if grant <= 0 or ( - grant < 0.9 * ttl_ms and grant <= 1.25 * elapsed_since_success + grant < 0.9 * ttl_ms + and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success ): # Lead-clamping server: the grant mirrors elapsed # time, not lease size — no cadence signal exists # on the wire. Hold a bounded cadence; never # tighten toward the floor (that would burn the - # max_extensions budget in seconds). + # max_extensions budget in seconds). The lower + # band keeps this non-sticky: a real small grant + # seen across a skip-doubled gap lands here once, + # but at the held cadence its grant/elapsed ratio + # falls below 0.75 and cadence re-tightens. delay_ms = held_delay_ms if not clamp_warned: clamp_warned = True diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 13f4fee..d184123 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -196,6 +196,29 @@ def test_clamped_grants_extend_every_beat( # Cadence re-derived from the MEASURED grant (ttl/4 → beat at ttl/8). assert timeouts[1] == (TTL / 4 / 2) / 1000.0 + def test_grant_clamp_misclassification_after_skip_is_transient( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # After a skip, the next measured grant arrives across a doubled + # gap, so grant ≈ elapsed and the beat lands in the lead-clamp arm + # once. The classifier's lower band (0.75×elapsed) must make that + # non-sticky: at the held cadence the ratio falls to ~0.5, the + # regime reads normal again, and cadence re-tightens — without the + # band the hold sticks and a ttl/4-grant lease decays to a lapse. + lifecycle, client = _make_sync() + grant = TTL // 4 # 15000 → cadence ttl/8 = 7500ms + client.extend_reservation.side_effect = [ + _extend_ok(INITIAL_EXPIRY + (n + 1) * grant) for n in range(6) + ] + + timeouts = _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=7) + + # b1-b3 extend @7.5s, b4 skips (lead 22.5k ≥ 1.5×15k), b5 extends + # across the doubled gap (misclassified → one 30s hold), b6 + # re-tightens to 7.5s, b7 extends on cadence. + assert client.extend_reservation.call_count == 6 + assert timeouts == [0.0, 7.5, 7.5, 7.5, 7.5, 30.0, 7.5, 7.5] + def test_missing_expires_in_response_falls_back_to_plus_ttl( self, monkeypatch: pytest.MonkeyPatch, ) -> None: From 4c58ee19a3f3131751fe444522b27df321e55d78 Mon Sep 17 00:00:00 2001 From: Albert Mavashev Date: Tue, 28 Jul 2026 07:26:10 -0400 Subject: [PATCH 6/9] feat: schedule heartbeats from server-authoritative remaining_ttl_ms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec review round 5 (cycles-protocol#148) proved the heuristic's regime detection formally undecidable — the band has a sticky window grant in [0.75*min(ttl/2,30s), 0.9*ttl) where a grant-clamped server stays misclassified while its lease erodes to a lapse — and immediate priming schedule-dependent under maximum-lead clamping. The adopted fix is a wire field: spec v0.1.25.16 adds remaining_ttl_ms (same clock snapshot as expires_at_ms) to create and extend responses; cycles-server v0.1.25.59 emits it (runcycles/cycles-server#260). When a response carries the field, the heartbeat schedules from it directly (all four loops): - lead_floor = max(0, remaining_ttl_ms - rtt), rtt measured monotonically around each extend call (max tracked per heartbeat) - next beat at lead_floor - min(lead_floor/2, max(1s, 2*rtt_max)) after response receipt; recomputed on every field-bearing response; never accumulates expiry differences in this mode - the create response's field drives the FIRST delay - no primed extension is spent, and a 24h request capped to a 1s lease gets its first beat at 500ms, inside the real lease - the heuristic lead_min skip is bypassed (scheduling is exact) - transient failures retry with the SAME idempotency key after clamp(current_lead/4, 1s, 30s), current_lead = last lead_floor decayed by monotonic elapsed Grants/lead bookkeeping keeps running in both modes, so the v2.3+band heuristic - now explicitly a best-effort fallback - resumes seamlessly when the field disappears; fieldless servers see unchanged behavior. ReservationCreateResponse gains the optional remaining_ttl_ms field. Tests: field-driven first beat + steady 59s cadence with the skip provably bypassed; capped-lease first beat; lead-clamp server WITH the field (no warn, no collapse); mid-flight field disappearance resuming the heuristic; bounded same-key retries on 503 and exceptions across sync/async lifecycle and both stream heartbeats. 543 tests, 100% coverage; ruff + mypy clean. --- AUDIT.md | 18 +++ CHANGELOG.md | 1 + runcycles/lifecycle.py | 151 +++++++++++++++++---- runcycles/models.py | 4 + runcycles/streaming.py | 129 +++++++++++++++--- tests/test_heartbeat.py | 282 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 543 insertions(+), 42 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 259958e..fffef17 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -13,6 +13,24 @@ --- +## 2026-07-28 — Server-authoritative heartbeat via remaining_ttl_ms (v0.5.1, same PR) + +Spec review round 5 proved the fallback heuristic's regime detection +undecidable (sticky band window [0.75×min(ttl/2,30s), 0.9×ttl) — e.g. +ttl 24s / cap 10s / held 12s / ratio 0.833 forever, lease erodes to +lapse) and immediate priming schedule-dependent under maximum-lead +clamping. Adopted the reviewer's fix: spec v0.1.25.16 adds +remaining_ttl_ms to create+extend responses (cycles-server v0.1.25.59 +emits it; extend replays recompute it fresh). When present, the SDK +schedules from it directly — lead_floor = max(0, remaining − rtt), next +beat at lead_floor − min(lead_floor/2, max(1s, 2×rtt_max)) from response +receipt, recomputed per field-bearing response, lead_min skip bypassed, +no primed first extension (create field drives the first delay), bounded +same-key retry clamp(lead/4, 1s, 30s) on transient failures. Bookkeeping +keeps running so the v2.3+band heuristic (now explicitly best-effort +fallback) resumes seamlessly if the field disappears; fieldless servers +see unchanged behavior. 543 tests pass at 100% coverage. + ## 2026-07-27 — Heartbeat conservative-lead redesign + actual_source marker (v0.5.1) The heartbeat extended by full ttl_ms every ttl/2 beat while extend_by_ms diff --git a/CHANGELOG.md b/CHANGELOG.md index b81ff99..9784376 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Server-authoritative heartbeat scheduling via `remaining_ttl_ms`** (spec v0.1.25.16, cycles-protocol#148): when a create or extend response carries `remaining_ttl_ms` — the server's own statement of the live lease, measured on the clock that stamps `expires_at_ms` — the heartbeat schedules directly from it: `lead_floor = max(0, remaining − rtt)`, next beat at `lead_floor − min(lead_floor/2, max(1s, 2×max observed rtt))` after response receipt, recomputed on every field-bearing response; the heuristic `lead_min` skip is bypassed (scheduling is exact) and no primed first extension is spent when the create response carries the field — a 24h request silently capped to a 1s lease gets its first beat at 500ms, inside the real lease. Transient failures in this mode retry with the same idempotency key after `clamp(current_lead/4, 1s, 30s)`. Grants/lead bookkeeping keeps running in both modes, so the heuristic below takes over seamlessly if the field disappears; servers that never emit it get the unchanged fallback behavior. The heuristic (conservative lead lower bound + regime band) is now explicitly a **best-effort fallback**: spec review round 5 proved regime detection from `(grant, elapsed)` observables undecidable in general — the band's sticky window `grant ∈ [0.75×min(ttl/2, 30s), 0.9×ttl)` misclassifies permanently — which is precisely why the wire field exists. - `metadata.actual_source: "estimate"` is stamped on commits whose actual was structurally defaulted from the estimate (`@cycles` without an `actual` expression; streams with no recorded cost or a raised `cost_fn`), so audit evidence distinguishes measured spend from assumed spend. Defaults are unchanged; the marker flows into `/v1/events` recovery bodies via the shared metadata. ## [0.5.0] - 2026-07-27 diff --git a/runcycles/lifecycle.py b/runcycles/lifecycle.py index 47c1783..fbfd7aa 100644 --- a/runcycles/lifecycle.py +++ b/runcycles/lifecycle.py @@ -220,6 +220,14 @@ def _now_mono_ms() -> float: _LEAD_TARGET_FACTOR = 1.5 +def _authoritative_delay_ms(lead_floor_ms: float, rtt_max_ms: float) -> float: + """Next-beat delay from a server-authoritative remaining lease + (remaining_ttl_ms, spec v0.1.25.16): schedule inside the lease while + keeping a retry reserve of min(lead/2, max(1s, 2x max observed rtt)).""" + reserve = min(lead_floor_ms / 2, max(1000.0, 2.0 * rtt_max_ms)) + return max(0.0, lead_floor_ms - reserve) + + def _build_extend_body(ttl_ms: int) -> dict[str, Any]: validate_extend_by_ms(ttl_ms) return {"idempotency_key": str(uuid.uuid4()), "extend_by_ms": ttl_ms} @@ -376,7 +384,9 @@ def execute( # Start heartbeat heartbeat_stop = threading.Event() - heartbeat_thread = self._start_heartbeat(reservation_id, cfg.ttl_ms, ctx, heartbeat_stop) + heartbeat_thread = self._start_heartbeat( + reservation_id, cfg.ttl_ms, ctx, heartbeat_stop, res_result.remaining_ttl_ms, + ) try: result = fn(*args, **kwargs) @@ -492,7 +502,12 @@ def _handle_release(self, reservation_id: str, reason: str) -> None: logger.exception("Failed to release: id=%s", reservation_id) def _start_heartbeat( - self, reservation_id: str, ttl_ms: int, ctx: CyclesContext, stop_event: threading.Event, + self, + reservation_id: str, + ttl_ms: int, + ctx: CyclesContext, + stop_event: threading.Event, + initial_remaining_ms: int | None = None, ) -> threading.Thread | None: if ttl_ms <= 0: return None @@ -518,25 +533,41 @@ def heartbeat_loop() -> None: grants_sum = 0.0 last_grant: float | None = None pending_body: dict[str, Any] | None = None - # Immediate first extension: with lead_min starting at 0 and no - # trustworthy effective-TTL signal on the wire, any bounded first - # delay can outlive a policy-capped lease. Priming costs one - # extension; total protected runtime is unchanged. - delay_ms = 0.0 last_success_ms = anchor_ms held_delay_ms = min(ttl_ms / 2, 30_000.0) clamp_warned = False + rtt_max_ms = 0.0 + # Authoritative scheduling (spec v0.1.25.16): a response carrying + # remaining_ttl_ms is the server's own statement of the live + # lease, so the beat is scheduled from it directly. When the + # create response carried it, the first beat derives from it and + # no primed extension is spent. + lead_floor_ms: float | None = None + lead_anchor_ms = anchor_ms + authoritative = initial_remaining_ms is not None + if initial_remaining_ms is not None: + lead_floor_ms = float(initial_remaining_ms) + delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + else: + # Immediate first extension (fallback): with lead_min starting + # at 0 and no lease signal on the wire, any bounded first + # delay can outlive a policy-capped lease. Priming costs one + # extension; total protected runtime is unchanged. + delay_ms = 0.0 while not stop_event.wait(timeout=delay_ms / 1000.0): # After the primed (delay-0) first beat, the baseline cadence # is the held delay — a transient failure must not hot-loop. delay_ms = delay_ms or held_delay_ms - lead_min = grants_sum - (_now_mono_ms() - anchor_ms) - if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: - continue + if not authoritative: + lead_min = grants_sum - (_now_mono_ms() - anchor_ms) + if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: + continue try: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) pending_body = body + sent_ms = _now_mono_ms() response = self._client.extend_reservation(reservation_id, body) + recv_ms = _now_mono_ms() if response.is_success: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") @@ -559,7 +590,18 @@ def heartbeat_loop() -> None: last_success_ms = now_ms grants_sum += grant last_grant = grant - if grant <= 0 or ( + rtt_ms = recv_ms - sent_ms + rtt_max_ms = max(rtt_max_ms, rtt_ms) + remaining = response.get_body_attribute("remaining_ttl_ms") + if remaining is not None: + # Server-authoritative lease (spec v0.1.25.16): + # schedule from it directly; the heuristic arms + # below only serve servers that omit the field. + authoritative = True + lead_floor_ms = max(0.0, float(int(remaining)) - rtt_ms) + lead_anchor_ms = recv_ms + delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + elif grant <= 0 or ( grant < 0.9 * ttl_ms and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success ): @@ -572,6 +614,7 @@ def heartbeat_loop() -> None: # seen across a skip-doubled gap lands here once, # but at the held cadence its grant/elapsed ratio # falls below 0.75 and cadence re-tightens. + authoritative = False delay_ms = held_delay_ms if not clamp_warned: clamp_warned = True @@ -580,6 +623,7 @@ def heartbeat_loop() -> None: reservation_id, ) else: + authoritative = False delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) logger.debug("Heartbeat extend ok: id=%s", reservation_id) else: @@ -591,8 +635,22 @@ def heartbeat_loop() -> None: ) return logger.warning("Heartbeat extend failed: id=%s, status=%d", reservation_id, response.status) + if authoritative and lead_floor_ms is not None: + # Retry inside the known lease: the failed extend may + # have been applied server-side, so the SAME body (same + # idempotency key) is retried at a bounded fraction of + # the remaining lead. + lead_now = max(0.0, lead_floor_ms - (_now_mono_ms() - lead_anchor_ms)) + delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) except Exception: logger.warning("Heartbeat extend error: id=%s", reservation_id, exc_info=True) + if authoritative and lead_floor_ms is not None: + # Retry inside the known lease: the failed extend may + # have been applied server-side, so the SAME body (same + # idempotency key) is retried at a bounded fraction of + # the remaining lead. + lead_now = max(0.0, lead_floor_ms - (_now_mono_ms() - lead_anchor_ms)) + delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) t = threading.Thread(target=heartbeat_loop, daemon=True, name=f"cycles-heartbeat-{reservation_id[:12]}") t.start() @@ -668,7 +726,9 @@ async def execute( ) _set_context(ctx) - heartbeat_task = self._start_heartbeat(reservation_id, cfg.ttl_ms, ctx) + heartbeat_task = self._start_heartbeat( + reservation_id, cfg.ttl_ms, ctx, res_result.remaining_ttl_ms, + ) try: result = await fn(*args, **kwargs) @@ -780,7 +840,11 @@ async def _handle_release(self, reservation_id: str, reason: str) -> None: logger.exception("Failed to release: id=%s", reservation_id) def _start_heartbeat( - self, reservation_id: str, ttl_ms: int, ctx: CyclesContext, + self, + reservation_id: str, + ttl_ms: int, + ctx: CyclesContext, + initial_remaining_ms: int | None = None, ) -> asyncio.Task[None] | None: if ttl_ms <= 0: return None @@ -792,14 +856,27 @@ async def heartbeat_loop() -> None: grants_sum = 0.0 last_grant: float | None = None pending_body: dict[str, Any] | None = None - # Immediate first extension: with lead_min starting at 0 and no - # trustworthy effective-TTL signal on the wire, any bounded first - # delay can outlive a policy-capped lease. Priming costs one - # extension; total protected runtime is unchanged. - delay_ms = 0.0 last_success_ms = anchor_ms held_delay_ms = min(ttl_ms / 2, 30_000.0) clamp_warned = False + rtt_max_ms = 0.0 + # Authoritative scheduling (spec v0.1.25.16): a response carrying + # remaining_ttl_ms is the server's own statement of the live + # lease, so the beat is scheduled from it directly. When the + # create response carried it, the first beat derives from it and + # no primed extension is spent. + lead_floor_ms: float | None = None + lead_anchor_ms = anchor_ms + authoritative = initial_remaining_ms is not None + if initial_remaining_ms is not None: + lead_floor_ms = float(initial_remaining_ms) + delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + else: + # Immediate first extension (fallback): with lead_min starting + # at 0 and no lease signal on the wire, any bounded first + # delay can outlive a policy-capped lease. Priming costs one + # extension; total protected runtime is unchanged. + delay_ms = 0.0 try: while True: await asyncio.sleep(delay_ms / 1000.0) @@ -807,13 +884,16 @@ async def heartbeat_loop() -> None: # cadence is the held delay — a transient failure must # not hot-loop. delay_ms = delay_ms or held_delay_ms - lead_min = grants_sum - (_now_mono_ms() - anchor_ms) - if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: - continue + if not authoritative: + lead_min = grants_sum - (_now_mono_ms() - anchor_ms) + if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: + continue try: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) pending_body = body + sent_ms = _now_mono_ms() response = await self._client.extend_reservation(reservation_id, body) + recv_ms = _now_mono_ms() if response.is_success: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") @@ -836,7 +916,18 @@ async def heartbeat_loop() -> None: last_success_ms = now_ms grants_sum += grant last_grant = grant - if grant <= 0 or ( + rtt_ms = recv_ms - sent_ms + rtt_max_ms = max(rtt_max_ms, rtt_ms) + remaining = response.get_body_attribute("remaining_ttl_ms") + if remaining is not None: + # Server-authoritative lease (spec v0.1.25.16): + # schedule from it directly; the heuristic arms + # below only serve servers that omit the field. + authoritative = True + lead_floor_ms = max(0.0, float(int(remaining)) - rtt_ms) + lead_anchor_ms = recv_ms + delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + elif grant <= 0 or ( grant < 0.9 * ttl_ms and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success ): @@ -849,6 +940,7 @@ async def heartbeat_loop() -> None: # seen across a skip-doubled gap lands here once, # but at the held cadence its grant/elapsed ratio # falls below 0.75 and cadence re-tightens. + authoritative = False delay_ms = held_delay_ms if not clamp_warned: clamp_warned = True @@ -857,6 +949,7 @@ async def heartbeat_loop() -> None: reservation_id, ) else: + authoritative = False delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) else: code = _extract_error_code(response) @@ -867,8 +960,22 @@ async def heartbeat_loop() -> None: ) return logger.warning("Heartbeat extend failed: id=%s", reservation_id) + if authoritative and lead_floor_ms is not None: + # Retry inside the known lease: the failed extend may + # have been applied server-side, so the SAME body (same + # idempotency key) is retried at a bounded fraction of + # the remaining lead. + lead_now = max(0.0, lead_floor_ms - (_now_mono_ms() - lead_anchor_ms)) + delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) except Exception: logger.warning("Heartbeat extend error: id=%s", reservation_id, exc_info=True) + if authoritative and lead_floor_ms is not None: + # Retry inside the known lease: the failed extend may + # have been applied server-side, so the SAME body (same + # idempotency key) is retried at a bounded fraction of + # the remaining lead. + lead_now = max(0.0, lead_floor_ms - (_now_mono_ms() - lead_anchor_ms)) + delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) except asyncio.CancelledError: return diff --git a/runcycles/models.py b/runcycles/models.py index 90b6dfe..3b87c8b 100644 --- a/runcycles/models.py +++ b/runcycles/models.py @@ -266,6 +266,10 @@ class ReservationCreateResponse(BaseModel): reservation_id: str | None = None affected_scopes: list[str] expires_at_ms: int | None = None + # Server-authoritative remaining lease (ms) at response evaluation + # (spec v0.1.25.16). Optional: older servers omit it; when present the + # heartbeat schedules from it directly. + remaining_ttl_ms: int | None = None scope_path: str | None = None reserved: Amount | None = None caps: Caps | None = None diff --git a/runcycles/streaming.py b/runcycles/streaming.py index 5a627b2..e825eee 100644 --- a/runcycles/streaming.py +++ b/runcycles/streaming.py @@ -19,6 +19,7 @@ from runcycles.lifecycle import ( _LEAD_TARGET_FACTOR, _PERMANENT_EXTEND_CODES, + _authoritative_delay_ms, _build_commit_body, _build_event_fallback_body, _build_extend_body, @@ -177,6 +178,7 @@ def __init__( self._usage = StreamUsage() self._reservation_id: str | None = None + self._initial_remaining: int | None = None self._caps: Caps | None = None self._decision: Decision = Decision.ALLOW self._ctx: CyclesContext | None = None @@ -233,6 +235,7 @@ def __enter__(self) -> StreamReservation: ) self._reservation_id = result.reservation_id + self._initial_remaining = result.remaining_ttl_ms self._decision = result.decision self._caps = result.caps @@ -385,25 +388,42 @@ def heartbeat_loop() -> None: grants_sum = 0.0 last_grant: float | None = None pending_body: dict[str, Any] | None = None - # Immediate first extension: with lead_min starting at 0 and no - # trustworthy effective-TTL signal on the wire, any bounded first - # delay can outlive a policy-capped lease. Priming costs one - # extension; total protected runtime is unchanged. - delay_ms = 0.0 + initial_remaining_ms = self._initial_remaining last_success_ms = anchor_ms held_delay_ms = min(ttl_ms / 2, 30_000.0) clamp_warned = False + rtt_max_ms = 0.0 + # Authoritative scheduling (spec v0.1.25.16): a response carrying + # remaining_ttl_ms is the server's own statement of the live + # lease, so the beat is scheduled from it directly. When the + # create response carried it, the first beat derives from it and + # no primed extension is spent. + lead_floor_ms: float | None = None + lead_anchor_ms = anchor_ms + authoritative = initial_remaining_ms is not None + if initial_remaining_ms is not None: + lead_floor_ms = float(initial_remaining_ms) + delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + else: + # Immediate first extension (fallback): with lead_min starting + # at 0 and no lease signal on the wire, any bounded first + # delay can outlive a policy-capped lease. Priming costs one + # extension; total protected runtime is unchanged. + delay_ms = 0.0 while not self._heartbeat_stop.wait(timeout=delay_ms / 1000.0): # After the primed (delay-0) first beat, the baseline cadence # is the held delay — a transient failure must not hot-loop. delay_ms = delay_ms or held_delay_ms - lead_min = grants_sum - (_lifecycle._now_mono_ms() - anchor_ms) - if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: - continue + if not authoritative: + lead_min = grants_sum - (_lifecycle._now_mono_ms() - anchor_ms) + if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: + continue try: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) pending_body = body + sent_ms = _lifecycle._now_mono_ms() response = self._client.extend_reservation(reservation_id, body) + recv_ms = _lifecycle._now_mono_ms() if response.is_success: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") @@ -427,7 +447,18 @@ def heartbeat_loop() -> None: last_success_ms = now_ms grants_sum += grant last_grant = grant - if grant <= 0 or ( + rtt_ms = recv_ms - sent_ms + rtt_max_ms = max(rtt_max_ms, rtt_ms) + remaining = response.get_body_attribute("remaining_ttl_ms") + if remaining is not None: + # Server-authoritative lease (spec v0.1.25.16): + # schedule from it directly; the heuristic arms + # below only serve servers that omit the field. + authoritative = True + lead_floor_ms = max(0.0, float(int(remaining)) - rtt_ms) + lead_anchor_ms = recv_ms + delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + elif grant <= 0 or ( grant < 0.9 * ttl_ms and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success ): @@ -440,6 +471,7 @@ def heartbeat_loop() -> None: # seen across a skip-doubled gap lands here once, # but at the held cadence its grant/elapsed ratio # falls below 0.75 and cadence re-tightens. + authoritative = False delay_ms = held_delay_ms if not clamp_warned: clamp_warned = True @@ -448,6 +480,7 @@ def heartbeat_loop() -> None: reservation_id, ) else: + authoritative = False delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) else: code = _extract_error_code(response) @@ -458,8 +491,22 @@ def heartbeat_loop() -> None: ) return logger.warning("Stream heartbeat failed: id=%s", reservation_id) + if authoritative and lead_floor_ms is not None: + # Retry inside the known lease: the failed extend may + # have been applied server-side, so the SAME body (same + # idempotency key) is retried at a bounded fraction of + # the remaining lead. + lead_now = max(0.0, lead_floor_ms - (_lifecycle._now_mono_ms() - lead_anchor_ms)) + delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) except Exception: logger.warning("Stream heartbeat error: id=%s", reservation_id, exc_info=True) + if authoritative and lead_floor_ms is not None: + # Retry inside the known lease: the failed extend may + # have been applied server-side, so the SAME body (same + # idempotency key) is retried at a bounded fraction of + # the remaining lead. + lead_now = max(0.0, lead_floor_ms - (_lifecycle._now_mono_ms() - lead_anchor_ms)) + delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) t = threading.Thread( target=heartbeat_loop, @@ -511,6 +558,7 @@ def __init__( self._usage = StreamUsage() self._reservation_id: str | None = None + self._initial_remaining: int | None = None self._caps: Caps | None = None self._decision: Decision = Decision.ALLOW self._ctx: CyclesContext | None = None @@ -566,6 +614,7 @@ async def __aenter__(self) -> AsyncStreamReservation: ) self._reservation_id = result.reservation_id + self._initial_remaining = result.remaining_ttl_ms self._decision = result.decision self._caps = result.caps @@ -724,14 +773,28 @@ async def heartbeat_loop() -> None: grants_sum = 0.0 last_grant: float | None = None pending_body: dict[str, Any] | None = None - # Immediate first extension: with lead_min starting at 0 and no - # trustworthy effective-TTL signal on the wire, any bounded first - # delay can outlive a policy-capped lease. Priming costs one - # extension; total protected runtime is unchanged. - delay_ms = 0.0 + initial_remaining_ms = self._initial_remaining last_success_ms = anchor_ms held_delay_ms = min(ttl_ms / 2, 30_000.0) clamp_warned = False + rtt_max_ms = 0.0 + # Authoritative scheduling (spec v0.1.25.16): a response carrying + # remaining_ttl_ms is the server's own statement of the live + # lease, so the beat is scheduled from it directly. When the + # create response carried it, the first beat derives from it and + # no primed extension is spent. + lead_floor_ms: float | None = None + lead_anchor_ms = anchor_ms + authoritative = initial_remaining_ms is not None + if initial_remaining_ms is not None: + lead_floor_ms = float(initial_remaining_ms) + delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + else: + # Immediate first extension (fallback): with lead_min starting + # at 0 and no lease signal on the wire, any bounded first + # delay can outlive a policy-capped lease. Priming costs one + # extension; total protected runtime is unchanged. + delay_ms = 0.0 try: while True: await asyncio.sleep(delay_ms / 1000.0) @@ -739,13 +802,16 @@ async def heartbeat_loop() -> None: # cadence is the held delay — a transient failure must # not hot-loop. delay_ms = delay_ms or held_delay_ms - lead_min = grants_sum - (_lifecycle._now_mono_ms() - anchor_ms) - if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: - continue + if not authoritative: + lead_min = grants_sum - (_lifecycle._now_mono_ms() - anchor_ms) + if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: + continue try: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) pending_body = body + sent_ms = _lifecycle._now_mono_ms() response = await client.extend_reservation(reservation_id, body) + recv_ms = _lifecycle._now_mono_ms() if response.is_success: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") @@ -769,7 +835,18 @@ async def heartbeat_loop() -> None: last_success_ms = now_ms grants_sum += grant last_grant = grant - if grant <= 0 or ( + rtt_ms = recv_ms - sent_ms + rtt_max_ms = max(rtt_max_ms, rtt_ms) + remaining = response.get_body_attribute("remaining_ttl_ms") + if remaining is not None: + # Server-authoritative lease (spec v0.1.25.16): + # schedule from it directly; the heuristic arms + # below only serve servers that omit the field. + authoritative = True + lead_floor_ms = max(0.0, float(int(remaining)) - rtt_ms) + lead_anchor_ms = recv_ms + delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + elif grant <= 0 or ( grant < 0.9 * ttl_ms and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success ): @@ -782,6 +859,7 @@ async def heartbeat_loop() -> None: # seen across a skip-doubled gap lands here once, # but at the held cadence its grant/elapsed ratio # falls below 0.75 and cadence re-tightens. + authoritative = False delay_ms = held_delay_ms if not clamp_warned: clamp_warned = True @@ -790,6 +868,7 @@ async def heartbeat_loop() -> None: reservation_id, ) else: + authoritative = False delay_ms = min(max(grant / 2, 500.0), ttl_ms / 2) else: code = _extract_error_code(response) @@ -800,8 +879,22 @@ async def heartbeat_loop() -> None: ) return logger.warning("Async stream heartbeat failed: id=%s", reservation_id) + if authoritative and lead_floor_ms is not None: + # Retry inside the known lease: the failed extend may + # have been applied server-side, so the SAME body (same + # idempotency key) is retried at a bounded fraction of + # the remaining lead. + lead_now = max(0.0, lead_floor_ms - (_lifecycle._now_mono_ms() - lead_anchor_ms)) + delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) except Exception: logger.warning("Async stream heartbeat error: id=%s", reservation_id, exc_info=True) + if authoritative and lead_floor_ms is not None: + # Retry inside the known lease: the failed extend may + # have been applied server-side, so the SAME body (same + # idempotency key) is retried at a bounded fraction of + # the remaining lead. + lead_now = max(0.0, lead_floor_ms - (_lifecycle._now_mono_ms() - lead_anchor_ms)) + delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) except asyncio.CancelledError: return diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index d184123..9e64da2 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -39,10 +39,14 @@ def _config() -> CyclesConfig: ) -def _extend_ok(expires_at_ms: int | None) -> CyclesResponse: +def _extend_ok( + expires_at_ms: int | None, remaining_ttl_ms: int | None = None, +) -> CyclesResponse: body: dict[str, Any] = {"status": "ACTIVE"} if expires_at_ms is not None: body["expires_at_ms"] = expires_at_ms + if remaining_ttl_ms is not None: + body["remaining_ttl_ms"] = remaining_ttl_ms return CyclesResponse.success(200, body) @@ -88,6 +92,7 @@ def _run_sync_beats( beats: int, ttl: int = TTL, ctx: MagicMock | None = None, + initial_remaining_ms: int | None = None, ) -> list[float]: """Drive the sync heartbeat for `beats` iterations, advancing the fake clock by the beat interval on every wait. Returns the wait timeouts.""" @@ -105,7 +110,9 @@ def wait(timeout: float | None = None) -> bool: stop = threading.Event() stop.wait = wait # type: ignore[method-assign] - thread = lifecycle._start_heartbeat("rsv_1", ttl, ctx or _ctx(), stop) + thread = lifecycle._start_heartbeat( + "rsv_1", ttl, ctx or _ctx(), stop, initial_remaining_ms, + ) assert thread is not None thread.join(timeout=5) assert not thread.is_alive() @@ -424,6 +431,97 @@ def wait(timeout: float | None = None) -> bool: # Stops at the 410 — remaining beats never call extend. assert client.extend_reservation.call_count == 5 + def test_sync_stream_field_mode_cycle( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Authoritative mode in the sync stream heartbeat: create field + # drives the first delay; 503 and exception retries stay bounded. + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + client = MagicMock() + client._config = _config() + client.extend_reservation.side_effect = [ + _extend_ok(None, remaining_ttl_ms=60_000), + CyclesResponse.http_error(503, "unavailable"), + ConnectionError("down"), + _extend_ok(None, remaining_ttl_ms=60_000), + ] + stream = StreamReservation( + client, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=TTL, + ) + stream._reservation_id = "rsv_1" + stream._ctx = _ctx() + stream._initial_remaining = 60_000 + calls = {"n": 0} + timeouts: list[float] = [] + + def wait(timeout: float | None = None) -> bool: + timeouts.append(timeout or 0.0) + calls["n"] += 1 + if calls["n"] > 4: + return True + clock.t += (timeout or 0.0) * 1000.0 + return False + + stream._heartbeat_stop.wait = wait # type: ignore[method-assign] + thread = stream._start_heartbeat() + assert thread is not None + thread.join(timeout=5) + + assert client.extend_reservation.call_count == 4 + assert timeouts[0] == 59.0 + assert timeouts[2] == 1.0 + assert timeouts[3] == 1.0 + + @pytest.mark.asyncio + async def test_async_stream_field_mode_cycle( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + client = AsyncMock() + client._config = _config() + client.extend_reservation.side_effect = [ + _extend_ok(None, remaining_ttl_ms=60_000), + CyclesResponse.http_error(503, "unavailable"), + ConnectionError("down"), + _extend_ok(None, remaining_ttl_ms=60_000), + ] + stream = AsyncStreamReservation( + client, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=TTL, + ) + stream._reservation_id = "rsv_1" + stream._ctx = _ctx() + stream._initial_remaining = 60_000 + count = 0 + sleeps: list[float] = [] + + async def fake_sleep(s: float) -> None: + nonlocal count + sleeps.append(s) + count += 1 + if count > 4: + raise asyncio.CancelledError + clock.t += s * 1000.0 + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + task = stream._start_heartbeat() + assert task is not None + await task + + assert client.extend_reservation.await_count == 4 + assert sleeps[0] == 59.0 + assert sleeps[2] == 1.0 + assert sleeps[3] == 1.0 + @pytest.mark.asyncio async def test_async_stream_lead_estimate_pattern( self, monkeypatch: pytest.MonkeyPatch, @@ -501,6 +599,186 @@ async def fake_sleep(s: float) -> None: assert client.extend_reservation.await_count == 5 +# --------------------------------------------------------------------------- +# Server-authoritative scheduling (remaining_ttl_ms, spec v0.1.25.16) +# --------------------------------------------------------------------------- + + +class TestAuthoritativeScheduling: + def test_create_remaining_drives_first_beat_and_steady_cadence( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # remaining=60000, rtt=0 → reserve = min(30000, max(1000, 0)) = 1000, + # first delay 59000ms. Every extend echoes the field, so the cadence + # holds at 59s and the heuristic lead_min skip NEVER fires even + # though accumulated fallback grants would trip it. + lifecycle, client = _make_sync() + client.extend_reservation.return_value = _extend_ok(None, remaining_ttl_ms=60_000) + + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=4, initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 4 + assert timeouts == [59.0, 59.0, 59.0, 59.0, 59.0] + + def test_capped_create_first_beat_lands_inside_small_lease( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # 24h request, tenant caps to 1s: remaining=1000 → reserve = + # min(500, 1000) = 500 → first beat at 500ms, inside the real lease. + # This is the exact case that motivated the wire field. + lifecycle, client = _make_sync() + client.extend_reservation.return_value = _extend_ok(None, remaining_ttl_ms=1_000) + + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=1, + ttl=86_400_000, initial_remaining_ms=1_000, + ) + + assert timeouts[0] == 0.5 + assert client.extend_reservation.call_count == 1 + + def test_lead_clamp_server_with_field_no_warn_no_collapse( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + # A maximum-lead-clamping server that DOES emit remaining_ttl_ms: + # expiry echoes elapsed (grant ≈ elapsed, the heuristic's worst + # case) but the field carries the true lease → authoritative arm + # schedules cleanly and the clamp warning never fires. + clock = FakeClock() + lifecycle, client = _make_sync() + + def clamped_extend(rid: str, body: dict[str, Any]) -> CyclesResponse: + return _extend_ok(INITIAL_EXPIRY + int(clock.t), remaining_ttl_ms=15_000) + + client.extend_reservation.side_effect = clamped_extend + + with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): + timeouts = _run_sync_beats( + lifecycle, clock, monkeypatch, beats=3, initial_remaining_ms=15_000, + ) + + assert client.extend_reservation.call_count == 3 + # remaining=15000 → reserve min(7500, 1000) = 1000 → delay 14s. + assert timeouts == [14.0, 14.0, 14.0, 14.0] + assert not [r for r in caplog.records if "clamp lease lead" in r.message] + + def test_field_disappearing_mid_flight_resumes_heuristic( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Beat 1's response lacks the field → the v2.3+band heuristic takes + # over seamlessly from its maintained bookkeeping. + lifecycle, client = _make_sync() + client.extend_reservation.side_effect = [ + _extend_ok(INITIAL_EXPIRY + TTL), # no field: fallback + _extend_ok(INITIAL_EXPIRY + 2 * TTL), + ] + + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=2, initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 2 + # First delay authoritative (59s); after the fieldless response the + # normal-regime cadence (ttl/2 = 30s) applies. + assert timeouts[0] == 59.0 + assert timeouts[1] == 30.0 + + def test_transient_failure_in_field_mode_retries_bounded_same_key( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # At the scheduled beat the remaining lead estimate is ~the retry + # reserve (1000ms) → retry delay clamp(lead/4, 1s, 30s) = 1s, with + # the SAME idempotency key; the following success uses a fresh key. + lifecycle, client = _make_sync() + client.extend_reservation.side_effect = [ + CyclesResponse.http_error(503, "unavailable"), + _extend_ok(None, remaining_ttl_ms=60_000), + ] + + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=2, initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 2 + assert timeouts[0] == 59.0 + assert timeouts[1] == 1.0 + bodies = [c.args[1] for c in client.extend_reservation.call_args_list] + assert bodies[0]["idempotency_key"] == bodies[1]["idempotency_key"] + + def test_exception_in_field_mode_retries_bounded_same_key( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.side_effect = [ + ConnectionError("down"), + _extend_ok(None, remaining_ttl_ms=60_000), + ] + + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=2, initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 2 + assert timeouts[1] == 1.0 + bodies = [c.args[1] for c in client.extend_reservation.call_args_list] + assert bodies[0]["idempotency_key"] == bodies[1]["idempotency_key"] + + @pytest.mark.asyncio + async def test_async_field_mode_full_cycle( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Covers the async authoritative arms: initial field delay, 503 + # bounded retry, exception bounded retry, and field-driven success. + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + lifecycle, client = _make_async() + client.extend_reservation.side_effect = [ + _extend_ok(None, remaining_ttl_ms=60_000), + CyclesResponse.http_error(503, "unavailable"), + ConnectionError("down"), + _extend_ok(None, remaining_ttl_ms=60_000), + ] + count = 0 + sleeps: list[float] = [] + + async def fake_sleep(s: float) -> None: + nonlocal count + sleeps.append(s) + count += 1 + if count > 4: + raise asyncio.CancelledError + clock.t += s * 1000.0 + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + task = lifecycle._start_heartbeat("rsv_1", TTL, _ctx(), 60_000) + assert task is not None + await task + + assert client.extend_reservation.await_count == 4 + assert sleeps[0] == 59.0 # from the create field + assert sleeps[2] == 1.0 # bounded 503 retry inside the known lease + assert sleeps[3] == 1.0 # bounded exception retry + + def test_create_response_model_parses_remaining(self) -> None: + from runcycles.models import ReservationCreateResponse + + body = { + "decision": "ALLOW", + "reservation_id": "rsv_1", + "affected_scopes": ["tenant:acme"], + "expires_at_ms": 1_000_000, + "remaining_ttl_ms": 10_000, + } + parsed = ReservationCreateResponse.model_validate(body) + assert parsed.remaining_ttl_ms == 10_000 + body.pop("remaining_ttl_ms") + assert ReservationCreateResponse.model_validate(body).remaining_ttl_ms is None + + # --------------------------------------------------------------------------- # Date header accessor (kept as a general response accessor; the heartbeat # no longer consumes it — RFC 9110 §6.6.1 makes it a different clock). From d652caa876438dd89a07b907a3bcc2741b84ceda Mon Sep 17 00:00:00 2001 From: Albert Mavashev Date: Tue, 28 Jul 2026 08:39:22 -0400 Subject: [PATCH 7/9] feat: implement the hardened normative heartbeat algorithm Aligns the field mode with the settled HEARTBEAT GUIDANCE on cycles-protocol#148 (head dd60c27), which hardened the round-5 draft: - Scheduling: per-attempt monotonic rtt (max tracked); lead_floor = max(0, remaining_ttl_ms - rtt); attempt_budget = max(request_timeout_budget, 1s, 2*rtt_max) with the budget wired from the SDK's enforced httpx timeouts (connect + read + write = 12s by default); safety_margin = max(1s, 2*rtt_max); retry_reserve = 2*attempt_budget + safety_margin; next beat at lead_floor - retry_reserve after response receipt, recomputed on every field-bearing schema-valid HTTP 200. - Success predicate: only a schema-valid 200 counts; an ambiguous non-200 2xx is never applied and recovers with the SAME key. - Recovery: retry_window = lead_estimate - attempt_budget - safety_margin, unclamped; window < 0 stops and surfaces; recovery repeats with the same key while the freshly recomputed window shrinks (progress guard against zero-time loops; window 0 permits one immediate retry); 429 honors delta-seconds Retry-After only inside the window; other 4xx stop without key rotation. - Zero-delay guard: a success whose lease cannot hold the reserve permits ONE immediate fresh attempt, then the heartbeat stops and surfaces that the lease is shorter than the retry-safety budget. - Create rtt is now measured around the create call and feeds the first beat's lead_floor; the fallback-only backstop no longer clobbers meaningful authoritative zero delays. The v2.3+band heuristic is unchanged as the fieldless fallback; all its tests pass unedited. New coverage: scheduler edge cases, steady 35s cadence, shrinking recovery windows (6250 -> 4687.5 -> 1062.5 -> 0 -> stop), ambiguous-2xx same-key recovery, 429 within/exceeding window, 4xx stop, zero-delay guard, and a stop/recovery matrix across all four loops. 568 tests, 100% coverage, ruff + mypy clean. --- AUDIT.md | 33 +-- CHANGELOG.md | 2 +- runcycles/lifecycle.py | 339 ++++++++++++++++++++++------- runcycles/streaming.py | 251 +++++++++++++++------- tests/test_heartbeat.py | 460 ++++++++++++++++++++++++++++++++++++---- 5 files changed, 874 insertions(+), 211 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index fffef17..650e5d7 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -16,20 +16,25 @@ ## 2026-07-28 — Server-authoritative heartbeat via remaining_ttl_ms (v0.5.1, same PR) Spec review round 5 proved the fallback heuristic's regime detection -undecidable (sticky band window [0.75×min(ttl/2,30s), 0.9×ttl) — e.g. -ttl 24s / cap 10s / held 12s / ratio 0.833 forever, lease erodes to -lapse) and immediate priming schedule-dependent under maximum-lead -clamping. Adopted the reviewer's fix: spec v0.1.25.16 adds -remaining_ttl_ms to create+extend responses (cycles-server v0.1.25.59 -emits it; extend replays recompute it fresh). When present, the SDK -schedules from it directly — lead_floor = max(0, remaining − rtt), next -beat at lead_floor − min(lead_floor/2, max(1s, 2×rtt_max)) from response -receipt, recomputed per field-bearing response, lead_min skip bypassed, -no primed first extension (create field drives the first delay), bounded -same-key retry clamp(lead/4, 1s, 30s) on transient failures. Bookkeeping -keeps running so the v2.3+band heuristic (now explicitly best-effort -fallback) resumes seamlessly if the field disappears; fieldless servers -see unchanged behavior. 543 tests pass at 100% coverage. +undecidable (sticky band window [0.75×min(ttl/2,30s), 0.9×ttl)) and +immediate priming schedule-dependent under maximum-lead clamping, so the +protocol adopted remaining_ttl_ms on create+extend responses (spec +v0.1.25.16; cycles-server v0.1.25.59 emits it, recomputed fresh on +idempotent replays and excluded from evidence). The SDK implements the +spec's hardened NORMATIVE algorithm when the field is present: per-attempt +rtt, lead_floor = max(0, remaining − rtt), retry_reserve = +2×max(request_timeout_budget, 1s, 2×rtt_max) + max(1s, 2×rtt_max) (the +enforced httpx timeouts define the budget), next beat at lead_floor − +retry_reserve from response receipt, recomputed per schema-valid 200 +(non-200 2xx is ambiguous → same-key recovery); recovery repeats while +retry_window = lead_estimate − attempt_budget − safety_margin stays +positive with a no-progress guard, 429 Retry-After honored only inside the +window, other 4xx stop without key rotation; a lease that cannot hold the +reserve gets one immediate fresh attempt then stop-and-surface; lead_min +skip bypassed; no primed extension when the create carries the field. +Bookkeeping keeps running so the v2.3+band heuristic (now explicitly +best-effort fallback) resumes seamlessly if the field disappears; +fieldless servers see unchanged behavior. 568 tests pass at 100% coverage. ## 2026-07-27 — Heartbeat conservative-lead redesign + actual_source marker (v0.5.1) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9784376..a9c85f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **Server-authoritative heartbeat scheduling via `remaining_ttl_ms`** (spec v0.1.25.16, cycles-protocol#148): when a create or extend response carries `remaining_ttl_ms` — the server's own statement of the live lease, measured on the clock that stamps `expires_at_ms` — the heartbeat schedules directly from it: `lead_floor = max(0, remaining − rtt)`, next beat at `lead_floor − min(lead_floor/2, max(1s, 2×max observed rtt))` after response receipt, recomputed on every field-bearing response; the heuristic `lead_min` skip is bypassed (scheduling is exact) and no primed first extension is spent when the create response carries the field — a 24h request silently capped to a 1s lease gets its first beat at 500ms, inside the real lease. Transient failures in this mode retry with the same idempotency key after `clamp(current_lead/4, 1s, 30s)`. Grants/lead bookkeeping keeps running in both modes, so the heuristic below takes over seamlessly if the field disappears; servers that never emit it get the unchanged fallback behavior. The heuristic (conservative lead lower bound + regime band) is now explicitly a **best-effort fallback**: spec review round 5 proved regime detection from `(grant, elapsed)` observables undecidable in general — the band's sticky window `grant ∈ [0.75×min(ttl/2, 30s), 0.9×ttl)` misclassifies permanently — which is precisely why the wire field exists. +- **Server-authoritative heartbeat scheduling via `remaining_ttl_ms`** (spec v0.1.25.16, cycles-protocol#148): when a create or extend response carries `remaining_ttl_ms` — the server's own statement of the live lease, measured on the clock that stamps `expires_at_ms` — the heartbeat runs the spec's NORMATIVE algorithm: per-attempt monotonic rtt (max tracked), `lead_floor = max(0, remaining − rtt)`, `attempt_budget = max(request_timeout_budget, 1s, 2×max rtt)` (the SDK's enforced httpx timeouts: connect + read + write), `safety_margin = max(1s, 2×max rtt)`, `retry_reserve = 2×attempt_budget + safety_margin`, next beat at `lead_floor − retry_reserve` after response receipt, recomputed on every field-bearing schema-valid HTTP 200 — never accumulated from expiry differences; the heuristic `lead_min` skip is bypassed and no primed first extension is spent (the create response's field drives the first delay). Only a schema-valid 200 counts as an observed success: an ambiguous non-200 2xx is recovered with the SAME idempotency key. Transient failures (timeout/connection/5xx/429/ambiguous 2xx) recover while the freshly recomputed `retry_window = lead_estimate − attempt_budget − safety_margin` stays positive (progress-guarded against zero-time loops; window < 0 stops and surfaces); a 429's `Retry-After` (delta-seconds) is honored only inside the window, never re-invented earlier; other 4xx stop without key rotation. A success whose lease cannot hold the retry reserve permits ONE immediate fresh attempt, then the heartbeat stops and surfaces that the lease is shorter than the retry-safety budget. Grants/lead bookkeeping keeps running in both modes, so the heuristic below takes over seamlessly if the field disappears; fieldless servers see unchanged fallback behavior. The heuristic (conservative lead lower bound + regime band) is now explicitly a **best-effort fallback**: spec review round 5 proved regime detection from `(grant, elapsed)` observables undecidable in general — the band's sticky window `grant ∈ [0.75×min(ttl/2, 30s), 0.9×ttl)` misclassifies permanently — which is precisely why the wire field exists. - `metadata.actual_source: "estimate"` is stamped on commits whose actual was structurally defaulted from the estimate (`@cycles` without an `actual` expression; streams with no recorded cost or a raised `cost_fn`), so audit evidence distinguishes measured spend from assumed spend. Defaults are unchanged; the marker flows into `/v1/events` recovery bodies via the shared metadata. ## [0.5.0] - 2026-07-27 diff --git a/runcycles/lifecycle.py b/runcycles/lifecycle.py index fbfd7aa..7fea460 100644 --- a/runcycles/lifecycle.py +++ b/runcycles/lifecycle.py @@ -220,12 +220,89 @@ def _now_mono_ms() -> float: _LEAD_TARGET_FACTOR = 1.5 -def _authoritative_delay_ms(lead_floor_ms: float, rtt_max_ms: float) -> float: - """Next-beat delay from a server-authoritative remaining lease - (remaining_ttl_ms, spec v0.1.25.16): schedule inside the lease while - keeping a retry reserve of min(lead/2, max(1s, 2x max observed rtt)).""" - reserve = min(lead_floor_ms / 2, max(1000.0, 2.0 * rtt_max_ms)) - return max(0.0, lead_floor_ms - reserve) +def _timeout_budget_ms(config: Any) -> float: + """The client's enforced upper bound for one complete extend attempt in + ms (connect + read + the fixed 5s write timeout) — the spec's + request_timeout_budget. The SDK always enforces finite httpx timeouts, + so the unknown/unbounded-timeout arm of the spec never applies here.""" + return (float(config.connect_timeout) + float(config.read_timeout) + 5.0) * 1000.0 + + +class _AuthoritativeScheduler: + """Field-mode heartbeat scheduling per the NORMATIVE algorithm in the + spec's HEARTBEAT GUIDANCE (v0.1.25.16). All methods return the next + delay in ms, or ``None`` when the spec requires the heartbeat to stop + and surface. State: max observed rtt, the lead floor established by the + last schema-valid response, a zero-delay streak (a success that cannot + hold the retry reserve permits ONE immediate fresh attempt, then stop), + and the last failure's retry window (recovery may repeat with the same + idempotency key only while the freshly recomputed window shrinks).""" + + def __init__(self, timeout_budget_ms: float) -> None: + self._timeout_budget_ms = timeout_budget_ms + self._rtt_max_ms = 0.0 + self._lead_floor_ms: float | None = None + self._lead_anchor_ms: float | None = None + self._zero_streak = 0 + self._prev_fail_window: float | None = None + + def _attempt_budget_ms(self) -> float: + return max(self._timeout_budget_ms, 1000.0, 2.0 * self._rtt_max_ms) + + def _safety_margin_ms(self) -> float: + return max(1000.0, 2.0 * self._rtt_max_ms) + + def on_valid_success( + self, remaining_ms: int, rtt_ms: float, now_ms: float, + ) -> float | None: + """Schema-valid HTTP 200 carrying remaining_ttl_ms. retry_reserve = + 2×attempt_budget + safety_margin covers one failed attempt, one + same-key retry, and margin.""" + self._rtt_max_ms = max(self._rtt_max_ms, rtt_ms) + self._prev_fail_window = None + self._lead_floor_ms = max(0.0, float(remaining_ms) - max(rtt_ms, 0.0)) + self._lead_anchor_ms = now_ms + reserve = 2.0 * self._attempt_budget_ms() + self._safety_margin_ms() + delay = self._lead_floor_ms - reserve + if delay > 0: + self._zero_streak = 0 + return delay + # The lease cannot hold the retry reserve: one immediate fresh + # attempt (new key) is permitted — an additive-delta server may + # establish positive lead — then the client must stop rather than + # burn a maximum-lead server's extension budget in a tight loop. + self._zero_streak += 1 + if self._zero_streak >= 2: + return None + return 0.0 + + def lead_estimate_ms(self, now_ms: float) -> float: + if self._lead_floor_ms is None or self._lead_anchor_ms is None: + return 0.0 + return max(0.0, self._lead_floor_ms - (now_ms - self._lead_anchor_ms)) + + def on_transient_failure( + self, + now_ms: float, + rate_limited: bool = False, + retry_after_ms: int | None = None, + ) -> float | None: + """Timeout / connection error / 5xx / 429 / ambiguous 2xx. The + retry_window is recomputed from the same last schema-valid response; + recovery repeats only while it shrinks (progress guard), and a 429 + may only be honored inside the window — never re-invented earlier.""" + lead_est = self.lead_estimate_ms(now_ms) + window = lead_est - self._attempt_budget_ms() - self._safety_margin_ms() + if window < 0: + return None + if self._prev_fail_window is not None and window >= self._prev_fail_window: + return None # no progress between consecutive failures + self._prev_fail_window = window + if rate_limited: + if retry_after_ms is None or retry_after_ms < 0 or retry_after_ms > window: + return None + return float(retry_after_ms) + return min(30_000.0, lead_est / 4.0, window) def _build_extend_body(ttl_ms: int) -> dict[str, Any]: @@ -324,7 +401,9 @@ def execute( logger.debug("Creating reservation: body=%s", create_body) res_t1 = time.monotonic() + _create_sent_ms = _now_mono_ms() res_response = self._client.create_reservation(create_body) + _create_rtt_ms = _now_mono_ms() - _create_sent_ms if not res_response.is_success: logger.error("Reservation failed: response=%s", res_response) @@ -385,7 +464,8 @@ def execute( # Start heartbeat heartbeat_stop = threading.Event() heartbeat_thread = self._start_heartbeat( - reservation_id, cfg.ttl_ms, ctx, heartbeat_stop, res_result.remaining_ttl_ms, + reservation_id, cfg.ttl_ms, ctx, heartbeat_stop, + res_result.remaining_ttl_ms, _create_rtt_ms, ) try: @@ -508,6 +588,7 @@ def _start_heartbeat( ctx: CyclesContext, stop_event: threading.Event, initial_remaining_ms: int | None = None, + initial_rtt_ms: float | None = None, ) -> threading.Thread | None: if ttl_ms <= 0: return None @@ -536,18 +617,25 @@ def heartbeat_loop() -> None: last_success_ms = anchor_ms held_delay_ms = min(ttl_ms / 2, 30_000.0) clamp_warned = False - rtt_max_ms = 0.0 - # Authoritative scheduling (spec v0.1.25.16): a response carrying - # remaining_ttl_ms is the server's own statement of the live - # lease, so the beat is scheduled from it directly. When the - # create response carried it, the first beat derives from it and - # no primed extension is spent. - lead_floor_ms: float | None = None - lead_anchor_ms = anchor_ms + # Authoritative scheduling (spec v0.1.25.16 PRIMARY ALGORITHM): + # a schema-valid 200 carrying remaining_ttl_ms drives scheduling + # exactly; the measured-grant heuristic below is the + # NON-NORMATIVE fallback for servers that omit the field. + sched = _AuthoritativeScheduler(_timeout_budget_ms(self._client._config)) authoritative = initial_remaining_ms is not None if initial_remaining_ms is not None: - lead_floor_ms = float(initial_remaining_ms) - delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + first_delay = sched.on_valid_success( + initial_remaining_ms, initial_rtt_ms or 0.0, anchor_ms, + ) + # Unreachable on the first scheduler call (the zero-delay + # streak cannot be exhausted yet); kept as a typed guard. + if first_delay is None: # pragma: no cover + logger.warning( + "Heartbeat not started: lease shorter than the retry-safety budget: id=%s", + reservation_id, + ) + return + delay_ms = first_delay else: # Immediate first extension (fallback): with lead_min starting # at 0 and no lease signal on the wire, any bounded first @@ -555,9 +643,12 @@ def heartbeat_loop() -> None: # extension; total protected runtime is unchanged. delay_ms = 0.0 while not stop_event.wait(timeout=delay_ms / 1000.0): - # After the primed (delay-0) first beat, the baseline cadence - # is the held delay — a transient failure must not hot-loop. - delay_ms = delay_ms or held_delay_ms + if not authoritative: + # After the primed (delay-0) first beat, the baseline + # cadence is the held delay — a transient failure must + # not hot-loop. (Authoritative zero delays are meaningful: + # the one-immediate-attempt guards bound them.) + delay_ms = delay_ms or held_delay_ms if not authoritative: lead_min = grants_sum - (_now_mono_ms() - anchor_ms) if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: @@ -568,7 +659,22 @@ def heartbeat_loop() -> None: sent_ms = _now_mono_ms() response = self._client.extend_reservation(reservation_id, body) recv_ms = _now_mono_ms() - if response.is_success: + if response.is_success and authoritative and response.status != 200: + # Ambiguous non-200 2xx in authoritative mode: NOT an + # observed success (spec) — same-key recovery. + logger.warning( + "Heartbeat ambiguous 2xx (status=%d): id=%s", + response.status, reservation_id, + ) + nxt = sched.on_transient_failure(_now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt + elif response.is_success: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: @@ -591,16 +697,21 @@ def heartbeat_loop() -> None: grants_sum += grant last_grant = grant rtt_ms = recv_ms - sent_ms - rtt_max_ms = max(rtt_max_ms, rtt_ms) remaining = response.get_body_attribute("remaining_ttl_ms") - if remaining is not None: - # Server-authoritative lease (spec v0.1.25.16): - # schedule from it directly; the heuristic arms - # below only serve servers that omit the field. + if response.status == 200 and remaining is not None: + # Server-authoritative lease (spec v0.1.25.16 + # PRIMARY ALGORITHM): schedule from this response + # alone; the heuristic arms below only serve + # servers that omit the field. authoritative = True - lead_floor_ms = max(0.0, float(int(remaining)) - rtt_ms) - lead_anchor_ms = recv_ms - delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + nxt = sched.on_valid_success(int(remaining), rtt_ms, recv_ms) + if nxt is None: + logger.warning( + "Heartbeat stopping: lease shorter than the retry-safety budget: id=%s", + reservation_id, + ) + return + delay_ms = nxt elif grant <= 0 or ( grant < 0.9 * ttl_ms and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success @@ -635,22 +746,38 @@ def heartbeat_loop() -> None: ) return logger.warning("Heartbeat extend failed: id=%s, status=%d", reservation_id, response.status) - if authoritative and lead_floor_ms is not None: - # Retry inside the known lease: the failed extend may - # have been applied server-side, so the SAME body (same - # idempotency key) is retried at a bounded fraction of - # the remaining lead. - lead_now = max(0.0, lead_floor_ms - (_now_mono_ms() - lead_anchor_ms)) - delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) + if authoritative: + rate_limited = response.status == 429 or code == "LIMIT_EXCEEDED" + if not rate_limited and 400 <= response.status < 500: + # Unrecoverable request/auth failure (spec): never + # rotate the key on an unchanged request. + logger.warning( + "Heartbeat stopping on client error (%s, status=%d): id=%s", + code, response.status, reservation_id, + ) + return + retry_after = response.retry_after_ms_header if rate_limited else None + nxt = sched.on_transient_failure( + _now_mono_ms(), rate_limited=rate_limited, retry_after_ms=retry_after, + ) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt except Exception: logger.warning("Heartbeat extend error: id=%s", reservation_id, exc_info=True) - if authoritative and lead_floor_ms is not None: - # Retry inside the known lease: the failed extend may - # have been applied server-side, so the SAME body (same - # idempotency key) is retried at a bounded fraction of - # the remaining lead. - lead_now = max(0.0, lead_floor_ms - (_now_mono_ms() - lead_anchor_ms)) - delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) + if authoritative: + nxt = sched.on_transient_failure(_now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt t = threading.Thread(target=heartbeat_loop, daemon=True, name=f"cycles-heartbeat-{reservation_id[:12]}") t.start() @@ -679,7 +806,9 @@ async def execute( logger.debug("Estimated usage: estimate=%d", estimate) create_body = _build_reservation_body(cfg, estimate, self._default_subject, args, kwargs) + _create_sent_ms = _now_mono_ms() res_response = await self._client.create_reservation(create_body) + _create_rtt_ms = _now_mono_ms() - _create_sent_ms if not res_response.is_success: raise _build_protocol_exception("Failed to create reservation", res_response) @@ -727,7 +856,8 @@ async def execute( _set_context(ctx) heartbeat_task = self._start_heartbeat( - reservation_id, cfg.ttl_ms, ctx, res_result.remaining_ttl_ms, + reservation_id, cfg.ttl_ms, ctx, + res_result.remaining_ttl_ms, _create_rtt_ms, ) try: @@ -845,6 +975,7 @@ def _start_heartbeat( ttl_ms: int, ctx: CyclesContext, initial_remaining_ms: int | None = None, + initial_rtt_ms: float | None = None, ) -> asyncio.Task[None] | None: if ttl_ms <= 0: return None @@ -859,18 +990,25 @@ async def heartbeat_loop() -> None: last_success_ms = anchor_ms held_delay_ms = min(ttl_ms / 2, 30_000.0) clamp_warned = False - rtt_max_ms = 0.0 - # Authoritative scheduling (spec v0.1.25.16): a response carrying - # remaining_ttl_ms is the server's own statement of the live - # lease, so the beat is scheduled from it directly. When the - # create response carried it, the first beat derives from it and - # no primed extension is spent. - lead_floor_ms: float | None = None - lead_anchor_ms = anchor_ms + # Authoritative scheduling (spec v0.1.25.16 PRIMARY ALGORITHM): + # a schema-valid 200 carrying remaining_ttl_ms drives scheduling + # exactly; the measured-grant heuristic below is the + # NON-NORMATIVE fallback for servers that omit the field. + sched = _AuthoritativeScheduler(_timeout_budget_ms(self._client._config)) authoritative = initial_remaining_ms is not None if initial_remaining_ms is not None: - lead_floor_ms = float(initial_remaining_ms) - delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + first_delay = sched.on_valid_success( + initial_remaining_ms, initial_rtt_ms or 0.0, anchor_ms, + ) + # Unreachable on the first scheduler call (the zero-delay + # streak cannot be exhausted yet); kept as a typed guard. + if first_delay is None: # pragma: no cover + logger.warning( + "Heartbeat not started: lease shorter than the retry-safety budget: id=%s", + reservation_id, + ) + return + delay_ms = first_delay else: # Immediate first extension (fallback): with lead_min starting # at 0 and no lease signal on the wire, any bounded first @@ -880,10 +1018,13 @@ async def heartbeat_loop() -> None: try: while True: await asyncio.sleep(delay_ms / 1000.0) - # After the primed (delay-0) first beat, the baseline - # cadence is the held delay — a transient failure must - # not hot-loop. - delay_ms = delay_ms or held_delay_ms + if not authoritative: + # After the primed (delay-0) first beat, the baseline + # cadence is the held delay — a transient failure must + # not hot-loop. (Authoritative zero delays are + # meaningful: the one-immediate-attempt guards bound + # them.) + delay_ms = delay_ms or held_delay_ms if not authoritative: lead_min = grants_sum - (_now_mono_ms() - anchor_ms) if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: @@ -894,7 +1035,22 @@ async def heartbeat_loop() -> None: sent_ms = _now_mono_ms() response = await self._client.extend_reservation(reservation_id, body) recv_ms = _now_mono_ms() - if response.is_success: + if response.is_success and authoritative and response.status != 200: + # Ambiguous non-200 2xx in authoritative mode: NOT an + # observed success (spec) — same-key recovery. + logger.warning( + "Heartbeat ambiguous 2xx (status=%d): id=%s", + response.status, reservation_id, + ) + nxt = sched.on_transient_failure(_now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt + elif response.is_success: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: @@ -917,16 +1073,21 @@ async def heartbeat_loop() -> None: grants_sum += grant last_grant = grant rtt_ms = recv_ms - sent_ms - rtt_max_ms = max(rtt_max_ms, rtt_ms) remaining = response.get_body_attribute("remaining_ttl_ms") - if remaining is not None: - # Server-authoritative lease (spec v0.1.25.16): - # schedule from it directly; the heuristic arms - # below only serve servers that omit the field. + if response.status == 200 and remaining is not None: + # Server-authoritative lease (spec v0.1.25.16 + # PRIMARY ALGORITHM): schedule from this response + # alone; the heuristic arms below only serve + # servers that omit the field. authoritative = True - lead_floor_ms = max(0.0, float(int(remaining)) - rtt_ms) - lead_anchor_ms = recv_ms - delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + nxt = sched.on_valid_success(int(remaining), rtt_ms, recv_ms) + if nxt is None: + logger.warning( + "Heartbeat stopping: lease shorter than the retry-safety budget: id=%s", + reservation_id, + ) + return + delay_ms = nxt elif grant <= 0 or ( grant < 0.9 * ttl_ms and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success @@ -960,22 +1121,38 @@ async def heartbeat_loop() -> None: ) return logger.warning("Heartbeat extend failed: id=%s", reservation_id) - if authoritative and lead_floor_ms is not None: - # Retry inside the known lease: the failed extend may - # have been applied server-side, so the SAME body (same - # idempotency key) is retried at a bounded fraction of - # the remaining lead. - lead_now = max(0.0, lead_floor_ms - (_now_mono_ms() - lead_anchor_ms)) - delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) + if authoritative: + rate_limited = response.status == 429 or code == "LIMIT_EXCEEDED" + if not rate_limited and 400 <= response.status < 500: + # Unrecoverable request/auth failure (spec): never + # rotate the key on an unchanged request. + logger.warning( + "Heartbeat stopping on client error (%s, status=%d): id=%s", + code, response.status, reservation_id, + ) + return + retry_after = response.retry_after_ms_header if rate_limited else None + nxt = sched.on_transient_failure( + _now_mono_ms(), rate_limited=rate_limited, retry_after_ms=retry_after, + ) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt except Exception: logger.warning("Heartbeat extend error: id=%s", reservation_id, exc_info=True) - if authoritative and lead_floor_ms is not None: - # Retry inside the known lease: the failed extend may - # have been applied server-side, so the SAME body (same - # idempotency key) is retried at a bounded fraction of - # the remaining lead. - lead_now = max(0.0, lead_floor_ms - (_now_mono_ms() - lead_anchor_ms)) - delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) + if authoritative: + nxt = sched.on_transient_failure(_now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt except asyncio.CancelledError: return diff --git a/runcycles/streaming.py b/runcycles/streaming.py index e825eee..dcc57a5 100644 --- a/runcycles/streaming.py +++ b/runcycles/streaming.py @@ -19,12 +19,13 @@ from runcycles.lifecycle import ( _LEAD_TARGET_FACTOR, _PERMANENT_EXTEND_CODES, - _authoritative_delay_ms, + _AuthoritativeScheduler, _build_commit_body, _build_event_fallback_body, _build_extend_body, _build_protocol_exception, _build_release_body, + _timeout_budget_ms, ) from runcycles.models import ( Action, @@ -179,6 +180,7 @@ def __init__( self._usage = StreamUsage() self._reservation_id: str | None = None self._initial_remaining: int | None = None + self._create_rtt: float | None = None self._caps: Caps | None = None self._decision: Decision = Decision.ALLOW self._ctx: CyclesContext | None = None @@ -218,7 +220,9 @@ def __enter__(self) -> StreamReservation: self._grace_period_ms, ) + _create_sent_ms = _lifecycle._now_mono_ms() response = self._client.create_reservation(body) + _create_rtt_ms = _lifecycle._now_mono_ms() - _create_sent_ms if not response.is_success: raise _build_protocol_exception("Failed to create reservation", response) @@ -236,6 +240,7 @@ def __enter__(self) -> StreamReservation: self._reservation_id = result.reservation_id self._initial_remaining = result.remaining_ttl_ms + self._create_rtt = _create_rtt_ms self._decision = result.decision self._caps = result.caps @@ -389,21 +394,29 @@ def heartbeat_loop() -> None: last_grant: float | None = None pending_body: dict[str, Any] | None = None initial_remaining_ms = self._initial_remaining + initial_rtt_ms = self._create_rtt last_success_ms = anchor_ms held_delay_ms = min(ttl_ms / 2, 30_000.0) clamp_warned = False - rtt_max_ms = 0.0 - # Authoritative scheduling (spec v0.1.25.16): a response carrying - # remaining_ttl_ms is the server's own statement of the live - # lease, so the beat is scheduled from it directly. When the - # create response carried it, the first beat derives from it and - # no primed extension is spent. - lead_floor_ms: float | None = None - lead_anchor_ms = anchor_ms + # Authoritative scheduling (spec v0.1.25.16 PRIMARY ALGORITHM): + # a schema-valid 200 carrying remaining_ttl_ms drives scheduling + # exactly; the measured-grant heuristic below is the + # NON-NORMATIVE fallback for servers that omit the field. + sched = _AuthoritativeScheduler(_timeout_budget_ms(self._client._config)) authoritative = initial_remaining_ms is not None if initial_remaining_ms is not None: - lead_floor_ms = float(initial_remaining_ms) - delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + first_delay = sched.on_valid_success( + initial_remaining_ms, initial_rtt_ms or 0.0, anchor_ms, + ) + # Unreachable on the first scheduler call (the zero-delay + # streak cannot be exhausted yet); kept as a typed guard. + if first_delay is None: # pragma: no cover + logger.warning( + "Heartbeat not started: lease shorter than the retry-safety budget: id=%s", + reservation_id, + ) + return + delay_ms = first_delay else: # Immediate first extension (fallback): with lead_min starting # at 0 and no lease signal on the wire, any bounded first @@ -411,9 +424,12 @@ def heartbeat_loop() -> None: # extension; total protected runtime is unchanged. delay_ms = 0.0 while not self._heartbeat_stop.wait(timeout=delay_ms / 1000.0): - # After the primed (delay-0) first beat, the baseline cadence - # is the held delay — a transient failure must not hot-loop. - delay_ms = delay_ms or held_delay_ms + if not authoritative: + # After the primed (delay-0) first beat, the baseline + # cadence is the held delay — a transient failure must + # not hot-loop. (Authoritative zero delays are meaningful: + # the one-immediate-attempt guards bound them.) + delay_ms = delay_ms or held_delay_ms if not authoritative: lead_min = grants_sum - (_lifecycle._now_mono_ms() - anchor_ms) if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: @@ -424,7 +440,22 @@ def heartbeat_loop() -> None: sent_ms = _lifecycle._now_mono_ms() response = self._client.extend_reservation(reservation_id, body) recv_ms = _lifecycle._now_mono_ms() - if response.is_success: + if response.is_success and authoritative and response.status != 200: + # Ambiguous non-200 2xx in authoritative mode: NOT an + # observed success (spec) — same-key recovery. + logger.warning( + "Heartbeat ambiguous 2xx (status=%d): id=%s", + response.status, reservation_id, + ) + nxt = sched.on_transient_failure(_lifecycle._now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt + elif response.is_success: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: @@ -448,16 +479,21 @@ def heartbeat_loop() -> None: grants_sum += grant last_grant = grant rtt_ms = recv_ms - sent_ms - rtt_max_ms = max(rtt_max_ms, rtt_ms) remaining = response.get_body_attribute("remaining_ttl_ms") - if remaining is not None: - # Server-authoritative lease (spec v0.1.25.16): - # schedule from it directly; the heuristic arms - # below only serve servers that omit the field. + if response.status == 200 and remaining is not None: + # Server-authoritative lease (spec v0.1.25.16 + # PRIMARY ALGORITHM): schedule from this response + # alone; the heuristic arms below only serve + # servers that omit the field. authoritative = True - lead_floor_ms = max(0.0, float(int(remaining)) - rtt_ms) - lead_anchor_ms = recv_ms - delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + nxt = sched.on_valid_success(int(remaining), rtt_ms, recv_ms) + if nxt is None: + logger.warning( + "Heartbeat stopping: lease shorter than the retry-safety budget: id=%s", + reservation_id, + ) + return + delay_ms = nxt elif grant <= 0 or ( grant < 0.9 * ttl_ms and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success @@ -491,22 +527,38 @@ def heartbeat_loop() -> None: ) return logger.warning("Stream heartbeat failed: id=%s", reservation_id) - if authoritative and lead_floor_ms is not None: - # Retry inside the known lease: the failed extend may - # have been applied server-side, so the SAME body (same - # idempotency key) is retried at a bounded fraction of - # the remaining lead. - lead_now = max(0.0, lead_floor_ms - (_lifecycle._now_mono_ms() - lead_anchor_ms)) - delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) + if authoritative: + rate_limited = response.status == 429 or code == "LIMIT_EXCEEDED" + if not rate_limited and 400 <= response.status < 500: + # Unrecoverable request/auth failure (spec): never + # rotate the key on an unchanged request. + logger.warning( + "Heartbeat stopping on client error (%s, status=%d): id=%s", + code, response.status, reservation_id, + ) + return + retry_after = response.retry_after_ms_header if rate_limited else None + nxt = sched.on_transient_failure( + _lifecycle._now_mono_ms(), rate_limited=rate_limited, retry_after_ms=retry_after, + ) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt except Exception: logger.warning("Stream heartbeat error: id=%s", reservation_id, exc_info=True) - if authoritative and lead_floor_ms is not None: - # Retry inside the known lease: the failed extend may - # have been applied server-side, so the SAME body (same - # idempotency key) is retried at a bounded fraction of - # the remaining lead. - lead_now = max(0.0, lead_floor_ms - (_lifecycle._now_mono_ms() - lead_anchor_ms)) - delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) + if authoritative: + nxt = sched.on_transient_failure(_lifecycle._now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt t = threading.Thread( target=heartbeat_loop, @@ -559,6 +611,7 @@ def __init__( self._usage = StreamUsage() self._reservation_id: str | None = None self._initial_remaining: int | None = None + self._create_rtt: float | None = None self._caps: Caps | None = None self._decision: Decision = Decision.ALLOW self._ctx: CyclesContext | None = None @@ -597,7 +650,9 @@ async def __aenter__(self) -> AsyncStreamReservation: self._grace_period_ms, ) + _create_sent_ms = _lifecycle._now_mono_ms() response = await self._client.create_reservation(body) + _create_rtt_ms = _lifecycle._now_mono_ms() - _create_sent_ms if not response.is_success: raise _build_protocol_exception("Failed to create reservation", response) @@ -615,6 +670,7 @@ async def __aenter__(self) -> AsyncStreamReservation: self._reservation_id = result.reservation_id self._initial_remaining = result.remaining_ttl_ms + self._create_rtt = _create_rtt_ms self._decision = result.decision self._caps = result.caps @@ -774,21 +830,29 @@ async def heartbeat_loop() -> None: last_grant: float | None = None pending_body: dict[str, Any] | None = None initial_remaining_ms = self._initial_remaining + initial_rtt_ms = self._create_rtt last_success_ms = anchor_ms held_delay_ms = min(ttl_ms / 2, 30_000.0) clamp_warned = False - rtt_max_ms = 0.0 - # Authoritative scheduling (spec v0.1.25.16): a response carrying - # remaining_ttl_ms is the server's own statement of the live - # lease, so the beat is scheduled from it directly. When the - # create response carried it, the first beat derives from it and - # no primed extension is spent. - lead_floor_ms: float | None = None - lead_anchor_ms = anchor_ms + # Authoritative scheduling (spec v0.1.25.16 PRIMARY ALGORITHM): + # a schema-valid 200 carrying remaining_ttl_ms drives scheduling + # exactly; the measured-grant heuristic below is the + # NON-NORMATIVE fallback for servers that omit the field. + sched = _AuthoritativeScheduler(_timeout_budget_ms(self._client._config)) authoritative = initial_remaining_ms is not None if initial_remaining_ms is not None: - lead_floor_ms = float(initial_remaining_ms) - delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + first_delay = sched.on_valid_success( + initial_remaining_ms, initial_rtt_ms or 0.0, anchor_ms, + ) + # Unreachable on the first scheduler call (the zero-delay + # streak cannot be exhausted yet); kept as a typed guard. + if first_delay is None: # pragma: no cover + logger.warning( + "Heartbeat not started: lease shorter than the retry-safety budget: id=%s", + reservation_id, + ) + return + delay_ms = first_delay else: # Immediate first extension (fallback): with lead_min starting # at 0 and no lease signal on the wire, any bounded first @@ -798,10 +862,13 @@ async def heartbeat_loop() -> None: try: while True: await asyncio.sleep(delay_ms / 1000.0) - # After the primed (delay-0) first beat, the baseline - # cadence is the held delay — a transient failure must - # not hot-loop. - delay_ms = delay_ms or held_delay_ms + if not authoritative: + # After the primed (delay-0) first beat, the baseline + # cadence is the held delay — a transient failure must + # not hot-loop. (Authoritative zero delays are + # meaningful: the one-immediate-attempt guards bound + # them.) + delay_ms = delay_ms or held_delay_ms if not authoritative: lead_min = grants_sum - (_lifecycle._now_mono_ms() - anchor_ms) if last_grant is not None and lead_min >= _LEAD_TARGET_FACTOR * last_grant: @@ -812,7 +879,22 @@ async def heartbeat_loop() -> None: sent_ms = _lifecycle._now_mono_ms() response = await client.extend_reservation(reservation_id, body) recv_ms = _lifecycle._now_mono_ms() - if response.is_success: + if response.is_success and authoritative and response.status != 200: + # Ambiguous non-200 2xx in authoritative mode: NOT an + # observed success (spec) — same-key recovery. + logger.warning( + "Heartbeat ambiguous 2xx (status=%d): id=%s", + response.status, reservation_id, + ) + nxt = sched.on_transient_failure(_lifecycle._now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt + elif response.is_success: pending_body = None new_expires = response.get_body_attribute("expires_at_ms") if new_expires is not None: @@ -836,16 +918,21 @@ async def heartbeat_loop() -> None: grants_sum += grant last_grant = grant rtt_ms = recv_ms - sent_ms - rtt_max_ms = max(rtt_max_ms, rtt_ms) remaining = response.get_body_attribute("remaining_ttl_ms") - if remaining is not None: - # Server-authoritative lease (spec v0.1.25.16): - # schedule from it directly; the heuristic arms - # below only serve servers that omit the field. + if response.status == 200 and remaining is not None: + # Server-authoritative lease (spec v0.1.25.16 + # PRIMARY ALGORITHM): schedule from this response + # alone; the heuristic arms below only serve + # servers that omit the field. authoritative = True - lead_floor_ms = max(0.0, float(int(remaining)) - rtt_ms) - lead_anchor_ms = recv_ms - delay_ms = _authoritative_delay_ms(lead_floor_ms, rtt_max_ms) + nxt = sched.on_valid_success(int(remaining), rtt_ms, recv_ms) + if nxt is None: + logger.warning( + "Heartbeat stopping: lease shorter than the retry-safety budget: id=%s", + reservation_id, + ) + return + delay_ms = nxt elif grant <= 0 or ( grant < 0.9 * ttl_ms and 0.75 * elapsed_since_success <= grant <= 1.25 * elapsed_since_success @@ -879,22 +966,38 @@ async def heartbeat_loop() -> None: ) return logger.warning("Async stream heartbeat failed: id=%s", reservation_id) - if authoritative and lead_floor_ms is not None: - # Retry inside the known lease: the failed extend may - # have been applied server-side, so the SAME body (same - # idempotency key) is retried at a bounded fraction of - # the remaining lead. - lead_now = max(0.0, lead_floor_ms - (_lifecycle._now_mono_ms() - lead_anchor_ms)) - delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) + if authoritative: + rate_limited = response.status == 429 or code == "LIMIT_EXCEEDED" + if not rate_limited and 400 <= response.status < 500: + # Unrecoverable request/auth failure (spec): never + # rotate the key on an unchanged request. + logger.warning( + "Heartbeat stopping on client error (%s, status=%d): id=%s", + code, response.status, reservation_id, + ) + return + retry_after = response.retry_after_ms_header if rate_limited else None + nxt = sched.on_transient_failure( + _lifecycle._now_mono_ms(), rate_limited=rate_limited, retry_after_ms=retry_after, + ) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt except Exception: logger.warning("Async stream heartbeat error: id=%s", reservation_id, exc_info=True) - if authoritative and lead_floor_ms is not None: - # Retry inside the known lease: the failed extend may - # have been applied server-side, so the SAME body (same - # idempotency key) is retried at a bounded fraction of - # the remaining lead. - lead_now = max(0.0, lead_floor_ms - (_lifecycle._now_mono_ms() - lead_anchor_ms)) - delay_ms = min(max(lead_now / 4, 1000.0), 30_000.0) + if authoritative: + nxt = sched.on_transient_failure(_lifecycle._now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt except asyncio.CancelledError: return diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 9e64da2..926d77f 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -473,9 +473,9 @@ def wait(timeout: float | None = None) -> bool: thread.join(timeout=5) assert client.extend_reservation.call_count == 4 - assert timeouts[0] == 59.0 - assert timeouts[2] == 1.0 - assert timeouts[3] == 1.0 + assert timeouts[0] == 35.0 + assert timeouts[2] == 6.25 + assert timeouts[3] == 4.6875 @pytest.mark.asyncio async def test_async_stream_field_mode_cycle( @@ -518,9 +518,9 @@ async def fake_sleep(s: float) -> None: await task assert client.extend_reservation.await_count == 4 - assert sleeps[0] == 59.0 - assert sleeps[2] == 1.0 - assert sleeps[3] == 1.0 + assert sleeps[0] == 35.0 + assert sleeps[2] == 6.25 + assert sleeps[3] == 4.6875 @pytest.mark.asyncio async def test_async_stream_lead_estimate_pattern( @@ -605,13 +605,18 @@ async def fake_sleep(s: float) -> None: class TestAuthoritativeScheduling: + # With the SDK's enforced httpx timeouts (connect 2s + read 5s + write + # 5s), request_timeout_budget = 12000ms; at rtt 0: attempt_budget = + # 12000, safety_margin = 1000, retry_reserve = 2×12000 + 1000 = 25000. + RESERVE = 25_000.0 + def test_create_remaining_drives_first_beat_and_steady_cadence( self, monkeypatch: pytest.MonkeyPatch, ) -> None: - # remaining=60000, rtt=0 → reserve = min(30000, max(1000, 0)) = 1000, - # first delay 59000ms. Every extend echoes the field, so the cadence - # holds at 59s and the heuristic lead_min skip NEVER fires even - # though accumulated fallback grants would trip it. + # remaining=60000 → next_delay = 60000 − 25000 = 35000ms. Every + # extend echoes the field, so the cadence holds at 35s and the + # heuristic lead_min skip NEVER fires even though accumulated + # fallback grants would trip it. lifecycle, client = _make_sync() client.extend_reservation.return_value = _extend_ok(None, remaining_ttl_ms=60_000) @@ -620,24 +625,31 @@ def test_create_remaining_drives_first_beat_and_steady_cadence( ) assert client.extend_reservation.call_count == 4 - assert timeouts == [59.0, 59.0, 59.0, 59.0, 59.0] + assert timeouts == [35.0, 35.0, 35.0, 35.0, 35.0] - def test_capped_create_first_beat_lands_inside_small_lease( - self, monkeypatch: pytest.MonkeyPatch, + def test_lease_below_reserve_one_immediate_attempt_then_stop( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: - # 24h request, tenant caps to 1s: remaining=1000 → reserve = - # min(500, 1000) = 500 → first beat at 500ms, inside the real lease. - # This is the exact case that motivated the wire field. + # 24h request capped to a 1s lease: the lease cannot hold the + # 25s retry reserve → next_delay 0 → ONE immediate fresh attempt is + # permitted; when its success also yields 0, the client MUST stop + # and surface (spec zero-delay guard) rather than tight-loop a + # maximum-lead server's extension budget away. lifecycle, client = _make_sync() client.extend_reservation.return_value = _extend_ok(None, remaining_ttl_ms=1_000) - timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=1, - ttl=86_400_000, initial_remaining_ms=1_000, - ) + with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=4, + ttl=86_400_000, initial_remaining_ms=1_000, + ) - assert timeouts[0] == 0.5 + # Immediate first beat (streak 1), its success hits streak 2 → stop. + assert timeouts[0] == 0.0 assert client.extend_reservation.call_count == 1 + assert [r for r in caplog.records if "retry-safety budget" in r.message] def test_lead_clamp_server_with_field_no_warn_no_collapse( self, @@ -646,24 +658,23 @@ def test_lead_clamp_server_with_field_no_warn_no_collapse( ) -> None: # A maximum-lead-clamping server that DOES emit remaining_ttl_ms: # expiry echoes elapsed (grant ≈ elapsed, the heuristic's worst - # case) but the field carries the true lease → authoritative arm + # case) but the field carries the true 60s lead → authoritative arm # schedules cleanly and the clamp warning never fires. clock = FakeClock() lifecycle, client = _make_sync() def clamped_extend(rid: str, body: dict[str, Any]) -> CyclesResponse: - return _extend_ok(INITIAL_EXPIRY + int(clock.t), remaining_ttl_ms=15_000) + return _extend_ok(INITIAL_EXPIRY + int(clock.t), remaining_ttl_ms=60_000) client.extend_reservation.side_effect = clamped_extend with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): timeouts = _run_sync_beats( - lifecycle, clock, monkeypatch, beats=3, initial_remaining_ms=15_000, + lifecycle, clock, monkeypatch, beats=3, initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 3 - # remaining=15000 → reserve min(7500, 1000) = 1000 → delay 14s. - assert timeouts == [14.0, 14.0, 14.0, 14.0] + assert timeouts == [35.0, 35.0, 35.0, 35.0] assert not [r for r in caplog.records if "clamp lease lead" in r.message] def test_field_disappearing_mid_flight_resumes_heuristic( @@ -682,17 +693,17 @@ def test_field_disappearing_mid_flight_resumes_heuristic( ) assert client.extend_reservation.call_count == 2 - # First delay authoritative (59s); after the fieldless response the + # First delay authoritative (35s); after the fieldless response the # normal-regime cadence (ttl/2 = 30s) applies. - assert timeouts[0] == 59.0 + assert timeouts[0] == 35.0 assert timeouts[1] == 30.0 - def test_transient_failure_in_field_mode_retries_bounded_same_key( + def test_transient_failure_recovery_window_shrinks_then_same_key( self, monkeypatch: pytest.MonkeyPatch, ) -> None: - # At the scheduled beat the remaining lead estimate is ~the retry - # reserve (1000ms) → retry delay clamp(lead/4, 1s, 30s) = 1s, with - # the SAME idempotency key; the following success uses a fresh key. + # 503 at the scheduled beat (t=35s): lead_est = 60000 − 35000 = + # 25000; retry_window = 25000 − 12000 − 1000 = 12000 → retry after + # min(30000, 25000/4, 12000) = 6250ms, with the SAME key. lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ CyclesResponse.http_error(503, "unavailable"), @@ -704,17 +715,41 @@ def test_transient_failure_in_field_mode_retries_bounded_same_key( ) assert client.extend_reservation.call_count == 2 - assert timeouts[0] == 59.0 - assert timeouts[1] == 1.0 + assert timeouts[0] == 35.0 + assert timeouts[1] == 6.25 bodies = [c.args[1] for c in client.extend_reservation.call_args_list] assert bodies[0]["idempotency_key"] == bodies[1]["idempotency_key"] - def test_exception_in_field_mode_retries_bounded_same_key( + def test_repeated_failures_stop_when_no_window_progress( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + # Recovery repeats with the same key while the freshly recomputed + # retry_window shrinks: 6250 → 4687.5 → 1062.5 → 0 (one immediate + # retry) → no progress at 0 → MUST stop. Five attempts total. + lifecycle, client = _make_sync() + client.extend_reservation.return_value = CyclesResponse.http_error(503, "down") + + with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=10, initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 5 + assert timeouts == [35.0, 6.25, 4.6875, 1.0625, 0.0] + assert [r for r in caplog.records if "no safe recovery window" in r.message] + keys = {c.args[1]["idempotency_key"] for c in client.extend_reservation.call_args_list} + assert len(keys) == 1 # every recovery reused the same key + + def test_ambiguous_2xx_is_not_applied_same_key_recovery( self, monkeypatch: pytest.MonkeyPatch, ) -> None: + # A non-200 2xx in authoritative mode is ambiguous (spec): never + # scheduled from — recovered with the SAME idempotency key. lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ - ConnectionError("down"), + CyclesResponse.success(202, {"status": "ACTIVE", "remaining_ttl_ms": 60_000}), _extend_ok(None, remaining_ttl_ms=60_000), ] @@ -723,16 +758,74 @@ def test_exception_in_field_mode_retries_bounded_same_key( ) assert client.extend_reservation.call_count == 2 - assert timeouts[1] == 1.0 + assert timeouts[1] == 6.25 bodies = [c.args[1] for c in client.extend_reservation.call_args_list] assert bodies[0]["idempotency_key"] == bodies[1]["idempotency_key"] + def test_429_honored_only_within_window( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Retry-After 3s ≤ window 12000 → retried after exactly 3s, same key. + lifecycle, client = _make_sync() + client.extend_reservation.side_effect = [ + CyclesResponse.http_error(429, "limited", headers={"retry-after": "3"}), + _extend_ok(None, remaining_ttl_ms=60_000), + ] + + timeouts = _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=2, initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 2 + assert timeouts[1] == 3.0 + bodies = [c.args[1] for c in client.extend_reservation.call_args_list] + assert bodies[0]["idempotency_key"] == bodies[1]["idempotency_key"] + + def test_429_missing_or_oversized_retry_after_stops( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + # Retry-After exceeding the retry window (20s > 12000ms) must not be + # re-invented earlier: stop and surface. + lifecycle, client = _make_sync() + client.extend_reservation.return_value = CyclesResponse.http_error( + 429, "limited", headers={"retry-after": "20"}, + ) + + with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): + _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=4, initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 1 + assert [r for r in caplog.records if "no safe recovery window" in r.message] + + def test_other_4xx_stops_without_key_rotation( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.return_value = CyclesResponse.http_error( + 400, "bad", + body={"error": "INVALID_REQUEST", "message": "m", "request_id": "r"}, + ) + + with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): + _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=4, initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 1 + assert [r for r in caplog.records if "client error" in r.message] + @pytest.mark.asyncio async def test_async_field_mode_full_cycle( self, monkeypatch: pytest.MonkeyPatch, ) -> None: # Covers the async authoritative arms: initial field delay, 503 - # bounded retry, exception bounded retry, and field-driven success. + # recovery, exception recovery with a shrinking window, success. clock = FakeClock() monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) lifecycle, client = _make_async() @@ -759,9 +852,56 @@ async def fake_sleep(s: float) -> None: await task assert client.extend_reservation.await_count == 4 - assert sleeps[0] == 59.0 # from the create field - assert sleeps[2] == 1.0 # bounded 503 retry inside the known lease - assert sleeps[3] == 1.0 # bounded exception retry + assert sleeps[0] == 35.0 # from the create field + assert sleeps[2] == 6.25 # 503: window 12000, lead_est/4 = 6250 + assert sleeps[3] == 4.6875 # exception: window shrank to 5750 + + def test_scheduler_edge_cases_direct(self) -> None: + from runcycles.lifecycle import _AuthoritativeScheduler + + sched = _AuthoritativeScheduler(12_000.0) + # No schema-valid response yet → lead estimate is 0 and any failure + # has no recovery window. + assert sched.lead_estimate_ms(1_000.0) == 0.0 + assert sched.on_transient_failure(1_000.0) is None + # Establish a lead, then: missing and negative Retry-After stop. + assert sched.on_valid_success(60_000, 0.0, 0.0) == 35_000.0 + assert sched.on_transient_failure(35_000.0, rate_limited=True, retry_after_ms=None) is None + sched2 = _AuthoritativeScheduler(12_000.0) + sched2.on_valid_success(60_000, 0.0, 0.0) + assert sched2.on_transient_failure(35_000.0, rate_limited=True, retry_after_ms=-1) is None + + def test_sync_repeated_ambiguous_2xx_stops( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.return_value = CyclesResponse.success(202, {"status": "ACTIVE"}) + + with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): + _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=10, initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 5 + assert [r for r in caplog.records if "no safe recovery window" in r.message] + + def test_sync_repeated_exceptions_stop( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.side_effect = ConnectionError("down") + + with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): + _run_sync_beats( + lifecycle, FakeClock(), monkeypatch, beats=10, initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 5 + assert [r for r in caplog.records if "no safe recovery window" in r.message] def test_create_response_model_parses_remaining(self) -> None: from runcycles.models import ReservationCreateResponse @@ -779,6 +919,244 @@ def test_create_response_model_parses_remaining(self) -> None: assert ReservationCreateResponse.model_validate(body).remaining_ttl_ms is None +# Scenario matrix for the authoritative stop/recovery arms, exercised +# against every loop variant (async lifecycle, sync stream, async stream). +# Each entry: (responses factory, initial_remaining, expected extend calls). +# - repeated 503 / ambiguous 202 / exceptions: recovery windows shrink +# 6250 → 4687.5 → 1062.5 → 0 → no-progress stop = 5 attempts; +# - 400: unrecoverable client error, stop after 1; +# - tiny lease: zero-delay guard, 1 immediate attempt then stop; +# - 429 with Retry-After 3s inside the window: honored, then success. +_STOP_MATRIX = [ + ("s503", lambda: CyclesResponse.http_error(503, "down"), 60_000, 5), + ("s202", lambda: CyclesResponse.success(202, {"status": "ACTIVE"}), 60_000, 5), + ("s400", lambda: CyclesResponse.http_error( + 400, "bad", body={"error": "INVALID_REQUEST", "message": "m", "request_id": "r"}, + ), 60_000, 1), + ("szero", lambda: _extend_ok(None, remaining_ttl_ms=1_000), 1_000, 1), + ("sexc", lambda: ConnectionError("down"), 60_000, 5), +] + + +def _matrix_side_effect(factory: Any) -> Any: + def _effect(*args: Any, **kwargs: Any) -> CyclesResponse: + result = factory() + if isinstance(result, Exception): + raise result + return result + return _effect + + +class TestAuthoritativeStopMatrix: + @pytest.mark.parametrize( + ("name", "factory", "initial", "expected"), + _STOP_MATRIX, + ids=[m[0] for m in _STOP_MATRIX], + ) + @pytest.mark.asyncio + async def test_async_lifecycle_arms( + self, name: str, factory: Any, initial: int, expected: int, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + lifecycle, client = _make_async() + client.extend_reservation.side_effect = _matrix_side_effect(factory) + count = 0 + + async def fake_sleep(s: float) -> None: + nonlocal count + count += 1 + if count > 10: + raise asyncio.CancelledError + clock.t += s * 1000.0 + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + task = lifecycle._start_heartbeat("rsv_1", TTL, _ctx(), initial) + assert task is not None + await task + + assert client.extend_reservation.await_count == expected + + @pytest.mark.asyncio + async def test_async_lifecycle_429_honored( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + lifecycle, client = _make_async() + client.extend_reservation.side_effect = [ + CyclesResponse.http_error(429, "limited", headers={"retry-after": "3"}), + _extend_ok(None, remaining_ttl_ms=60_000), + ] + count = 0 + sleeps: list[float] = [] + + async def fake_sleep(s: float) -> None: + nonlocal count + sleeps.append(s) + count += 1 + if count > 2: + raise asyncio.CancelledError + clock.t += s * 1000.0 + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + task = lifecycle._start_heartbeat("rsv_1", TTL, _ctx(), 60_000) + assert task is not None + await task + + assert client.extend_reservation.await_count == 2 + assert sleeps[1] == 3.0 + + def _make_stream(self) -> tuple[StreamReservation, MagicMock]: + client = MagicMock() + client._config = _config() + stream = StreamReservation( + client, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=TTL, + ) + stream._reservation_id = "rsv_1" + stream._ctx = _ctx() + return stream, client + + @pytest.mark.parametrize( + ("name", "factory", "initial", "expected"), + _STOP_MATRIX, + ids=[m[0] for m in _STOP_MATRIX], + ) + def test_sync_stream_arms( + self, name: str, factory: Any, initial: int, expected: int, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + stream, client = self._make_stream() + client.extend_reservation.side_effect = _matrix_side_effect(factory) + stream._initial_remaining = initial + calls = {"n": 0} + + def wait(timeout: float | None = None) -> bool: + calls["n"] += 1 + if calls["n"] > 10: + return True + clock.t += (timeout or 0.0) * 1000.0 + return False + + stream._heartbeat_stop.wait = wait # type: ignore[method-assign] + thread = stream._start_heartbeat() + assert thread is not None + thread.join(timeout=5) + + assert client.extend_reservation.call_count == expected + + def test_sync_stream_429_honored(self, monkeypatch: pytest.MonkeyPatch) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + stream, client = self._make_stream() + client.extend_reservation.side_effect = [ + CyclesResponse.http_error(429, "limited", headers={"retry-after": "3"}), + _extend_ok(None, remaining_ttl_ms=60_000), + ] + stream._initial_remaining = 60_000 + timeouts: list[float] = [] + calls = {"n": 0} + + def wait(timeout: float | None = None) -> bool: + timeouts.append(timeout or 0.0) + calls["n"] += 1 + if calls["n"] > 2: + return True + clock.t += (timeout or 0.0) * 1000.0 + return False + + stream._heartbeat_stop.wait = wait # type: ignore[method-assign] + thread = stream._start_heartbeat() + assert thread is not None + thread.join(timeout=5) + + assert client.extend_reservation.call_count == 2 + assert timeouts[1] == 3.0 + + def _make_async_stream(self) -> tuple[AsyncStreamReservation, AsyncMock]: + client = AsyncMock() + client._config = _config() + stream = AsyncStreamReservation( + client, + subject=Subject(tenant="acme"), + action=Action(kind="k", name="n"), + estimate=Amount(unit=Unit.USD_MICROCENTS, amount=1000), + ttl_ms=TTL, + ) + stream._reservation_id = "rsv_1" + stream._ctx = _ctx() + return stream, client + + @pytest.mark.parametrize( + ("name", "factory", "initial", "expected"), + _STOP_MATRIX, + ids=[m[0] for m in _STOP_MATRIX], + ) + @pytest.mark.asyncio + async def test_async_stream_arms( + self, name: str, factory: Any, initial: int, expected: int, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + stream, client = self._make_async_stream() + client.extend_reservation.side_effect = _matrix_side_effect(factory) + stream._initial_remaining = initial + count = 0 + + async def fake_sleep(s: float) -> None: + nonlocal count + count += 1 + if count > 10: + raise asyncio.CancelledError + clock.t += s * 1000.0 + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + task = stream._start_heartbeat() + assert task is not None + await task + + assert client.extend_reservation.await_count == expected + + @pytest.mark.asyncio + async def test_async_stream_429_honored( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + clock = FakeClock() + monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + stream, client = self._make_async_stream() + client.extend_reservation.side_effect = [ + CyclesResponse.http_error(429, "limited", headers={"retry-after": "3"}), + _extend_ok(None, remaining_ttl_ms=60_000), + ] + stream._initial_remaining = 60_000 + count = 0 + sleeps: list[float] = [] + + async def fake_sleep(s: float) -> None: + nonlocal count + sleeps.append(s) + count += 1 + if count > 2: + raise asyncio.CancelledError + clock.t += s * 1000.0 + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + task = stream._start_heartbeat() + assert task is not None + await task + + assert client.extend_reservation.await_count == 2 + assert sleeps[1] == 3.0 + + # --------------------------------------------------------------------------- # Date header accessor (kept as a general response accessor; the heartbeat # no longer consumes it — RFC 9110 §6.6.1 makes it a different clock). From 982cab3dcd387e9b4af9c97fe507fc001b4338db Mon Sep 17 00:00:00 2001 From: Albert Mavashev Date: Tue, 28 Jul 2026 11:31:22 -0400 Subject: [PATCH 8/9] fix: harden heartbeat lease recovery --- AUDIT.md | 8 +- CHANGELOG.md | 1 + runcycles/__init__.py | 2 + runcycles/lifecycle.py | 566 ++++++++++++++++++++++++++++--------- runcycles/models.py | 37 ++- runcycles/response.py | 26 +- runcycles/streaming.py | 266 ++++++++++-------- tests/test_heartbeat.py | 606 +++++++++++++++++++++++++++++++--------- tests/test_response.py | 41 ++- 9 files changed, 1157 insertions(+), 396 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 650e5d7..82674ec 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -33,8 +33,12 @@ window, other 4xx stop without key rotation; a lease that cannot hold the reserve gets one immediate fresh attempt then stop-and-surface; lead_min skip bypassed; no primed extension when the create carries the field. Bookkeeping keeps running so the v2.3+band heuristic (now explicitly -best-effort fallback) resumes seamlessly if the field disappears; -fieldless servers see unchanged behavior. 568 tests pass at 100% coverage. +best-effort fallback) resumes seamlessly if the field disappears. Final +self-review also made the response contract uniform across both scheduling +modes: only a complete schema-valid HTTP 200 create/extend response succeeds; +ambiguous 2xx keeps the same key. The enforced timeout covers the whole +attempt, first-delay setup time is deducted, and reliable pre-field RTT +samples remain in the safety budget. 597 tests pass at 99.07% coverage. ## 2026-07-27 — Heartbeat conservative-lead redesign + actual_source marker (v0.5.1) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9c85f6..3d551cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Final lease-response and timing conformance (supersedes the older fallback wording below).** Both server-authoritative and fieldless fallback scheduling count only a complete, schema-valid HTTP 200 create/extend response as success; malformed or non-200 2xx responses remain ambiguous and are recovered with the same idempotency key. Create performs one same-key recovery attempt, the enforced timeout covers the whole attempt, post-receipt setup time is deducted from the first delay, and reliable RTT samples gathered before a rolling-upgrade server starts sending `remaining_ttl_ms` remain part of the safety budget. - **Heartbeat redesign (conservative lead lower bound, immediate prime, regime-aware cadence)**: four adversarial review rounds refined the design. The heartbeat maintains `lead_min = Σ measured grants − monotonic elapsed` (grants = differences of successive returned `expires_at_ms` — the same server frame; no cross-clock arithmetic anywhere), starting at 0, so the FIRST extension fires immediately: any bounded first delay could outlive a tenant-policy-capped lease (a 24h request silently capped to seconds would expire before a delayed first beat), and priming costs one extension without changing total protected runtime. Cadence then splits by regime, detected from the measured grant: when the grant tracks the requested lease, cadence = `clamp(grant/2, 500ms, ttl/2)` — per-extend policy clamps automatically tighten the beat; when the grant merely mirrors elapsed time (maximum-lead clamping: `grant ≤ 0`, or `grant < 0.9×ttl` with `grant` inside `[0.75, 1.25]×elapsed-since-last-success`), no cadence signal exists on the wire — the loop holds `min(ttl/2, 30s)` and warns once that the extension budget will deplete, instead of collapsing to the floor and burning `max_extensions` in seconds. The band's lower edge makes misclassification non-sticky: a real per-extend grant seen across a skip-doubled gap (where grant ≈ elapsed exactly) lands in the hold once, but at the held cadence its grant/elapsed ratio falls below 0.75 and cadence re-tightens — with an upper bound alone the hold would stick and a clamped-grant lease would decay to a lapse. A transient failure on the primed (delay-0) beat backs off to the held cadence — never a hot loop. Skip when `lead_min ≥ 1.5×last_grant`. The HTTP `Date` header plays no role in the heartbeat (RFC 9110 §6.6.1: best-effort, whole-second, and possibly a different clock than the one stamping `expires_at_ms` — Redis TIME in the reference server); `CyclesResponse.server_date_ms` remains as a general accessor. Failed extends retry with the same idempotency key; permanent codes (`RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`, raw 404/410) stop the heartbeat; no interval floor. Spec guidance: cycles-protocol#148. - **Heartbeat (superseded intermediate designs, kept for history)**: the initial alternate-beat fix traded outward drift for inward-drift liveness hazards (single-failure lead-0 retry, guaranteed lapse for ttl < 2s under the old 1s interval floor, RTT slippage) found by adversarial self-review. The heartbeat now estimates its remaining lead from the authoritative `expires_at_ms` the server returns — compared clock-skew-free (server-frame differences + client-monotonic elapsed only) — and extends when lead < 1.5×ttl. It also: derives the **effective TTL** from `expires_at_ms − Date` header (tenant policy `max_reservation_ttl_ms`, default 1h, silently caps grants — a 24h request would otherwise heartbeat 12h late); reuses the same idempotency key when retrying a failed extend (a lost response cannot double-extend); stops permanently on `RESERVATION_EXPIRED`/`RESERVATION_FINALIZED`/`MAX_EXTENSIONS_EXCEEDED`/`TENANT_CLOSED`/`NOT_FOUND`; and drops the 1s interval floor. Spec guidance: cycles-protocol#148. - **Heartbeat extend drift** (superseded by the redesign above, kept for history): `extend_by_ms` is relative to the reservation's *current* `expires_at_ms` (spec), but the heartbeat extended by the full `ttl_ms` on every `ttl/2` beat — drifting expiry outward by `ttl/2` per beat. A killed process left its reserved budget locked until the drifted expiry (a zombie-reservation window scaling with runtime, bounded only by `max_extensions`), and long runs burned the `max_extensions` budget twice as fast as needed, losing heartbeat protection mid-flight. All four heartbeats (both lifecycles, both streaming context managers) now use alternate-beat extension: extend on the first beat and every second beat after a success, retrying immediately after a failure. Expiry lead stays within `[ttl/2, 1.5×ttl]`; extension consumption is halved. Fleet-wide fix (TS/Java/Rust ship the same change); spec guidance added in cycles-protocol#148. diff --git a/runcycles/__init__.py b/runcycles/__init__.py index beca392..44d8ec0 100644 --- a/runcycles/__init__.py +++ b/runcycles/__init__.py @@ -25,6 +25,7 @@ CommitRequest, CommitResponse, CommitStatus, + CyclesEvidenceRef, CyclesMetrics, Decision, DecisionRequest, @@ -92,6 +93,7 @@ "Caps", "Decision", "CyclesMetrics", + "CyclesEvidenceRef", "Balance", "CommitOveragePolicy", "ErrorCode", diff --git a/runcycles/lifecycle.py b/runcycles/lifecycle.py index 7fea460..f7701fc 100644 --- a/runcycles/lifecycle.py +++ b/runcycles/lifecycle.py @@ -3,11 +3,14 @@ from __future__ import annotations import asyncio +import json import logging +import math +import queue import threading import time import uuid -from collections.abc import Callable +from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import Any @@ -34,6 +37,7 @@ Decision, DryRunResult, ReservationCreateResponse, + ReservationExtendResponse, Subject, ) from runcycles.response import CyclesResponse @@ -155,7 +159,10 @@ def _build_reservation_body( def _build_commit_body( - actual: int, unit: str, metrics: CyclesMetrics | None, metadata: dict[str, Any] | None, + actual: int, + unit: str, + metrics: CyclesMetrics | None, + metadata: dict[str, Any] | None, ) -> dict[str, Any]: body: dict[str, Any] = { "idempotency_key": str(uuid.uuid4()), @@ -169,7 +176,10 @@ def _build_commit_body( def _build_event_fallback_body( - reservation_id: str, subject: dict[str, Any], action: dict[str, Any], commit_body: dict[str, Any], + reservation_id: str, + subject: dict[str, Any], + action: dict[str, Any], + commit_body: dict[str, Any], ) -> dict[str, Any]: """Build a POST /v1/events body that records the spend of a commit whose reservation expired before the commit landed (the server has already @@ -221,11 +231,225 @@ def _now_mono_ms() -> float: def _timeout_budget_ms(config: Any) -> float: - """The client's enforced upper bound for one complete extend attempt in - ms (connect + read + the fixed 5s write timeout) — the spec's - request_timeout_budget. The SDK always enforces finite httpx timeouts, - so the unknown/unbounded-timeout arm of the spec never applies here.""" - return (float(config.connect_timeout) + float(config.read_timeout) + 5.0) * 1000.0 + """Outer deadline enforced around one complete create/extend attempt. + + HTTPX's connect/read/write/pool settings are phase or inactivity + timeouts, not a whole-request deadline. The lifecycle therefore wraps + each lease-bearing attempt in this total deadline. Pool acquisition and + response-body parsing are inside the same bound. + """ + parts = (float(config.connect_timeout), float(config.read_timeout), 5.0) + if any(not math.isfinite(part) or part <= 0 for part in parts): + return math.inf + return sum(parts) * 1000.0 + + +def _remaining_at_schedule_start( + remaining_ms: int, + received_ms: float | None, + now_ms: float, +) -> int: + """Deduct local setup time elapsed after the create response arrived.""" + if received_ms is None: + return remaining_ms + elapsed_ms = now_ms - received_ms + if not math.isfinite(elapsed_ms) or elapsed_ms < 0: + return 0 + return max(0, math.floor(remaining_ms - elapsed_ms)) + + +def _run_sync_attempt(call: Callable[[], CyclesResponse], timeout_budget_ms: float) -> CyclesResponse: + """Run one synchronous HTTP attempt under a real whole-attempt deadline. + + HTTPX cannot impose a total deadline over all pool/connect/write/read + phases. A daemon worker lets the heartbeat regain control at the + configured deadline. A timed-out request may still finish in the + background, which is why every recovery reuses the same idempotency key. + """ + if not math.isfinite(timeout_budget_ms): + return call() + + result: queue.Queue[tuple[bool, CyclesResponse | Exception]] = queue.Queue(maxsize=1) + + def invoke() -> None: + try: + result.put((True, call())) + except Exception as exc: # propagate the original client failure + result.put((False, exc)) + + worker = threading.Thread(target=invoke, daemon=True, name="cycles-http-attempt") + worker.start() + worker.join(timeout_budget_ms / 1000.0) + if worker.is_alive(): + raise TimeoutError(f"Cycles HTTP attempt exceeded {timeout_budget_ms:g}ms") + ok, value = result.get_nowait() + if ok: + return value # type: ignore[return-value] + raise value # type: ignore[misc] + + +async def _run_async_attempt( + call: Callable[[], Awaitable[CyclesResponse]], + timeout_budget_ms: float, +) -> CyclesResponse: + """Run one asynchronous HTTP attempt under a whole-attempt deadline.""" + if not math.isfinite(timeout_budget_ms): + return await call() + return await asyncio.wait_for(call(), timeout=timeout_budget_ms / 1000.0) + + +_CREATE_RESPONSE_FIELDS = frozenset( + { + "decision", + "reservation_id", + "affected_scopes", + "expires_at_ms", + "remaining_ttl_ms", + "scope_path", + "reserved", + "caps", + "reason_code", + "retry_after_ms", + "balances", + "cycles_evidence", + } +) + + +def _contains_json_null(value: Any) -> bool: + """Whether a lease response contains an OpenAPI-non-nullable JSON null.""" + if value is None: + return True + if isinstance(value, dict): + return any(_contains_json_null(item) for item in value.values()) + if isinstance(value, list): + return any(_contains_json_null(item) for item in value) + return False + + +def _schema_valid_create(response: CyclesResponse) -> ReservationCreateResponse | None: + """Return a fully validated create body only for exact HTTP 200.""" + if response.status != 200 or not isinstance(response.body, dict): + return None + body = response.body + if set(body) - _CREATE_RESPONSE_FIELDS: + return None + if _contains_json_null(body): + return None + remaining = body.get("remaining_ttl_ms") + if remaining is not None and (not isinstance(remaining, int) or isinstance(remaining, bool) or remaining < 0): + return None + expires = body.get("expires_at_ms") + if expires is not None and (not isinstance(expires, int) or isinstance(expires, bool) or expires < 0): + return None + affected = body.get("affected_scopes") + if not isinstance(affected, list) or any(not isinstance(scope, str) for scope in affected): + return None + try: + return ReservationCreateResponse.model_validate_json(json.dumps(body), strict=True) + except Exception: + return None + + +def _schema_valid_extend(response: CyclesResponse) -> ReservationExtendResponse | None: + """Return a fully validated extend body only for exact HTTP 200.""" + if response.status != 200 or not isinstance(response.body, dict): + return None + body = response.body + if set(body) - {"status", "expires_at_ms", "remaining_ttl_ms", "balances"}: + return None + if _contains_json_null(body): + return None + for name in ("expires_at_ms", "remaining_ttl_ms"): + value = body.get(name) + if value is not None and (not isinstance(value, int) or isinstance(value, bool) or value < 0): + return None + try: + return ReservationExtendResponse.model_validate_json(json.dumps(body), strict=True) + except Exception: + return None + + +def _create_is_recoverable(response: CyclesResponse) -> bool: + return response.status < 0 or response.status >= 500 or 200 <= response.status < 300 + + +def _extend_error_is_recoverable(response: CyclesResponse) -> bool: + """Whether a non-2xx extend outcome is recoverable in field mode.""" + return response.is_transport_error or response.is_server_error or response.status == 429 + + +def _ambiguous_create_error(response: CyclesResponse) -> CyclesProtocolError: + return CyclesProtocolError( + "Create reservation did not produce a schema-valid HTTP 200 response", + status=response.status, + ) + + +def _create_reservation_with_recovery( + client: CyclesClient, + body: dict[str, Any], +) -> tuple[CyclesResponse, ReservationCreateResponse, float, float]: + """Create with at most one immediate same-key ambiguity recovery.""" + timeout_budget_ms = _timeout_budget_ms(client._config) + last_exception: Exception | None = None + for attempt in range(2): + sent_ms = _now_mono_ms() + try: + response = _run_sync_attempt( + lambda: client.create_reservation(body), + timeout_budget_ms, + ) + except Exception as exc: + last_exception = exc + if attempt == 0: + continue + raise CyclesProtocolError( + f"Create reservation remained ambiguous after same-key retry: {exc}", + ) from exc + parsed = _schema_valid_create(response) + if parsed is not None: + received_ms = _now_mono_ms() + return response, parsed, received_ms - sent_ms, received_ms + if attempt == 0 and _create_is_recoverable(response): + continue + if response.status < 200 or response.status >= 300: + raise _build_protocol_exception("Failed to create reservation", response) + raise _ambiguous_create_error(response) + raise AssertionError(last_exception) # pragma: no cover + + +async def _create_reservation_with_recovery_async( + client: AsyncCyclesClient, + body: dict[str, Any], +) -> tuple[CyclesResponse, ReservationCreateResponse, float, float]: + """Async create with at most one immediate same-key ambiguity recovery.""" + timeout_budget_ms = _timeout_budget_ms(client._config) + last_exception: Exception | None = None + for attempt in range(2): + sent_ms = _now_mono_ms() + try: + response = await _run_async_attempt( + lambda: client.create_reservation(body), + timeout_budget_ms, + ) + except Exception as exc: + last_exception = exc + if attempt == 0: + continue + raise CyclesProtocolError( + f"Create reservation remained ambiguous after same-key retry: {exc}", + ) from exc + parsed = _schema_valid_create(response) + if parsed is not None: + received_ms = _now_mono_ms() + return response, parsed, received_ms - sent_ms, received_ms + if attempt == 0 and _create_is_recoverable(response): + continue + if response.status < 200 or response.status >= 300: + raise _build_protocol_exception("Failed to create reservation", response) + raise _ambiguous_create_error(response) + raise AssertionError(last_exception) # pragma: no cover class _AuthoritativeScheduler: @@ -252,15 +476,29 @@ def _attempt_budget_ms(self) -> float: def _safety_margin_ms(self) -> float: return max(1000.0, 2.0 * self._rtt_max_ms) + def observe_rtt(self, rtt_ms: float) -> None: + """Retain every reliable schema-valid create/extend RTT sample.""" + if math.isfinite(rtt_ms) and rtt_ms >= 0: + self._rtt_max_ms = max(self._rtt_max_ms, rtt_ms) + def on_valid_success( - self, remaining_ms: int, rtt_ms: float, now_ms: float, + self, + remaining_ms: int, + rtt_ms: float, + now_ms: float, ) -> float | None: """Schema-valid HTTP 200 carrying remaining_ttl_ms. retry_reserve = 2×attempt_budget + safety_margin covers one failed attempt, one same-key retry, and margin.""" - self._rtt_max_ms = max(self._rtt_max_ms, rtt_ms) + if not math.isfinite(rtt_ms) or rtt_ms < 0: + # Unknown/unreliable timing cannot be treated as zero elapsed. + self._rtt_max_ms = math.inf + lead_floor_ms = 0.0 + else: + self.observe_rtt(rtt_ms) + lead_floor_ms = max(0.0, float(remaining_ms) - rtt_ms) self._prev_fail_window = None - self._lead_floor_ms = max(0.0, float(remaining_ms) - max(rtt_ms, 0.0)) + self._lead_floor_ms = lead_floor_ms self._lead_anchor_ms = now_ms reserve = 2.0 * self._attempt_budget_ms() + self._safety_margin_ms() delay = self._lead_floor_ms - reserve @@ -279,7 +517,10 @@ def on_valid_success( def lead_estimate_ms(self, now_ms: float) -> float: if self._lead_floor_ms is None or self._lead_anchor_ms is None: return 0.0 - return max(0.0, self._lead_floor_ms - (now_ms - self._lead_anchor_ms)) + elapsed_ms = now_ms - self._lead_anchor_ms + if not math.isfinite(elapsed_ms) or elapsed_ms < 0: + return 0.0 + return max(0.0, self._lead_floor_ms - elapsed_ms) def on_transient_failure( self, @@ -378,7 +619,10 @@ class CyclesLifecycle: """Synchronous lifecycle orchestrator: reserve → execute → commit/release.""" def __init__( - self, client: CyclesClient, retry_engine: CommitRetryEngine, default_subject: dict[str, str | None], + self, + client: CyclesClient, + retry_engine: CommitRetryEngine, + default_subject: dict[str, str | None], ) -> None: self._client = client self._retry_engine = retry_engine @@ -401,15 +645,10 @@ def execute( logger.debug("Creating reservation: body=%s", create_body) res_t1 = time.monotonic() - _create_sent_ms = _now_mono_ms() - res_response = self._client.create_reservation(create_body) - _create_rtt_ms = _now_mono_ms() - _create_sent_ms - - if not res_response.is_success: - logger.error("Reservation failed: response=%s", res_response) - raise _build_protocol_exception("Failed to create reservation", res_response) - - res_result = ReservationCreateResponse.model_validate(res_response.body) + res_response, res_result, _create_rtt_ms, _create_received_ms = _create_reservation_with_recovery( + self._client, + create_body, + ) res_t2 = time.monotonic() decision = res_result.decision @@ -444,7 +683,9 @@ def execute( logger.info( "Reservation created: id=%s, decision=%s, elapsed=%dms", - reservation_id, decision, int((res_t2 - res_t1) * 1000), + reservation_id, + decision, + int((res_t2 - res_t1) * 1000), ) # Set context @@ -464,8 +705,13 @@ def execute( # Start heartbeat heartbeat_stop = threading.Event() heartbeat_thread = self._start_heartbeat( - reservation_id, cfg.ttl_ms, ctx, heartbeat_stop, - res_result.remaining_ttl_ms, _create_rtt_ms, + reservation_id, + cfg.ttl_ms, + ctx, + heartbeat_stop, + res_result.remaining_ttl_ms, + _create_rtt_ms, + _create_received_ms, ) try: @@ -492,7 +738,10 @@ def execute( commit_metadata = {**(commit_metadata or {}), "actual_source": "estimate"} commit_body = _build_commit_body(actual_amount, cfg.unit, metrics, commit_metadata) event_fallback = _build_event_fallback_body( - reservation_id, create_body["subject"], create_body["action"], commit_body, + reservation_id, + create_body["subject"], + create_body["action"], + commit_body, ) self._handle_commit(reservation_id, commit_body, event_fallback) @@ -509,7 +758,10 @@ def execute( _clear_context() def _handle_commit( - self, reservation_id: str, commit_body: dict[str, Any], event_fallback_body: dict[str, Any], + self, + reservation_id: str, + commit_body: dict[str, Any], + event_fallback_body: dict[str, Any], ) -> None: try: logger.debug("Committing: id=%s", reservation_id) @@ -530,7 +782,9 @@ def _handle_commit( # honoring the server's Retry-After. logger.warning("Commit rate-limited; scheduling retry: id=%s", reservation_id) self._retry_engine.schedule( - reservation_id, commit_body, event_fallback_body, + reservation_id, + commit_body, + event_fallback_body, retry_after_ms=response.retry_after_ms_header, ) elif response.status in (401, 403): @@ -539,7 +793,8 @@ def _handle_commit( # that would return budget for real spend. logger.error( "Commit got authentication failure (status=%d); journaling for replay: id=%s", - response.status, reservation_id, + response.status, + reservation_id, ) self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body) elif response.status == 410 or error_code == "RESERVATION_EXPIRED": @@ -558,9 +813,10 @@ def _handle_commit( # Codeless or forward-compat-unknown 4xx: neither release # nor drop — retain the spend record. logger.error( - "Commit got unclassifiable client error (status=%d, error=%s); " - "journaling for replay: id=%s", - response.status, error_code, reservation_id, + "Commit got unclassifiable client error (status=%d, error=%s); journaling for replay: id=%s", + response.status, + error_code, + reservation_id, ) self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body) else: @@ -589,6 +845,7 @@ def _start_heartbeat( stop_event: threading.Event, initial_remaining_ms: int | None = None, initial_rtt_ms: float | None = None, + initial_received_ms: float | None = None, ) -> threading.Thread | None: if ttl_ms <= 0: return None @@ -621,11 +878,21 @@ def heartbeat_loop() -> None: # a schema-valid 200 carrying remaining_ttl_ms drives scheduling # exactly; the measured-grant heuristic below is the # NON-NORMATIVE fallback for servers that omit the field. - sched = _AuthoritativeScheduler(_timeout_budget_ms(self._client._config)) + timeout_budget_ms = _timeout_budget_ms(self._client._config) + sched = _AuthoritativeScheduler(timeout_budget_ms) + if initial_rtt_ms is not None: + sched.observe_rtt(initial_rtt_ms) authoritative = initial_remaining_ms is not None if initial_remaining_ms is not None: + remaining_at_start = _remaining_at_schedule_start( + initial_remaining_ms, + initial_received_ms, + anchor_ms, + ) first_delay = sched.on_valid_success( - initial_remaining_ms, initial_rtt_ms or 0.0, anchor_ms, + remaining_at_start, + initial_rtt_ms or 0.0, + anchor_ms, ) # Unreachable on the first scheduler call (the zero-delay # streak cannot be exhausted yet); kept as a typed guard. @@ -657,39 +924,37 @@ def heartbeat_loop() -> None: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) pending_body = body sent_ms = _now_mono_ms() - response = self._client.extend_reservation(reservation_id, body) + response = _run_sync_attempt( + lambda: self._client.extend_reservation(reservation_id, body), + timeout_budget_ms, + ) + parsed_extend = _schema_valid_extend(response) recv_ms = _now_mono_ms() - if response.is_success and authoritative and response.status != 200: - # Ambiguous non-200 2xx in authoritative mode: NOT an - # observed success (spec) — same-key recovery. + if response.is_success and parsed_extend is None: + # Any non-200 or schema-invalid 2xx is ambiguous: it is + # never an observed success and the key stays pending. logger.warning( - "Heartbeat ambiguous 2xx (status=%d): id=%s", - response.status, reservation_id, + "Heartbeat ambiguous response (status=%d): id=%s", + response.status, + reservation_id, ) - nxt = sched.on_transient_failure(_now_mono_ms()) - if nxt is None: - logger.warning( - "Heartbeat stopping: no safe recovery window remains: id=%s", - reservation_id, - ) - return - delay_ms = nxt - elif response.is_success: - pending_body = None - new_expires = response.get_body_attribute("expires_at_ms") - if new_expires is not None: - new_expires = int(new_expires) - ctx.update_expires_at_ms(new_expires) - grant = ( - float(new_expires - prev_expiry) - if prev_expiry is not None - else float(ttl_ms) - ) - prev_expiry = new_expires + if authoritative: + nxt = sched.on_transient_failure(_now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt else: - grant = float(ttl_ms) - if prev_expiry is not None: - prev_expiry += ttl_ms + delay_ms = held_delay_ms + elif parsed_extend is not None: + pending_body = None + new_expires = parsed_extend.expires_at_ms + ctx.update_expires_at_ms(new_expires) + grant = float(new_expires - prev_expiry) if prev_expiry is not None else float(ttl_ms) + prev_expiry = new_expires grant = max(grant, 0.0) now_ms = _now_mono_ms() elapsed_since_success = now_ms - last_success_ms @@ -697,14 +962,15 @@ def heartbeat_loop() -> None: grants_sum += grant last_grant = grant rtt_ms = recv_ms - sent_ms - remaining = response.get_body_attribute("remaining_ttl_ms") - if response.status == 200 and remaining is not None: + sched.observe_rtt(rtt_ms) + remaining = parsed_extend.remaining_ttl_ms + if remaining is not None: # Server-authoritative lease (spec v0.1.25.16 # PRIMARY ALGORITHM): schedule from this response # alone; the heuristic arms below only serve # servers that omit the field. authoritative = True - nxt = sched.on_valid_success(int(remaining), rtt_ms, recv_ms) + nxt = sched.on_valid_success(remaining, rtt_ms, recv_ms) if nxt is None: logger.warning( "Heartbeat stopping: lease shorter than the retry-safety budget: id=%s", @@ -742,23 +1008,36 @@ def heartbeat_loop() -> None: if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: logger.warning( "Heartbeat stopping permanently (%s, status=%d): id=%s", - code, response.status, reservation_id, + code, + response.status, + reservation_id, ) return logger.warning("Heartbeat extend failed: id=%s, status=%d", reservation_id, response.status) if authoritative: - rate_limited = response.status == 429 or code == "LIMIT_EXCEEDED" + rate_limited = response.status == 429 if not rate_limited and 400 <= response.status < 500: # Unrecoverable request/auth failure (spec): never # rotate the key on an unchanged request. logger.warning( "Heartbeat stopping on client error (%s, status=%d): id=%s", - code, response.status, reservation_id, + code, + response.status, + reservation_id, + ) + return + if not _extend_error_is_recoverable(response): + logger.warning( + "Heartbeat stopping on unexpected HTTP status %d: id=%s", + response.status, + reservation_id, ) return retry_after = response.retry_after_ms_header if rate_limited else None nxt = sched.on_transient_failure( - _now_mono_ms(), rate_limited=rate_limited, retry_after_ms=retry_after, + _now_mono_ms(), + rate_limited=rate_limited, + retry_after_ms=retry_after, ) if nxt is None: logger.warning( @@ -788,7 +1067,10 @@ class AsyncCyclesLifecycle: """Asynchronous lifecycle orchestrator: reserve → execute → commit/release.""" def __init__( - self, client: AsyncCyclesClient, retry_engine: AsyncCommitRetryEngine, default_subject: dict[str, str | None], + self, + client: AsyncCyclesClient, + retry_engine: AsyncCommitRetryEngine, + default_subject: dict[str, str | None], ) -> None: self._client = client self._retry_engine = retry_engine @@ -806,14 +1088,9 @@ async def execute( logger.debug("Estimated usage: estimate=%d", estimate) create_body = _build_reservation_body(cfg, estimate, self._default_subject, args, kwargs) - _create_sent_ms = _now_mono_ms() - res_response = await self._client.create_reservation(create_body) - _create_rtt_ms = _now_mono_ms() - _create_sent_ms - - if not res_response.is_success: - raise _build_protocol_exception("Failed to create reservation", res_response) - - res_result = ReservationCreateResponse.model_validate(res_response.body) + res_response, res_result, _create_rtt_ms, _create_received_ms = await _create_reservation_with_recovery_async( + self._client, create_body + ) res_t2 = time.monotonic() decision = res_result.decision @@ -856,8 +1133,12 @@ async def execute( _set_context(ctx) heartbeat_task = self._start_heartbeat( - reservation_id, cfg.ttl_ms, ctx, - res_result.remaining_ttl_ms, _create_rtt_ms, + reservation_id, + cfg.ttl_ms, + ctx, + res_result.remaining_ttl_ms, + _create_rtt_ms, + _create_received_ms, ) try: @@ -880,7 +1161,10 @@ async def execute( commit_metadata = {**(commit_metadata or {}), "actual_source": "estimate"} commit_body = _build_commit_body(actual_amount, cfg.unit, metrics, commit_metadata) event_fallback = _build_event_fallback_body( - reservation_id, create_body["subject"], create_body["action"], commit_body, + reservation_id, + create_body["subject"], + create_body["action"], + commit_body, ) await self._handle_commit(reservation_id, commit_body, event_fallback) @@ -900,7 +1184,10 @@ async def execute( _clear_context() async def _handle_commit( - self, reservation_id: str, commit_body: dict[str, Any], event_fallback_body: dict[str, Any], + self, + reservation_id: str, + commit_body: dict[str, Any], + event_fallback_body: dict[str, Any], ) -> None: try: response = await self._client.commit_reservation(reservation_id, commit_body) @@ -919,7 +1206,9 @@ async def _handle_commit( # honoring the server's Retry-After. logger.warning("Commit rate-limited; scheduling retry: id=%s", reservation_id) self._retry_engine.schedule( - reservation_id, commit_body, event_fallback_body, + reservation_id, + commit_body, + event_fallback_body, retry_after_ms=response.retry_after_ms_header, ) elif response.status in (401, 403): @@ -928,7 +1217,8 @@ async def _handle_commit( # that would return budget for real spend. logger.error( "Commit got authentication failure (status=%d); journaling for replay: id=%s", - response.status, reservation_id, + response.status, + reservation_id, ) self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body) elif response.status == 410 or error_code == "RESERVATION_EXPIRED": @@ -947,9 +1237,10 @@ async def _handle_commit( # Codeless or forward-compat-unknown 4xx: neither release # nor drop — retain the spend record. logger.error( - "Commit got unclassifiable client error (status=%d, error=%s); " - "journaling for replay: id=%s", - response.status, error_code, reservation_id, + "Commit got unclassifiable client error (status=%d, error=%s); journaling for replay: id=%s", + response.status, + error_code, + reservation_id, ) self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body) else: @@ -976,6 +1267,7 @@ def _start_heartbeat( ctx: CyclesContext, initial_remaining_ms: int | None = None, initial_rtt_ms: float | None = None, + initial_received_ms: float | None = None, ) -> asyncio.Task[None] | None: if ttl_ms <= 0: return None @@ -994,11 +1286,21 @@ async def heartbeat_loop() -> None: # a schema-valid 200 carrying remaining_ttl_ms drives scheduling # exactly; the measured-grant heuristic below is the # NON-NORMATIVE fallback for servers that omit the field. - sched = _AuthoritativeScheduler(_timeout_budget_ms(self._client._config)) + timeout_budget_ms = _timeout_budget_ms(self._client._config) + sched = _AuthoritativeScheduler(timeout_budget_ms) + if initial_rtt_ms is not None: + sched.observe_rtt(initial_rtt_ms) authoritative = initial_remaining_ms is not None if initial_remaining_ms is not None: + remaining_at_start = _remaining_at_schedule_start( + initial_remaining_ms, + initial_received_ms, + anchor_ms, + ) first_delay = sched.on_valid_success( - initial_remaining_ms, initial_rtt_ms or 0.0, anchor_ms, + remaining_at_start, + initial_rtt_ms or 0.0, + anchor_ms, ) # Unreachable on the first scheduler call (the zero-delay # streak cannot be exhausted yet); kept as a typed guard. @@ -1033,39 +1335,37 @@ async def heartbeat_loop() -> None: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) pending_body = body sent_ms = _now_mono_ms() - response = await self._client.extend_reservation(reservation_id, body) + response = await _run_async_attempt( + lambda: self._client.extend_reservation(reservation_id, body), + timeout_budget_ms, + ) + parsed_extend = _schema_valid_extend(response) recv_ms = _now_mono_ms() - if response.is_success and authoritative and response.status != 200: - # Ambiguous non-200 2xx in authoritative mode: NOT an - # observed success (spec) — same-key recovery. + if response.is_success and parsed_extend is None: + # Any non-200 or schema-invalid 2xx is ambiguous: + # never rotate the pending idempotency key. logger.warning( - "Heartbeat ambiguous 2xx (status=%d): id=%s", - response.status, reservation_id, + "Heartbeat ambiguous response (status=%d): id=%s", + response.status, + reservation_id, ) - nxt = sched.on_transient_failure(_now_mono_ms()) - if nxt is None: - logger.warning( - "Heartbeat stopping: no safe recovery window remains: id=%s", - reservation_id, - ) - return - delay_ms = nxt - elif response.is_success: - pending_body = None - new_expires = response.get_body_attribute("expires_at_ms") - if new_expires is not None: - new_expires = int(new_expires) - ctx.update_expires_at_ms(new_expires) - grant = ( - float(new_expires - prev_expiry) - if prev_expiry is not None - else float(ttl_ms) - ) - prev_expiry = new_expires + if authoritative: + nxt = sched.on_transient_failure(_now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt else: - grant = float(ttl_ms) - if prev_expiry is not None: - prev_expiry += ttl_ms + delay_ms = held_delay_ms + elif parsed_extend is not None: + pending_body = None + new_expires = parsed_extend.expires_at_ms + ctx.update_expires_at_ms(new_expires) + grant = float(new_expires - prev_expiry) if prev_expiry is not None else float(ttl_ms) + prev_expiry = new_expires grant = max(grant, 0.0) now_ms = _now_mono_ms() elapsed_since_success = now_ms - last_success_ms @@ -1073,14 +1373,15 @@ async def heartbeat_loop() -> None: grants_sum += grant last_grant = grant rtt_ms = recv_ms - sent_ms - remaining = response.get_body_attribute("remaining_ttl_ms") - if response.status == 200 and remaining is not None: + sched.observe_rtt(rtt_ms) + remaining = parsed_extend.remaining_ttl_ms + if remaining is not None: # Server-authoritative lease (spec v0.1.25.16 # PRIMARY ALGORITHM): schedule from this response # alone; the heuristic arms below only serve # servers that omit the field. authoritative = True - nxt = sched.on_valid_success(int(remaining), rtt_ms, recv_ms) + nxt = sched.on_valid_success(remaining, rtt_ms, recv_ms) if nxt is None: logger.warning( "Heartbeat stopping: lease shorter than the retry-safety budget: id=%s", @@ -1117,23 +1418,36 @@ async def heartbeat_loop() -> None: if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: logger.warning( "Heartbeat stopping permanently (%s, status=%d): id=%s", - code, response.status, reservation_id, + code, + response.status, + reservation_id, ) return logger.warning("Heartbeat extend failed: id=%s", reservation_id) if authoritative: - rate_limited = response.status == 429 or code == "LIMIT_EXCEEDED" + rate_limited = response.status == 429 if not rate_limited and 400 <= response.status < 500: # Unrecoverable request/auth failure (spec): never # rotate the key on an unchanged request. logger.warning( "Heartbeat stopping on client error (%s, status=%d): id=%s", - code, response.status, reservation_id, + code, + response.status, + reservation_id, + ) + return + if not _extend_error_is_recoverable(response): + logger.warning( + "Heartbeat stopping on unexpected HTTP status %d: id=%s", + response.status, + reservation_id, ) return retry_after = response.retry_after_ms_header if rate_limited else None nxt = sched.on_transient_failure( - _now_mono_ms(), rate_limited=rate_limited, retry_after_ms=retry_after, + _now_mono_ms(), + rate_limited=rate_limited, + retry_after_ms=retry_after, ) if nxt is None: logger.warning( diff --git a/runcycles/models.py b/runcycles/models.py index 3b87c8b..a485593 100644 --- a/runcycles/models.py +++ b/runcycles/models.py @@ -4,8 +4,9 @@ from enum import Enum from typing import Annotated, Any +from urllib.parse import urlsplit -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator class Unit(str, Enum): @@ -97,21 +98,21 @@ def from_string(cls, value: str | None) -> ErrorCode | None: # --- Core value objects --- -_SNAKE_CASE_CONFIG = ConfigDict(populate_by_name=True) +_SNAKE_CASE_CONFIG = ConfigDict(populate_by_name=True, extra="forbid") class Amount(BaseModel): model_config = _SNAKE_CASE_CONFIG unit: Unit - amount: int = Field(ge=0) + amount: int = Field(ge=0, le=9_223_372_036_854_775_807) class SignedAmount(BaseModel): model_config = _SNAKE_CASE_CONFIG unit: Unit - amount: int # can be negative + amount: int = Field(ge=-9_223_372_036_854_775_808, le=9_223_372_036_854_775_807) class Subject(BaseModel): @@ -192,6 +193,22 @@ class Balance(BaseModel): is_over_limit: bool | None = None +class CyclesEvidenceRef(BaseModel): + """Transport reference to a server-issued CyclesEvidence envelope.""" + + model_config = _SNAKE_CASE_CONFIG + + evidence_id: str = Field(pattern=r"^[0-9a-f]{64}$") + cycles_evidence_url: str = Field(min_length=1) + + @field_validator("cycles_evidence_url") + @classmethod + def require_absolute_uri(cls, value: str) -> str: + if not urlsplit(value).scheme: + raise ValueError("cycles_evidence_url must be an absolute URI") + return value + + # --- Request models --- @@ -265,17 +282,18 @@ class ReservationCreateResponse(BaseModel): decision: Decision reservation_id: str | None = None affected_scopes: list[str] - expires_at_ms: int | None = None + expires_at_ms: Annotated[int, Field(ge=0, le=9_223_372_036_854_775_807)] | None = None # Server-authoritative remaining lease (ms) at response evaluation # (spec v0.1.25.16). Optional: older servers omit it; when present the # heartbeat schedules from it directly. - remaining_ttl_ms: int | None = None + remaining_ttl_ms: Annotated[int, Field(ge=0, le=9_223_372_036_854_775_807)] | None = None scope_path: str | None = None reserved: Amount | None = None caps: Caps | None = None - reason_code: str | None = None - retry_after_ms: int | None = None + reason_code: Annotated[str, Field(max_length=128)] | None = None + retry_after_ms: Annotated[int, Field(ge=0)] | None = None balances: list[Balance] | None = None + cycles_evidence: CyclesEvidenceRef | None = None def is_allowed(self) -> bool: return self.decision in (Decision.ALLOW, Decision.ALLOW_WITH_CAPS) @@ -305,7 +323,8 @@ class ReservationExtendResponse(BaseModel): model_config = _SNAKE_CASE_CONFIG status: ExtendStatus - expires_at_ms: int + expires_at_ms: Annotated[int, Field(ge=0, le=9_223_372_036_854_775_807)] + remaining_ttl_ms: Annotated[int, Field(ge=0, le=9_223_372_036_854_775_807)] | None = None balances: list[Balance] | None = None diff --git a/runcycles/response.py b/runcycles/response.py index 6836d0b..c3c86df 100644 --- a/runcycles/response.py +++ b/runcycles/response.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from dataclasses import dataclass, field from email.utils import parsedate_to_datetime from typing import Any @@ -26,7 +27,11 @@ def success(cls, status: int, body: dict[str, Any], headers: dict[str, str] | No @classmethod def http_error( - cls, status: int, error_message: str, body: dict[str, Any] | None = None, headers: dict[str, str] | None = None, + cls, + status: int, + error_message: str, + body: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ) -> CyclesResponse: return cls(status=status, body=body, error_message=error_message, headers=headers or {}) @@ -69,19 +74,22 @@ def retry_after_ms_header(self) -> int | None: val = self.headers.get("retry-after") if val is None: return None - try: - return int(val) * 1000 - except ValueError: + stripped = val.strip() + if re.fullmatch(r"[0-9]+", stripped) is None: + return None + seconds = int(stripped) + if seconds > 9_223_372_036_854_775_807 // 1000: return None + return seconds * 1000 @property def server_date_ms(self) -> int | None: - """HTTP ``Date`` header as epoch milliseconds (server wall clock). + """HTTP ``Date`` header as epoch milliseconds. - Server-frame, so differencing it against other server-frame values - (like ``expires_at_ms``) is clock-skew-free to within the header's - one-second resolution plus transit latency. Returns ``None`` when - absent or unparseable. + ``Date`` is best-effort HTTP metadata and may come from a different + clock than body timestamps or be replaced by an intermediary. It is + not used for heartbeat scheduling. Returns ``None`` when absent or + unparseable. """ val = self.headers.get("date") if val is None: diff --git a/runcycles/streaming.py b/runcycles/streaming.py index dcc57a5..706cc6d 100644 --- a/runcycles/streaming.py +++ b/runcycles/streaming.py @@ -25,6 +25,12 @@ _build_extend_body, _build_protocol_exception, _build_release_body, + _create_reservation_with_recovery, + _create_reservation_with_recovery_async, + _remaining_at_schedule_start, + _run_async_attempt, + _run_sync_attempt, + _schema_valid_extend, _timeout_budget_ms, ) from runcycles.models import ( @@ -33,7 +39,6 @@ Caps, CyclesMetrics, Decision, - ReservationCreateResponse, Subject, ) from runcycles.retry import ( @@ -181,6 +186,7 @@ def __init__( self._reservation_id: str | None = None self._initial_remaining: int | None = None self._create_rtt: float | None = None + self._create_received_ms: float | None = None self._caps: Caps | None = None self._decision: Decision = Decision.ALLOW self._ctx: CyclesContext | None = None @@ -220,14 +226,10 @@ def __enter__(self) -> StreamReservation: self._grace_period_ms, ) - _create_sent_ms = _lifecycle._now_mono_ms() - response = self._client.create_reservation(body) - _create_rtt_ms = _lifecycle._now_mono_ms() - _create_sent_ms - - if not response.is_success: - raise _build_protocol_exception("Failed to create reservation", response) - - result = ReservationCreateResponse.model_validate(response.body) + response, result, _create_rtt_ms, _create_received_ms = _create_reservation_with_recovery( + self._client, + body, + ) if result.decision == Decision.DENY: raise _build_protocol_exception("Reservation denied", response) @@ -241,6 +243,7 @@ def __enter__(self) -> StreamReservation: self._reservation_id = result.reservation_id self._initial_remaining = result.remaining_ttl_ms self._create_rtt = _create_rtt_ms + self._create_received_ms = _create_received_ms self._decision = result.decision self._caps = result.caps @@ -290,9 +293,7 @@ def __exit__( def _handle_commit(self) -> None: elapsed_ms = int((time.monotonic() - self._start_time) * 1000) - actual, actual_from_estimate = _resolve_actual_cost( - self._usage, self._cost_fn, self._estimate.amount - ) + actual, actual_from_estimate = _resolve_actual_cost(self._usage, self._cost_fn, self._estimate.amount) ctx_metrics = self._ctx.metrics if self._ctx else None metrics = _build_stream_metrics(self._usage, elapsed_ms, ctx_metrics) unit = self._estimate.unit if isinstance(self._estimate.unit, str) else self._estimate.unit.value @@ -327,7 +328,9 @@ def _handle_commit(self) -> None: # honoring the server's Retry-After. logger.warning("Stream commit rate-limited; scheduling retry: id=%s", self._reservation_id) self._retry_engine.schedule( - self._reservation_id, commit_body, event_fallback, + self._reservation_id, + commit_body, + event_fallback, retry_after_ms=response.retry_after_ms_header, ) elif response.status in (401, 403): @@ -336,7 +339,8 @@ def _handle_commit(self) -> None: # that would return budget for real spend. logger.error( "Stream commit got authentication failure (status=%d); journaling for replay: id=%s", - response.status, self._reservation_id, + response.status, + self._reservation_id, ) self._retry_engine.schedule(self._reservation_id, commit_body, event_fallback) elif response.status == 410 or error_code == "RESERVATION_EXPIRED": @@ -357,7 +361,9 @@ def _handle_commit(self) -> None: logger.error( "Stream commit got unclassifiable client error (status=%d, error=%s); " "journaling for replay: id=%s", - response.status, error_code, self._reservation_id, + response.status, + error_code, + self._reservation_id, ) self._retry_engine.schedule(self._reservation_id, commit_body, event_fallback) else: @@ -395,6 +401,7 @@ def heartbeat_loop() -> None: pending_body: dict[str, Any] | None = None initial_remaining_ms = self._initial_remaining initial_rtt_ms = self._create_rtt + initial_received_ms = self._create_received_ms last_success_ms = anchor_ms held_delay_ms = min(ttl_ms / 2, 30_000.0) clamp_warned = False @@ -402,11 +409,21 @@ def heartbeat_loop() -> None: # a schema-valid 200 carrying remaining_ttl_ms drives scheduling # exactly; the measured-grant heuristic below is the # NON-NORMATIVE fallback for servers that omit the field. - sched = _AuthoritativeScheduler(_timeout_budget_ms(self._client._config)) + timeout_budget_ms = _timeout_budget_ms(self._client._config) + sched = _AuthoritativeScheduler(timeout_budget_ms) + if initial_rtt_ms is not None: + sched.observe_rtt(initial_rtt_ms) authoritative = initial_remaining_ms is not None if initial_remaining_ms is not None: + remaining_at_start = _remaining_at_schedule_start( + initial_remaining_ms, + initial_received_ms, + anchor_ms, + ) first_delay = sched.on_valid_success( - initial_remaining_ms, initial_rtt_ms or 0.0, anchor_ms, + remaining_at_start, + initial_rtt_ms or 0.0, + anchor_ms, ) # Unreachable on the first scheduler call (the zero-delay # streak cannot be exhausted yet); kept as a typed guard. @@ -438,40 +455,36 @@ def heartbeat_loop() -> None: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) pending_body = body sent_ms = _lifecycle._now_mono_ms() - response = self._client.extend_reservation(reservation_id, body) + response = _run_sync_attempt( + lambda: self._client.extend_reservation(reservation_id, body), + timeout_budget_ms, + ) + parsed_extend = _schema_valid_extend(response) recv_ms = _lifecycle._now_mono_ms() - if response.is_success and authoritative and response.status != 200: - # Ambiguous non-200 2xx in authoritative mode: NOT an - # observed success (spec) — same-key recovery. + if response.is_success and parsed_extend is None: logger.warning( - "Heartbeat ambiguous 2xx (status=%d): id=%s", - response.status, reservation_id, + "Heartbeat ambiguous response (status=%d): id=%s", + response.status, + reservation_id, ) - nxt = sched.on_transient_failure(_lifecycle._now_mono_ms()) - if nxt is None: - logger.warning( - "Heartbeat stopping: no safe recovery window remains: id=%s", - reservation_id, - ) - return - delay_ms = nxt - elif response.is_success: - pending_body = None - new_expires = response.get_body_attribute("expires_at_ms") - if new_expires is not None: - new_expires = int(new_expires) - if ctx is not None: - ctx.update_expires_at_ms(new_expires) - grant = ( - float(new_expires - prev_expiry) - if prev_expiry is not None - else float(ttl_ms) - ) - prev_expiry = new_expires + if authoritative: + nxt = sched.on_transient_failure(_lifecycle._now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt else: - grant = float(ttl_ms) - if prev_expiry is not None: - prev_expiry += ttl_ms + delay_ms = held_delay_ms + elif parsed_extend is not None: + pending_body = None + new_expires = parsed_extend.expires_at_ms + if ctx is not None: + ctx.update_expires_at_ms(new_expires) + grant = float(new_expires - prev_expiry) if prev_expiry is not None else float(ttl_ms) + prev_expiry = new_expires grant = max(grant, 0.0) now_ms = _lifecycle._now_mono_ms() elapsed_since_success = now_ms - last_success_ms @@ -479,14 +492,15 @@ def heartbeat_loop() -> None: grants_sum += grant last_grant = grant rtt_ms = recv_ms - sent_ms - remaining = response.get_body_attribute("remaining_ttl_ms") - if response.status == 200 and remaining is not None: + sched.observe_rtt(rtt_ms) + remaining = parsed_extend.remaining_ttl_ms + if remaining is not None: # Server-authoritative lease (spec v0.1.25.16 # PRIMARY ALGORITHM): schedule from this response # alone; the heuristic arms below only serve # servers that omit the field. authoritative = True - nxt = sched.on_valid_success(int(remaining), rtt_ms, recv_ms) + nxt = sched.on_valid_success(remaining, rtt_ms, recv_ms) if nxt is None: logger.warning( "Heartbeat stopping: lease shorter than the retry-safety budget: id=%s", @@ -523,23 +537,36 @@ def heartbeat_loop() -> None: if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: logger.warning( "Stream heartbeat stopping permanently (%s, status=%d): id=%s", - code, response.status, reservation_id, + code, + response.status, + reservation_id, ) return logger.warning("Stream heartbeat failed: id=%s", reservation_id) if authoritative: - rate_limited = response.status == 429 or code == "LIMIT_EXCEEDED" + rate_limited = response.status == 429 if not rate_limited and 400 <= response.status < 500: # Unrecoverable request/auth failure (spec): never # rotate the key on an unchanged request. logger.warning( "Heartbeat stopping on client error (%s, status=%d): id=%s", - code, response.status, reservation_id, + code, + response.status, + reservation_id, + ) + return + if not _lifecycle._extend_error_is_recoverable(response): + logger.warning( + "Heartbeat stopping on unexpected HTTP status %d: id=%s", + response.status, + reservation_id, ) return retry_after = response.retry_after_ms_header if rate_limited else None nxt = sched.on_transient_failure( - _lifecycle._now_mono_ms(), rate_limited=rate_limited, retry_after_ms=retry_after, + _lifecycle._now_mono_ms(), + rate_limited=rate_limited, + retry_after_ms=retry_after, ) if nxt is None: logger.warning( @@ -612,6 +639,7 @@ def __init__( self._reservation_id: str | None = None self._initial_remaining: int | None = None self._create_rtt: float | None = None + self._create_received_ms: float | None = None self._caps: Caps | None = None self._decision: Decision = Decision.ALLOW self._ctx: CyclesContext | None = None @@ -650,14 +678,9 @@ async def __aenter__(self) -> AsyncStreamReservation: self._grace_period_ms, ) - _create_sent_ms = _lifecycle._now_mono_ms() - response = await self._client.create_reservation(body) - _create_rtt_ms = _lifecycle._now_mono_ms() - _create_sent_ms - - if not response.is_success: - raise _build_protocol_exception("Failed to create reservation", response) - - result = ReservationCreateResponse.model_validate(response.body) + response, result, _create_rtt_ms, _create_received_ms = await _create_reservation_with_recovery_async( + self._client, body + ) if result.decision == Decision.DENY: raise _build_protocol_exception("Reservation denied", response) @@ -671,6 +694,7 @@ async def __aenter__(self) -> AsyncStreamReservation: self._reservation_id = result.reservation_id self._initial_remaining = result.remaining_ttl_ms self._create_rtt = _create_rtt_ms + self._create_received_ms = _create_received_ms self._decision = result.decision self._caps = result.caps @@ -723,9 +747,7 @@ async def __aexit__( async def _handle_commit(self) -> None: elapsed_ms = int((time.monotonic() - self._start_time) * 1000) - actual, actual_from_estimate = _resolve_actual_cost( - self._usage, self._cost_fn, self._estimate.amount - ) + actual, actual_from_estimate = _resolve_actual_cost(self._usage, self._cost_fn, self._estimate.amount) ctx_metrics = self._ctx.metrics if self._ctx else None metrics = _build_stream_metrics(self._usage, elapsed_ms, ctx_metrics) unit = self._estimate.unit if isinstance(self._estimate.unit, str) else self._estimate.unit.value @@ -758,11 +780,11 @@ async def _handle_commit(self) -> None: # Rate-limited, not rejected: releasing here would return # budget for spend that already happened. Retry instead, # honoring the server's Retry-After. - logger.warning( - "Async stream commit rate-limited; scheduling retry: id=%s", self._reservation_id - ) + logger.warning("Async stream commit rate-limited; scheduling retry: id=%s", self._reservation_id) self._retry_engine.schedule( - self._reservation_id, commit_body, event_fallback, + self._reservation_id, + commit_body, + event_fallback, retry_after_ms=response.retry_after_ms_header, ) elif response.status in (401, 403): @@ -771,7 +793,8 @@ async def _handle_commit(self) -> None: # that would return budget for real spend. logger.error( "Stream commit got authentication failure (status=%d); journaling for replay: id=%s", - response.status, self._reservation_id, + response.status, + self._reservation_id, ) self._retry_engine.schedule(self._reservation_id, commit_body, event_fallback) elif response.status == 410 or error_code == "RESERVATION_EXPIRED": @@ -792,7 +815,9 @@ async def _handle_commit(self) -> None: logger.error( "Async stream commit got unclassifiable client error (status=%d, error=%s); " "journaling for replay: id=%s", - response.status, error_code, self._reservation_id, + response.status, + error_code, + self._reservation_id, ) self._retry_engine.schedule(self._reservation_id, commit_body, event_fallback) else: @@ -831,6 +856,7 @@ async def heartbeat_loop() -> None: pending_body: dict[str, Any] | None = None initial_remaining_ms = self._initial_remaining initial_rtt_ms = self._create_rtt + initial_received_ms = self._create_received_ms last_success_ms = anchor_ms held_delay_ms = min(ttl_ms / 2, 30_000.0) clamp_warned = False @@ -838,11 +864,21 @@ async def heartbeat_loop() -> None: # a schema-valid 200 carrying remaining_ttl_ms drives scheduling # exactly; the measured-grant heuristic below is the # NON-NORMATIVE fallback for servers that omit the field. - sched = _AuthoritativeScheduler(_timeout_budget_ms(self._client._config)) + timeout_budget_ms = _timeout_budget_ms(self._client._config) + sched = _AuthoritativeScheduler(timeout_budget_ms) + if initial_rtt_ms is not None: + sched.observe_rtt(initial_rtt_ms) authoritative = initial_remaining_ms is not None if initial_remaining_ms is not None: + remaining_at_start = _remaining_at_schedule_start( + initial_remaining_ms, + initial_received_ms, + anchor_ms, + ) first_delay = sched.on_valid_success( - initial_remaining_ms, initial_rtt_ms or 0.0, anchor_ms, + remaining_at_start, + initial_rtt_ms or 0.0, + anchor_ms, ) # Unreachable on the first scheduler call (the zero-delay # streak cannot be exhausted yet); kept as a typed guard. @@ -877,40 +913,36 @@ async def heartbeat_loop() -> None: body = pending_body if pending_body is not None else _build_extend_body(ttl_ms) pending_body = body sent_ms = _lifecycle._now_mono_ms() - response = await client.extend_reservation(reservation_id, body) + response = await _run_async_attempt( + lambda: client.extend_reservation(reservation_id, body), + timeout_budget_ms, + ) + parsed_extend = _schema_valid_extend(response) recv_ms = _lifecycle._now_mono_ms() - if response.is_success and authoritative and response.status != 200: - # Ambiguous non-200 2xx in authoritative mode: NOT an - # observed success (spec) — same-key recovery. + if response.is_success and parsed_extend is None: logger.warning( - "Heartbeat ambiguous 2xx (status=%d): id=%s", - response.status, reservation_id, + "Heartbeat ambiguous response (status=%d): id=%s", + response.status, + reservation_id, ) - nxt = sched.on_transient_failure(_lifecycle._now_mono_ms()) - if nxt is None: - logger.warning( - "Heartbeat stopping: no safe recovery window remains: id=%s", - reservation_id, - ) - return - delay_ms = nxt - elif response.is_success: - pending_body = None - new_expires = response.get_body_attribute("expires_at_ms") - if new_expires is not None: - new_expires = int(new_expires) - if ctx is not None: - ctx.update_expires_at_ms(new_expires) - grant = ( - float(new_expires - prev_expiry) - if prev_expiry is not None - else float(ttl_ms) - ) - prev_expiry = new_expires + if authoritative: + nxt = sched.on_transient_failure(_lifecycle._now_mono_ms()) + if nxt is None: + logger.warning( + "Heartbeat stopping: no safe recovery window remains: id=%s", + reservation_id, + ) + return + delay_ms = nxt else: - grant = float(ttl_ms) - if prev_expiry is not None: - prev_expiry += ttl_ms + delay_ms = held_delay_ms + elif parsed_extend is not None: + pending_body = None + new_expires = parsed_extend.expires_at_ms + if ctx is not None: + ctx.update_expires_at_ms(new_expires) + grant = float(new_expires - prev_expiry) if prev_expiry is not None else float(ttl_ms) + prev_expiry = new_expires grant = max(grant, 0.0) now_ms = _lifecycle._now_mono_ms() elapsed_since_success = now_ms - last_success_ms @@ -918,14 +950,15 @@ async def heartbeat_loop() -> None: grants_sum += grant last_grant = grant rtt_ms = recv_ms - sent_ms - remaining = response.get_body_attribute("remaining_ttl_ms") - if response.status == 200 and remaining is not None: + sched.observe_rtt(rtt_ms) + remaining = parsed_extend.remaining_ttl_ms + if remaining is not None: # Server-authoritative lease (spec v0.1.25.16 # PRIMARY ALGORITHM): schedule from this response # alone; the heuristic arms below only serve # servers that omit the field. authoritative = True - nxt = sched.on_valid_success(int(remaining), rtt_ms, recv_ms) + nxt = sched.on_valid_success(remaining, rtt_ms, recv_ms) if nxt is None: logger.warning( "Heartbeat stopping: lease shorter than the retry-safety budget: id=%s", @@ -962,23 +995,36 @@ async def heartbeat_loop() -> None: if response.status in (404, 410) or code in _PERMANENT_EXTEND_CODES: logger.warning( "Async stream heartbeat stopping permanently (%s, status=%d): id=%s", - code, response.status, reservation_id, + code, + response.status, + reservation_id, ) return logger.warning("Async stream heartbeat failed: id=%s", reservation_id) if authoritative: - rate_limited = response.status == 429 or code == "LIMIT_EXCEEDED" + rate_limited = response.status == 429 if not rate_limited and 400 <= response.status < 500: # Unrecoverable request/auth failure (spec): never # rotate the key on an unchanged request. logger.warning( "Heartbeat stopping on client error (%s, status=%d): id=%s", - code, response.status, reservation_id, + code, + response.status, + reservation_id, + ) + return + if not _lifecycle._extend_error_is_recoverable(response): + logger.warning( + "Heartbeat stopping on unexpected HTTP status %d: id=%s", + response.status, + reservation_id, ) return retry_after = response.retry_after_ms_header if rate_limited else None nxt = sched.on_transient_failure( - _lifecycle._now_mono_ms(), rate_limited=rate_limited, retry_after_ms=retry_after, + _lifecycle._now_mono_ms(), + rate_limited=rate_limited, + retry_after_ms=retry_after, ) if nxt is None: logger.warning( diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 926d77f..e2ef667 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -12,9 +12,21 @@ import pytest -import runcycles.lifecycle as lifecycle_mod from runcycles.config import CyclesConfig -from runcycles.lifecycle import AsyncCyclesLifecycle, CyclesLifecycle, DecoratorConfig +from runcycles.exceptions import CyclesProtocolError +from runcycles.lifecycle import ( + AsyncCyclesLifecycle, + CyclesLifecycle, + DecoratorConfig, + _AuthoritativeScheduler, + _create_reservation_with_recovery, + _create_reservation_with_recovery_async, + _remaining_at_schedule_start, + _run_async_attempt, + _run_sync_attempt, + _schema_valid_create, + _schema_valid_extend, +) from runcycles.models import Action, Amount, Subject, Unit from runcycles.response import CyclesResponse from runcycles.retry import AsyncCommitRetryEngine, CommitRetryEngine @@ -34,13 +46,16 @@ def now(self) -> float: def _config() -> CyclesConfig: return CyclesConfig( - base_url="http://localhost:7878", api_key="test-key", tenant="acme", + base_url="http://localhost:7878", + api_key="test-key", + tenant="acme", retry_enabled=False, ) def _extend_ok( - expires_at_ms: int | None, remaining_ttl_ms: int | None = None, + expires_at_ms: int | None, + remaining_ttl_ms: int | None = None, ) -> CyclesResponse: body: dict[str, Any] = {"status": "ACTIVE"} if expires_at_ms is not None: @@ -51,14 +66,17 @@ def _extend_ok( def _allow_response() -> CyclesResponse: - return CyclesResponse.success(200, { - "decision": "ALLOW", - "reservation_id": "rsv_test", - "expires_at_ms": int(time.time() * 1000) + 600_000, - "affected_scopes": ["tenant:acme"], - "scope_path": "tenant:acme", - "reserved": {"unit": "USD_MICROCENTS", "amount": 1000}, - }) + return CyclesResponse.success( + 200, + { + "decision": "ALLOW", + "reservation_id": "rsv_test", + "expires_at_ms": int(time.time() * 1000) + 600_000, + "affected_scopes": ["tenant:acme"], + "scope_path": "tenant:acme", + "reserved": {"unit": "USD_MICROCENTS", "amount": 1000}, + }, + ) def _commit_success() -> CyclesResponse: @@ -96,7 +114,7 @@ def _run_sync_beats( ) -> list[float]: """Drive the sync heartbeat for `beats` iterations, advancing the fake clock by the beat interval on every wait. Returns the wait timeouts.""" - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) timeouts: list[float] = [] calls = {"n": 0} @@ -111,7 +129,11 @@ def wait(timeout: float | None = None) -> bool: stop = threading.Event() stop.wait = wait # type: ignore[method-assign] thread = lifecycle._start_heartbeat( - "rsv_1", ttl, ctx or _ctx(), stop, initial_remaining_ms, + "rsv_1", + ttl, + ctx or _ctx(), + stop, + initial_remaining_ms, ) assert thread is not None thread.join(timeout=5) @@ -119,17 +141,201 @@ def wait(timeout: float | None = None) -> bool: return timeouts +class TestStrictLeaseAttemptContract: + def test_field_mode_retains_rtt_observed_before_field_appears(self) -> None: + scheduler = _AuthoritativeScheduler(7_000) + scheduler.observe_rtt(6_000) + + assert scheduler.on_valid_success(60_000, 100, 0) == 23_900 + + @pytest.mark.parametrize( + ("received_ms", "now_ms", "expected"), + [(1000.0, 3000.0, 58_000.0), (3000.0, 1000.0, 0.0), (None, 3000.0, 60_000.0)], + ) + def test_create_lead_deducts_post_receipt_setup_time( + self, + received_ms: float | None, + now_ms: float, + expected: float, + ) -> None: + assert _remaining_at_schedule_start(60_000, received_ms, now_ms) == expected + + def test_sync_create_recovers_once_with_same_body(self) -> None: + client = MagicMock() + client._config = _config() + body = {"idempotency_key": "same-key"} + client.create_reservation.side_effect = [ + CyclesResponse.success(202, _allow_response().body), + _allow_response(), + ] + + response, parsed, rtt_ms, received_ms = _create_reservation_with_recovery(client, body) + + assert response.status == 200 + assert parsed.reservation_id == "rsv_test" + assert rtt_ms >= 0 + assert received_ms >= rtt_ms + assert client.create_reservation.call_count == 2 + assert all(call.args[0] is body for call in client.create_reservation.call_args_list) + + def test_sync_create_rejects_malformed_200_after_one_recovery(self) -> None: + client = MagicMock() + client._config = _config() + malformed = CyclesResponse.success( + 200, + { + "decision": "ALLOW", + "reservation_id": "rsv_test", + "affected_scopes": ["tenant:acme"], + "unexpected": True, + }, + ) + client.create_reservation.return_value = malformed + + with pytest.raises(CyclesProtocolError, match="schema-valid HTTP 200"): + _create_reservation_with_recovery(client, {"idempotency_key": "same-key"}) + + assert client.create_reservation.call_count == 2 + + @pytest.mark.asyncio + async def test_async_create_recovers_transport_failure_with_same_body(self) -> None: + client = AsyncMock() + client._config = _config() + body = {"idempotency_key": "same-key"} + client.create_reservation.side_effect = [ConnectionError("lost"), _allow_response()] + + response, parsed, rtt_ms, received_ms = await _create_reservation_with_recovery_async(client, body) + + assert response.status == 200 + assert parsed.reservation_id == "rsv_test" + assert rtt_ms >= 0 + assert received_ms >= rtt_ms + assert client.create_reservation.await_count == 2 + assert all(call.args[0] is body for call in client.create_reservation.call_args_list) + + @pytest.mark.parametrize( + "response", + [ + CyclesResponse.success( + 202, + {"status": "ACTIVE", "expires_at_ms": 1, "remaining_ttl_ms": 0}, + ), + CyclesResponse.success(200, {"status": "ACTIVE"}), + CyclesResponse.success(200, {"status": "ACTIVE", "expires_at_ms": -1}), + CyclesResponse.success(200, {"status": "UNKNOWN", "expires_at_ms": 1}), + CyclesResponse.success(200, {"status": "ACTIVE", "expires_at_ms": 1, "balances": {}}), + CyclesResponse.success(200, {"status": "ACTIVE", "expires_at_ms": 1, "extra": True}), + ], + ) + def test_extend_rejects_non_200_or_non_schema_response( + self, + response: CyclesResponse, + ) -> None: + assert _schema_valid_extend(response) is None + + def test_strict_create_and_extend_accept_schema_valid_200(self) -> None: + assert _schema_valid_create(_allow_response()) is not None + parsed = _schema_valid_extend( + CyclesResponse.success( + 200, + {"status": "ACTIVE", "expires_at_ms": 1, "remaining_ttl_ms": 0, "balances": []}, + ), + ) + assert parsed is not None + assert parsed.remaining_ttl_ms == 0 + + @pytest.mark.parametrize( + "body", + [ + {"decision": "ALLOW", "affected_scopes": [], "caps": None}, + { + "decision": "ALLOW", + "affected_scopes": [], + "reserved": {"unit": "TOKENS", "amount": "1"}, + }, + { + "decision": "ALLOW", + "affected_scopes": [], + "balances": [ + { + "scope": "tenant:acme", + "scope_path": "tenant:acme", + "remaining": {"unit": "TOKENS", "amount": 1}, + "reserved": None, + } + ], + }, + { + "decision": "ALLOW", + "affected_scopes": [], + "cycles_evidence": { + "evidence_id": "a" * 64, + "cycles_evidence_url": "relative", + }, + }, + { + "decision": "ALLOW", + "affected_scopes": [], + "remaining_ttl_ms": 9_223_372_036_854_775_808, + }, + { + "decision": "ALLOW", + "affected_scopes": [], + "expires_at_ms": -1, + }, + ], + ) + def test_create_rejects_non_schema_optional_fields(self, body: dict[str, Any]) -> None: + assert _schema_valid_create(CyclesResponse.success(200, body)) is None + + def test_extend_rejects_nested_null_and_int64_overflow(self) -> None: + for body in ( + { + "status": "ACTIVE", + "expires_at_ms": 1, + "balances": [ + { + "scope": "tenant:acme", + "scope_path": "tenant:acme", + "remaining": {"unit": "TOKENS", "amount": 1}, + "debt": None, + } + ], + }, + { + "status": "ACTIVE", + "expires_at_ms": 9_223_372_036_854_775_808, + }, + ): + assert _schema_valid_extend(CyclesResponse.success(200, body)) is None + + def test_sync_outer_deadline_covers_complete_attempt(self) -> None: + with pytest.raises(TimeoutError, match="exceeded"): + _run_sync_attempt( + lambda: (time.sleep(0.05), _allow_response())[1], + timeout_budget_ms=1, + ) + + @pytest.mark.asyncio + async def test_async_outer_deadline_covers_complete_attempt(self) -> None: + async def slow() -> CyclesResponse: + await asyncio.sleep(0.05) + return _allow_response() + + with pytest.raises(asyncio.TimeoutError): + await _run_async_attempt(slow, timeout_budget_ms=1) + + class TestSyncHeartbeatLeadEstimate: def test_extends_only_when_lead_below_threshold( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # lead_min starts at 0 and the first beat fires IMMEDIATELY: beats # 1-3 extend (grants measured at +ttl each), beat 4 skips once # lead_min reaches 1.5*grant (180k-90k=90k), beat 5 extends again. lifecycle, client = _make_sync() - client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4) - ] + client.extend_reservation.side_effect = [_extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4)] timeouts = _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=5) @@ -138,7 +344,8 @@ def test_extends_only_when_lead_below_threshold( assert timeouts[1] == min(TTL / 2, 30_000) / 1000.0 def test_interval_has_no_floor_for_small_ttl( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # ttl=1200 → after the immediate first beat, cadence must be 600ms # (the old 1s floor guaranteed lapse in this spec-legal range). @@ -147,14 +354,20 @@ def test_interval_has_no_floor_for_small_ttl( ctx = _ctx(1200) timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=2, ttl=1200, ctx=ctx, + lifecycle, + FakeClock(), + monkeypatch, + beats=2, + ttl=1200, + ctx=ctx, ) assert timeouts[0] == 0.0 assert timeouts[1] == 0.6 def test_failed_extend_retries_with_same_idempotency_key( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # A lost/failed extend may have been applied server-side: the retry # must reuse the same key so it cannot double-extend. After a @@ -174,11 +387,13 @@ def test_failed_extend_retries_with_same_idempotency_key( assert bodies[2]["idempotency_key"] != bodies[0]["idempotency_key"] def test_permanent_code_stops_heartbeat( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle, client = _make_sync() client.extend_reservation.return_value = CyclesResponse.http_error( - 409, "capped", + 409, + "capped", body={"error": "MAX_EXTENSIONS_EXCEEDED", "message": "m", "request_id": "r"}, ) @@ -188,14 +403,13 @@ def test_permanent_code_stops_heartbeat( assert client.extend_reservation.call_count == 1 def test_clamped_grants_extend_every_beat( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # Server clamps each grant to ttl/4: the lead estimate sees the # small grants (authoritative expires_at) and keeps extending. lifecycle, client = _make_sync() - client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + (n + 1) * (TTL // 4)) for n in range(3) - ] + client.extend_reservation.side_effect = [_extend_ok(INITIAL_EXPIRY + (n + 1) * (TTL // 4)) for n in range(3)] timeouts = _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=3) @@ -204,7 +418,8 @@ def test_clamped_grants_extend_every_beat( assert timeouts[1] == (TTL / 4 / 2) / 1000.0 def test_grant_clamp_misclassification_after_skip_is_transient( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # After a skip, the next measured grant arrives across a doubled # gap, so grant ≈ elapsed and the beat lands in the lead-clamp arm @@ -214,9 +429,7 @@ def test_grant_clamp_misclassification_after_skip_is_transient( # band the hold sticks and a ttl/4-grant lease decays to a lapse. lifecycle, client = _make_sync() grant = TTL // 4 # 15000 → cadence ttl/8 = 7500ms - client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + (n + 1) * grant) for n in range(6) - ] + client.extend_reservation.side_effect = [_extend_ok(INITIAL_EXPIRY + (n + 1) * grant) for n in range(6)] timeouts = _run_sync_beats(lifecycle, FakeClock(), monkeypatch, beats=7) @@ -227,7 +440,8 @@ def test_grant_clamp_misclassification_after_skip_is_transient( assert timeouts == [0.0, 7.5, 7.5, 7.5, 7.5, 30.0, 7.5, 7.5] def test_missing_expires_in_response_falls_back_to_plus_ttl( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle, client = _make_sync() client.extend_reservation.return_value = _extend_ok(None) @@ -241,11 +455,12 @@ def test_missing_expires_in_response_falls_back_to_plus_ttl( ctx.update_expires_at_ms.assert_not_called() def test_unknown_initial_expiry_anchors_on_first_success( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ - _extend_ok(500_000), # beat 1: fallback grant, sets frame + _extend_ok(500_000), # beat 1: fallback grant, sets frame _extend_ok(500_000 + 3 * TTL), # beat 2: big measured grant ] @@ -259,11 +474,13 @@ def test_unknown_initial_expiry_anchors_on_first_success( assert client.extend_reservation.call_count == 3 def test_tenant_closed_stops_heartbeat( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle, client = _make_sync() client.extend_reservation.return_value = CyclesResponse.http_error( - 409, "closed", + 409, + "closed", body={"error": "TENANT_CLOSED", "message": "m", "request_id": "r"}, ) @@ -272,7 +489,8 @@ def test_tenant_closed_stops_heartbeat( assert client.extend_reservation.call_count == 1 def test_transient_failure_then_recovery( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # Non-permanent failures warn and keep the loop alive. lifecycle, client = _make_sync() @@ -297,7 +515,7 @@ async def _run( ctx: MagicMock | None = None, ) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) count = 0 async def fake_sleep(s: float) -> None: @@ -313,12 +531,11 @@ async def fake_sleep(s: float) -> None: await task async def test_extends_only_when_lead_below_threshold( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle, client = _make_async() - client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4) - ] + client.extend_reservation.side_effect = [_extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4)] await self._run(lifecycle, 5, monkeypatch) @@ -327,7 +544,8 @@ async def test_extends_only_when_lead_below_threshold( assert client.extend_reservation.await_count == 4 async def test_permanent_code_stops_heartbeat( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle, client = _make_async() client.extend_reservation.return_value = CyclesResponse.http_error(410, "gone") @@ -337,15 +555,16 @@ async def test_permanent_code_stops_heartbeat( assert client.extend_reservation.await_count == 1 async def test_transient_failure_missing_expires_and_late_anchor( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # Covers the async warn-continue, +=ttl fallback, and late-anchor branches. lifecycle, client = _make_async() client.extend_reservation.side_effect = [ CyclesResponse.http_error(500, "boom"), - _extend_ok(None), # frame not yet anchored - _extend_ok(700_000), # late anchor - _extend_ok(None), # anchored: += ttl fallback + _extend_ok(None), # frame not yet anchored + _extend_ok(700_000), # late anchor + _extend_ok(None), # anchored: += ttl fallback ] await self._run(lifecycle, 4, monkeypatch, ctx=_ctx(None)) @@ -355,15 +574,14 @@ async def test_transient_failure_missing_expires_and_late_anchor( class TestStreamingHeartbeatLeadEstimate: def test_sync_stream_lead_estimate_pattern( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) client = MagicMock() client._config = _config() - client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4) - ] + client.extend_reservation.side_effect = [_extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4)] stream = StreamReservation( client, subject=Subject(tenant="acme"), @@ -390,20 +608,21 @@ def wait(timeout: float | None = None) -> bool: assert client.extend_reservation.call_count == 4 def test_sync_stream_permanent_and_fallback_branches( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # Covers the sync-stream +=ttl fallback, late-anchor, transient-warn, # and permanent-stop branches in one deterministic run. clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) client = MagicMock() client._config = _config() client.extend_reservation.side_effect = [ - CyclesResponse.http_error(500, "boom"), # transient: warn, retry - _extend_ok(None), # no anchor yet: no-op - _extend_ok(900_000), # late anchor - _extend_ok(None), # anchored: += ttl fallback - CyclesResponse.http_error(410, "gone"), # permanent: stop + CyclesResponse.http_error(500, "boom"), # transient: warn, retry + _extend_ok(None), # no anchor yet: no-op + _extend_ok(900_000), # late anchor + _extend_ok(None), # anchored: += ttl fallback + CyclesResponse.http_error(410, "gone"), # permanent: stop ] stream = StreamReservation( client, @@ -432,19 +651,20 @@ def wait(timeout: float | None = None) -> bool: assert client.extend_reservation.call_count == 5 def test_sync_stream_field_mode_cycle( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # Authoritative mode in the sync stream heartbeat: create field # drives the first delay; 503 and exception retries stay bounded. clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) client = MagicMock() client._config = _config() client.extend_reservation.side_effect = [ - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + TTL, remaining_ttl_ms=60_000), CyclesResponse.http_error(503, "unavailable"), ConnectionError("down"), - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + 2 * TTL, remaining_ttl_ms=60_000), ] stream = StreamReservation( client, @@ -479,17 +699,18 @@ def wait(timeout: float | None = None) -> bool: @pytest.mark.asyncio async def test_async_stream_field_mode_cycle( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) client = AsyncMock() client._config = _config() client.extend_reservation.side_effect = [ - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + TTL, remaining_ttl_ms=60_000), CyclesResponse.http_error(503, "unavailable"), ConnectionError("down"), - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + 2 * TTL, remaining_ttl_ms=60_000), ] stream = AsyncStreamReservation( client, @@ -524,15 +745,14 @@ async def fake_sleep(s: float) -> None: @pytest.mark.asyncio async def test_async_stream_lead_estimate_pattern( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) client = AsyncMock() client._config = _config() - client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4) - ] + client.extend_reservation.side_effect = [_extend_ok(INITIAL_EXPIRY + (n + 1) * TTL) for n in range(4)] stream = AsyncStreamReservation( client, subject=Subject(tenant="acme"), @@ -560,17 +780,18 @@ async def fake_sleep(s: float) -> None: @pytest.mark.asyncio async def test_async_stream_permanent_and_fallback_branches( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) client = AsyncMock() client._config = _config() client.extend_reservation.side_effect = [ CyclesResponse.http_error(500, "boom"), - _extend_ok(None), # no anchor yet: no-op - _extend_ok(900_000), # late anchor - _extend_ok(None), # anchored: += ttl fallback + _extend_ok(None), # no anchor yet: no-op + _extend_ok(900_000), # late anchor + _extend_ok(None), # anchored: += ttl fallback CyclesResponse.http_error(410, "gone"), ] stream = AsyncStreamReservation( @@ -611,17 +832,25 @@ class TestAuthoritativeScheduling: RESERVE = 25_000.0 def test_create_remaining_drives_first_beat_and_steady_cadence( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # remaining=60000 → next_delay = 60000 − 25000 = 35000ms. Every # extend echoes the field, so the cadence holds at 35s and the # heuristic lead_min skip NEVER fires even though accumulated # fallback grants would trip it. lifecycle, client = _make_sync() - client.extend_reservation.return_value = _extend_ok(None, remaining_ttl_ms=60_000) + client.extend_reservation.return_value = _extend_ok( + INITIAL_EXPIRY + TTL, + remaining_ttl_ms=60_000, + ) timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=4, initial_remaining_ms=60_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=4, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 4 @@ -638,12 +867,19 @@ def test_lease_below_reserve_one_immediate_attempt_then_stop( # and surface (spec zero-delay guard) rather than tight-loop a # maximum-lead server's extension budget away. lifecycle, client = _make_sync() - client.extend_reservation.return_value = _extend_ok(None, remaining_ttl_ms=1_000) + client.extend_reservation.return_value = _extend_ok( + INITIAL_EXPIRY + TTL, + remaining_ttl_ms=1_000, + ) with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=4, - ttl=86_400_000, initial_remaining_ms=1_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=4, + ttl=86_400_000, + initial_remaining_ms=1_000, ) # Immediate first beat (streak 1), its success hits streak 2 → stop. @@ -670,7 +906,11 @@ def clamped_extend(rid: str, body: dict[str, Any]) -> CyclesResponse: with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): timeouts = _run_sync_beats( - lifecycle, clock, monkeypatch, beats=3, initial_remaining_ms=60_000, + lifecycle, + clock, + monkeypatch, + beats=3, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 3 @@ -678,18 +918,23 @@ def clamped_extend(rid: str, body: dict[str, Any]) -> CyclesResponse: assert not [r for r in caplog.records if "clamp lease lead" in r.message] def test_field_disappearing_mid_flight_resumes_heuristic( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # Beat 1's response lacks the field → the v2.3+band heuristic takes # over seamlessly from its maintained bookkeeping. lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ - _extend_ok(INITIAL_EXPIRY + TTL), # no field: fallback + _extend_ok(INITIAL_EXPIRY + TTL), # no field: fallback _extend_ok(INITIAL_EXPIRY + 2 * TTL), ] timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=2, initial_remaining_ms=60_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=2, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 2 @@ -699,7 +944,8 @@ def test_field_disappearing_mid_flight_resumes_heuristic( assert timeouts[1] == 30.0 def test_transient_failure_recovery_window_shrinks_then_same_key( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # 503 at the scheduled beat (t=35s): lead_est = 60000 − 35000 = # 25000; retry_window = 25000 − 12000 − 1000 = 12000 → retry after @@ -707,11 +953,15 @@ def test_transient_failure_recovery_window_shrinks_then_same_key( lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ CyclesResponse.http_error(503, "unavailable"), - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + TTL, remaining_ttl_ms=60_000), ] timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=2, initial_remaining_ms=60_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=2, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 2 @@ -733,7 +983,11 @@ def test_repeated_failures_stop_when_no_window_progress( with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=10, initial_remaining_ms=60_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=10, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 5 @@ -743,18 +997,23 @@ def test_repeated_failures_stop_when_no_window_progress( assert len(keys) == 1 # every recovery reused the same key def test_ambiguous_2xx_is_not_applied_same_key_recovery( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # A non-200 2xx in authoritative mode is ambiguous (spec): never # scheduled from — recovered with the SAME idempotency key. lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ CyclesResponse.success(202, {"status": "ACTIVE", "remaining_ttl_ms": 60_000}), - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + 2 * TTL, remaining_ttl_ms=60_000), ] timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=2, initial_remaining_ms=60_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=2, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 2 @@ -763,17 +1022,22 @@ def test_ambiguous_2xx_is_not_applied_same_key_recovery( assert bodies[0]["idempotency_key"] == bodies[1]["idempotency_key"] def test_429_honored_only_within_window( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # Retry-After 3s ≤ window 12000 → retried after exactly 3s, same key. lifecycle, client = _make_sync() client.extend_reservation.side_effect = [ CyclesResponse.http_error(429, "limited", headers={"retry-after": "3"}), - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + TTL, remaining_ttl_ms=60_000), ] timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=2, initial_remaining_ms=60_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=2, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 2 @@ -790,12 +1054,18 @@ def test_429_missing_or_oversized_retry_after_stops( # re-invented earlier: stop and surface. lifecycle, client = _make_sync() client.extend_reservation.return_value = CyclesResponse.http_error( - 429, "limited", headers={"retry-after": "20"}, + 429, + "limited", + headers={"retry-after": "20"}, ) with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=4, initial_remaining_ms=60_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=4, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 1 @@ -808,32 +1078,58 @@ def test_other_4xx_stops_without_key_rotation( ) -> None: lifecycle, client = _make_sync() client.extend_reservation.return_value = CyclesResponse.http_error( - 400, "bad", + 400, + "bad", body={"error": "INVALID_REQUEST", "message": "m", "request_id": "r"}, ) with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=4, initial_remaining_ms=60_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=4, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 1 assert [r for r in caplog.records if "client error" in r.message] + def test_unexpected_3xx_stops_without_retry( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + lifecycle, client = _make_sync() + client.extend_reservation.return_value = CyclesResponse.http_error(302, "redirect") + + with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): + _run_sync_beats( + lifecycle, + FakeClock(), + monkeypatch, + beats=4, + initial_remaining_ms=60_000, + ) + + assert client.extend_reservation.call_count == 1 + assert [r for r in caplog.records if "unexpected HTTP status" in r.message] + @pytest.mark.asyncio async def test_async_field_mode_full_cycle( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # Covers the async authoritative arms: initial field delay, 503 # recovery, exception recovery with a shrinking window, success. clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) lifecycle, client = _make_async() client.extend_reservation.side_effect = [ - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + TTL, remaining_ttl_ms=60_000), CyclesResponse.http_error(503, "unavailable"), ConnectionError("down"), - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + 2 * TTL, remaining_ttl_ms=60_000), ] count = 0 sleeps: list[float] = [] @@ -852,8 +1148,8 @@ async def fake_sleep(s: float) -> None: await task assert client.extend_reservation.await_count == 4 - assert sleeps[0] == 35.0 # from the create field - assert sleeps[2] == 6.25 # 503: window 12000, lead_est/4 = 6250 + assert sleeps[0] == 35.0 # from the create field + assert sleeps[2] == 6.25 # 503: window 12000, lead_est/4 = 6250 assert sleeps[3] == 4.6875 # exception: window shrank to 5750 def test_scheduler_edge_cases_direct(self) -> None: @@ -870,6 +1166,10 @@ def test_scheduler_edge_cases_direct(self) -> None: sched2 = _AuthoritativeScheduler(12_000.0) sched2.on_valid_success(60_000, 0.0, 0.0) assert sched2.on_transient_failure(35_000.0, rate_limited=True, retry_after_ms=-1) is None + # Unknown/backward timing never fabricates lease lead. + sched3 = _AuthoritativeScheduler(12_000.0) + assert sched3.on_valid_success(60_000, -1.0, 0.0) == 0.0 + assert sched3.lead_estimate_ms(-1.0) == 0.0 def test_sync_repeated_ambiguous_2xx_stops( self, @@ -881,7 +1181,11 @@ def test_sync_repeated_ambiguous_2xx_stops( with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=10, initial_remaining_ms=60_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=10, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 5 @@ -897,7 +1201,11 @@ def test_sync_repeated_exceptions_stop( with caplog.at_level(logging.WARNING, logger="runcycles.lifecycle"): _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=10, initial_remaining_ms=60_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=10, + initial_remaining_ms=60_000, ) assert client.extend_reservation.call_count == 5 @@ -930,10 +1238,18 @@ def test_create_response_model_parses_remaining(self) -> None: _STOP_MATRIX = [ ("s503", lambda: CyclesResponse.http_error(503, "down"), 60_000, 5), ("s202", lambda: CyclesResponse.success(202, {"status": "ACTIVE"}), 60_000, 5), - ("s400", lambda: CyclesResponse.http_error( - 400, "bad", body={"error": "INVALID_REQUEST", "message": "m", "request_id": "r"}, - ), 60_000, 1), - ("szero", lambda: _extend_ok(None, remaining_ttl_ms=1_000), 1_000, 1), + ( + "s400", + lambda: CyclesResponse.http_error( + 400, + "bad", + body={"error": "INVALID_REQUEST", "message": "m", "request_id": "r"}, + ), + 60_000, + 1, + ), + ("s302", lambda: CyclesResponse.http_error(302, "redirect"), 60_000, 1), + ("szero", lambda: _extend_ok(INITIAL_EXPIRY + TTL, remaining_ttl_ms=1_000), 1_000, 1), ("sexc", lambda: ConnectionError("down"), 60_000, 5), ] @@ -944,6 +1260,7 @@ def _effect(*args: Any, **kwargs: Any) -> CyclesResponse: if isinstance(result, Exception): raise result return result + return _effect @@ -955,11 +1272,15 @@ class TestAuthoritativeStopMatrix: ) @pytest.mark.asyncio async def test_async_lifecycle_arms( - self, name: str, factory: Any, initial: int, expected: int, + self, + name: str, + factory: Any, + initial: int, + expected: int, monkeypatch: pytest.MonkeyPatch, ) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) lifecycle, client = _make_async() client.extend_reservation.side_effect = _matrix_side_effect(factory) count = 0 @@ -980,14 +1301,15 @@ async def fake_sleep(s: float) -> None: @pytest.mark.asyncio async def test_async_lifecycle_429_honored( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) lifecycle, client = _make_async() client.extend_reservation.side_effect = [ CyclesResponse.http_error(429, "limited", headers={"retry-after": "3"}), - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + TTL, remaining_ttl_ms=60_000), ] count = 0 sleeps: list[float] = [] @@ -1028,11 +1350,15 @@ def _make_stream(self) -> tuple[StreamReservation, MagicMock]: ids=[m[0] for m in _STOP_MATRIX], ) def test_sync_stream_arms( - self, name: str, factory: Any, initial: int, expected: int, + self, + name: str, + factory: Any, + initial: int, + expected: int, monkeypatch: pytest.MonkeyPatch, ) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) stream, client = self._make_stream() client.extend_reservation.side_effect = _matrix_side_effect(factory) stream._initial_remaining = initial @@ -1054,11 +1380,11 @@ def wait(timeout: float | None = None) -> bool: def test_sync_stream_429_honored(self, monkeypatch: pytest.MonkeyPatch) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) stream, client = self._make_stream() client.extend_reservation.side_effect = [ CyclesResponse.http_error(429, "limited", headers={"retry-after": "3"}), - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + TTL, remaining_ttl_ms=60_000), ] stream._initial_remaining = 60_000 timeouts: list[float] = [] @@ -1101,11 +1427,15 @@ def _make_async_stream(self) -> tuple[AsyncStreamReservation, AsyncMock]: ) @pytest.mark.asyncio async def test_async_stream_arms( - self, name: str, factory: Any, initial: int, expected: int, + self, + name: str, + factory: Any, + initial: int, + expected: int, monkeypatch: pytest.MonkeyPatch, ) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) stream, client = self._make_async_stream() client.extend_reservation.side_effect = _matrix_side_effect(factory) stream._initial_remaining = initial @@ -1127,14 +1457,15 @@ async def fake_sleep(s: float) -> None: @pytest.mark.asyncio async def test_async_stream_429_honored( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: clock = FakeClock() - monkeypatch.setattr(lifecycle_mod, "_now_mono_ms", clock.now) + monkeypatch.setattr("runcycles.lifecycle._now_mono_ms", clock.now) stream, client = self._make_async_stream() client.extend_reservation.side_effect = [ CyclesResponse.http_error(429, "limited", headers={"retry-after": "3"}), - _extend_ok(None, remaining_ttl_ms=60_000), + _extend_ok(INITIAL_EXPIRY + TTL, remaining_ttl_ms=60_000), ] stream._initial_remaining = 60_000 count = 0 @@ -1166,16 +1497,15 @@ async def fake_sleep(s: float) -> None: class TestServerDateAccessor: def test_server_date_ms_parses_http_date(self) -> None: response = CyclesResponse.success( - 200, {}, headers={"date": "Mon, 27 Jul 2026 12:00:00 GMT"}, + 200, + {}, + headers={"date": "Mon, 27 Jul 2026 12:00:00 GMT"}, ) assert response.server_date_ms == 1785153600000 def test_server_date_ms_absent_or_garbage(self) -> None: assert CyclesResponse.success(200, {}).server_date_ms is None - assert ( - CyclesResponse.success(200, {}, headers={"date": "not a date"}).server_date_ms - is None - ) + assert CyclesResponse.success(200, {}, headers={"date": "not a date"}).server_date_ms is None # --------------------------------------------------------------------------- @@ -1191,7 +1521,8 @@ def _cfg(**kwargs: Any) -> DecoratorConfig: class TestFirstBeatAndRegimes: def test_first_beat_is_immediate_even_for_huge_ttl( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # A 24h request silently capped to a small lease by tenant policy # must still survive: only a zero first delay guarantees the first @@ -1199,12 +1530,17 @@ def test_first_beat_is_immediate_even_for_huge_ttl( lifecycle, client = _make_sync() client.extend_reservation.return_value = _extend_ok(None) timeouts = _run_sync_beats( - lifecycle, FakeClock(), monkeypatch, beats=1, ttl=86_400_000, + lifecycle, + FakeClock(), + monkeypatch, + beats=1, + ttl=86_400_000, ) assert timeouts[0] == 0.0 def test_first_beat_failure_does_not_hot_loop( - self, monkeypatch: pytest.MonkeyPatch, + self, + monkeypatch: pytest.MonkeyPatch, ) -> None: # A transient failure on the primed (delay-0) beat must back off to # the held cadence, not spin at 0ms against a down server. diff --git a/tests/test_response.py b/tests/test_response.py index 3f55a3d..f6851c8 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -76,10 +76,43 @@ def test_retry_after_header_seconds_to_ms(self) -> None: assert resp.retry_after_ms_header == 3000 def test_retry_after_header_non_numeric_ignored(self) -> None: + for value in ( + "Wed, 21 Oct 2026 07:28:00 GMT", + "-1", + "+1", + "1e2", + "1.5", + "9223372036854775808", + ): + resp = CyclesResponse.http_error( + 429, + "Rate limited", + headers={"retry-after": value}, + ) + assert resp.retry_after_ms_header is None + + def test_retry_after_header_allows_ows(self) -> None: resp = CyclesResponse.http_error( - 429, "Rate limited", headers={"retry-after": "Wed, 21 Oct 2026 07:28:00 GMT"}, + 429, + "Rate limited", + headers={"retry-after": " 0 "}, ) - assert resp.retry_after_ms_header is None + assert resp.retry_after_ms_header == 0 + + def test_retry_after_header_checks_millisecond_overflow(self) -> None: + max_seconds = 9_223_372_036_854_775_807 // 1000 + accepted = CyclesResponse.http_error( + 429, + "Rate limited", + headers={"retry-after": str(max_seconds)}, + ) + overflow = CyclesResponse.http_error( + 429, + "Rate limited", + headers={"retry-after": str(max_seconds + 1)}, + ) + assert accepted.retry_after_ms_header == max_seconds * 1000 + assert overflow.retry_after_ms_header is None def test_missing_headers_return_none(self) -> None: resp = CyclesResponse.success(200, {}) @@ -95,7 +128,5 @@ def test_transport_error_no_headers(self) -> None: assert resp.headers == {} def test_cycles_tenant_header(self) -> None: - resp = CyclesResponse.success( - 200, {}, headers={"x-cycles-tenant": "acme"} - ) + resp = CyclesResponse.success(200, {}, headers={"x-cycles-tenant": "acme"}) assert resp.cycles_tenant == "acme" From 58a2f9614ec4865cbcc02d88e90039f3ec8b0409 Mon Sep 17 00:00:00 2001 From: Albert Mavashev Date: Tue, 28 Jul 2026 11:34:44 -0400 Subject: [PATCH 9/9] test: make heartbeat task completion explicit --- tests/test_heartbeat.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index e2ef667..3bb6158 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -528,7 +528,7 @@ async def fake_sleep(s: float) -> None: monkeypatch.setattr(asyncio, "sleep", fake_sleep) task = lifecycle._start_heartbeat("rsv_1", TTL, ctx or _ctx()) assert task is not None - await task + assert await task is None async def test_extends_only_when_lead_below_threshold( self, @@ -736,7 +736,7 @@ async def fake_sleep(s: float) -> None: monkeypatch.setattr(asyncio, "sleep", fake_sleep) task = stream._start_heartbeat() assert task is not None - await task + assert await task is None assert client.extend_reservation.await_count == 4 assert sleeps[0] == 35.0 @@ -774,7 +774,7 @@ async def fake_sleep(s: float) -> None: monkeypatch.setattr(asyncio, "sleep", fake_sleep) task = stream._start_heartbeat() assert task is not None - await task + assert await task is None assert client.extend_reservation.await_count == 4 @@ -815,7 +815,7 @@ async def fake_sleep(s: float) -> None: monkeypatch.setattr(asyncio, "sleep", fake_sleep) task = stream._start_heartbeat() assert task is not None - await task + assert await task is None assert client.extend_reservation.await_count == 5 @@ -1145,7 +1145,7 @@ async def fake_sleep(s: float) -> None: monkeypatch.setattr(asyncio, "sleep", fake_sleep) task = lifecycle._start_heartbeat("rsv_1", TTL, _ctx(), 60_000) assert task is not None - await task + assert await task is None assert client.extend_reservation.await_count == 4 assert sleeps[0] == 35.0 # from the create field @@ -1295,7 +1295,7 @@ async def fake_sleep(s: float) -> None: monkeypatch.setattr(asyncio, "sleep", fake_sleep) task = lifecycle._start_heartbeat("rsv_1", TTL, _ctx(), initial) assert task is not None - await task + assert await task is None assert client.extend_reservation.await_count == expected @@ -1325,7 +1325,7 @@ async def fake_sleep(s: float) -> None: monkeypatch.setattr(asyncio, "sleep", fake_sleep) task = lifecycle._start_heartbeat("rsv_1", TTL, _ctx(), 60_000) assert task is not None - await task + assert await task is None assert client.extend_reservation.await_count == 2 assert sleeps[1] == 3.0 @@ -1451,7 +1451,7 @@ async def fake_sleep(s: float) -> None: monkeypatch.setattr(asyncio, "sleep", fake_sleep) task = stream._start_heartbeat() assert task is not None - await task + assert await task is None assert client.extend_reservation.await_count == expected @@ -1482,7 +1482,7 @@ async def fake_sleep(s: float) -> None: monkeypatch.setattr(asyncio, "sleep", fake_sleep) task = stream._start_heartbeat() assert task is not None - await task + assert await task is None assert client.extend_reservation.await_count == 2 assert sleeps[1] == 3.0