From 70df9c1aee5d49c6a1b966304b863808e3b61183 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Sat, 29 Aug 2026 11:06:00 +0800 Subject: [PATCH 01/13] perf: profile deterministic dense replay --- README.md | 8 +- benchmarks/benchmark_replay.py | 24 +++- docs/performance-m3a.md | 116 +++++++++++------- src/quant_execution/_json.py | 37 ++++++ src/quant_execution/broker.py | 44 +++++-- src/quant_execution/contracts.py | 15 ++- src/quant_execution/engine.py | 32 +++-- src/quant_execution/ledger.py | 79 ++++++++++-- src/quant_execution/matching.py | 26 ++-- src/quant_execution/rules.py | 22 +++- tests/test_engine.py | 21 ++++ tests/test_performance_equivalence.py | 51 ++++++++ .../m7-back-to-back-optimized-dense-2000.json | 35 ++++++ .../m7-back-to-back-v041-dense-2000.json | 23 ++++ .../performance/m7-baseline-profile.txt | 88 +++++++++++++ .../m7-baseline-v0.4.1-dense-2000.json | 23 ++++ validation/performance/m7-command-results.md | 69 +++++++++++ validation/performance/m7-final-all-2000.json | 68 ++++++++++ .../performance/m7-optimized-dense-2000.json | 35 ++++++ .../performance/m7-optimized-dense-20000.json | 35 ++++++ .../performance/m7-optimized-profile.txt | 88 +++++++++++++ 21 files changed, 848 insertions(+), 91 deletions(-) create mode 100644 src/quant_execution/_json.py create mode 100644 tests/test_performance_equivalence.py create mode 100644 validation/performance/m7-back-to-back-optimized-dense-2000.json create mode 100644 validation/performance/m7-back-to-back-v041-dense-2000.json create mode 100644 validation/performance/m7-baseline-profile.txt create mode 100644 validation/performance/m7-baseline-v0.4.1-dense-2000.json create mode 100644 validation/performance/m7-command-results.md create mode 100644 validation/performance/m7-final-all-2000.json create mode 100644 validation/performance/m7-optimized-dense-2000.json create mode 100644 validation/performance/m7-optimized-dense-20000.json create mode 100644 validation/performance/m7-optimized-profile.txt diff --git a/README.md b/README.md index 4e0ff81..9a7e631 100644 --- a/README.md +++ b/README.md @@ -125,8 +125,8 @@ python tools/check_branch_coverage.py coverage.json --threshold 90 \ python benchmarks/benchmark_replay.py --workload all --repeat 3 --require-rate 50000 ``` -The 50k-events/second replay objective is an explicit local performance gate. The -no-order workload passes; the exact 50%-fill workload remains below the gate while -retaining all 1,000 fills, 2,001 balanced transactions, risk checks and final hashes. -The measured shortfall and required follow-up architecture work are disclosed in +The 50k-events/second replay objective is an explicit local performance gate. The exact +50%-fill workload remains below the gate while retaining all 1,000 fills, 2,001 balanced +transactions, risk checks and byte-identical v0.4.1 final hashes. The reproduced measurements, +same-window control, profile evidence and required follow-up architecture work are disclosed in [`docs/performance-m3a.md`](docs/performance-m3a.md). diff --git a/benchmarks/benchmark_replay.py b/benchmarks/benchmark_replay.py index dd9443d..de93ef7 100644 --- a/benchmarks/benchmark_replay.py +++ b/benchmarks/benchmark_replay.py @@ -30,12 +30,22 @@ START = datetime(2026, 1, 2, tzinfo=UTC) INSTRUMENT = "crypto:benchmark:BTCUSDT" DENSE_ORDER_STRIDE = 2 +DENSE_2000_BASELINE = { + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", +} def fp(value: str | int, scale: int = 3) -> FixedPoint: return FixedPoint.from_decimal(Decimal(str(value)), scale) +DENSE_QUANTITY = fp("0.001") +DENSE_LIMIT_PRICE = fp("100", 2) + + def instrument() -> InstrumentSpec: return InstrumentSpec( instrument_id=INSTRUMENT, @@ -106,11 +116,11 @@ def on_event(self, context, event): strategy_id=context.strategy_id, instrument_id=INSTRUMENT, side=Side.BUY, - quantity=fp("0.001"), + quantity=DENSE_QUANTITY, order_type=OrderType.LIMIT, time_in_force=TimeInForce.GTC, created_at=event.available_at, - limit_price=fp("100", 2), + limit_price=DENSE_LIMIT_PRICE, ), ) @@ -208,6 +218,9 @@ def worker(workload: str, event_count: int) -> int: assert payload["fills"] == expected_orders assert payload["transactions"] == expected_orders * 2 + 1 assert payload["fill_density"] == 0.5 + if event_count == 2_000: + for field, expected in DENSE_2000_BASELINE.items(): + assert payload[field] == expected else: assert payload["orders"] == payload["order_events"] == payload["fills"] == 0 assert payload["transactions"] == 1 @@ -294,6 +307,7 @@ def main() -> int: parser.add_argument("--memory-limit-gib", type=float, default=16) parser.add_argument("--worker", choices=("release_no_orders", "dense_matching_exact_ledger")) parser.add_argument("--events", type=int) + parser.add_argument("--output", type=Path) args = parser.parse_args() if args.worker is not None: if args.events is None or args.events <= 0: @@ -315,7 +329,11 @@ def main() -> int: aggregate(name, count, args.repeat, args.require_rate, memory_limit) for name, count in selected ] - print(json.dumps(results, indent=2, sort_keys=True)) + encoded = json.dumps(results, indent=2, sort_keys=True) + print(encoded) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(encoded + "\n", encoding="utf-8") return 0 if all(item["rate_gate"] and item["memory_gate"] for item in results) else 1 diff --git a/docs/performance-m3a.md b/docs/performance-m3a.md index a95db6d..dd64e1d 100644 --- a/docs/performance-m3a.md +++ b/docs/performance-m3a.md @@ -1,52 +1,86 @@ -# M3a local performance gate +# M7 execution replay performance investigation -Recorded on 2026-08-28 with Python 3.12.5 on an AMD Ryzen 7 7800X3D -(8 cores/16 logical processors) and 31.12 GiB physical memory. +Recorded on 2026-08-29 with Python3.12.5 on an AMD Ryzen7 7800X3D +(8 cores/16 logical processors) and31.12GiB physical memory. -The benchmark constructs all input events before timing, then measures complete -`DeterministicRunEngine.replay`, including strategy callbacks, risk checks, matching, -exact ledger facts and final artifact hashes. Every measured run executes in a fresh -child process, so throughput caches and peak working-set values cannot leak between -workloads or repetitions. Three independent runs are reported and the median is the -gate value. +## Gate and workload integrity -| Workload | Events | Orders/Fills | Ledger transactions | Median throughput | Peak working set | 50k/s gate | -|---|---:|---:|---:|---:|---:|---| -| Deterministic bar replay, no orders | 10,000 | 0/0 | 1 | 163,895.20 events/s | 128.46 MiB | PASS | -| 50%-fill bar matching plus exact ledger | 2,000 | 1,000/1,000 | 2,001 | 10,747.90 events/s | 131.64 MiB | FAIL | +The benchmark constructs market events before timing and measures the complete +`DeterministicRunEngine.replay` call. The dense workload preserves the public chain: -The dense workload has exactly 2,000 market events, 1,000 orders, 1,000 fills, 2,000 -order events and 2,001 balanced transactions. It invokes strategy, risk, matching, -ledger mutation and final hashes without padding the denominator with empty events. -Its deterministic result hash is -`e638c2cb3a44b4fe4bb9a234b4451a905a63d8bdf611ef1b6c97042e8dc3efb9`. -The no-order result hash is -`ccef62b18e9f1c86af29481e29abfed2c09802495293c4fa60e0a49d59532841`. +```text +Strategy.on_event -> RuleBookRiskGate -> BarMatchingModel -> Fill +-> ExactAccountLedger -> canonical order/fill/ledger/result hashes +``` -Reproduce the release gate with: +Every measurement runs in a fresh child process. Dense order cadence remains one order for every +two market events: 2,000 events produce exactly1,000 orders, 1,000 fills, 2,000 order events and +2,001 balanced transactions. The20,000-event capacity run preserves the same50% density. The +denominator contains only input market events. -```bash -python benchmarks/benchmark_replay.py --workload all --repeat 3 --require-rate 50000 -``` +The2,000-event benchmark now fails immediately if any v0.4.1 baseline hash changes: -The benchmark exits nonzero if either median is below 50,000 events/s or any independent -worker reaches 16 GiB. A single workload can be reproduced without changing its facts: +| Fact stream | v0.4.1 SHA-256 | +|---|---| +| Order events | `b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2` | +| Fills | `692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9` | +| Ledger | `b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2` | +| Result | `1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f` | -```bash -python benchmarks/benchmark_replay.py --workload dense --repeat 3 --require-rate 0 -``` +No validation, fee, accounting fact, canonical hash or precision check is disabled. No output is +cached, order density is unchanged, and the event denominator is not padded. + +## Reproduced measurements + +The former document mixed`10,747.90`and`39,252.10 events/s`; neither is retained as current +evidence. Actual reproductions are below. Short Windows scheduling windows varied materially, so +both the initial runs and the same-window back-to-back control are disclosed. + +| Candidate/workload | Events | Median throughput | Peak working set | 50k/s gate | +|---|---:|---:|---:|---| +| v0.4.1 initial baseline | 2,000 | 9,583.10 events/s | 130.23MiB | FAIL | +| v0.4.1 same-window control | 2,000 | 6,807.35 events/s | 129.87MiB | FAIL | +| M7 same-window candidate | 2,000 | 10,672.48 events/s | 129.92MiB | FAIL | +| M7 earlier gate run | 2,000 | 8,341.27 events/s | 130.08MiB | FAIL | +| M7 final all-workload gate | 2,000 | 11,731.93 events/s | 130.24MiB | FAIL | +| M7 capacity run | 20,000 | 9,222.43 events/s | 243.88MiB | FAIL | -The dense median is 39,252.10 events/s below the required threshold, a 78.50% shortfall. -Profiling identifies the cumulative hot path as immutable order/fill/transaction -construction, generic ledger translation/posting, repeated risk and open-order -validation, and final canonical hashing. Low-risk optimizations improved no-order -replay and removed derivative maintenance calculations from cash-only accounts, but -they do not close the dense gap. Closing it now requires a separately reviewed integer -ledger hot path, batch boundary, or compatible native extension; that is a material -architecture expansion rather than a safe M3a defect fix. +The same-window candidate is56.78% faster than its immediately preceding v0.4.1 control. The +20,000-event candidate is16.81% faster than the independently reproduced7,895.48 events/s +v0.4.1 capacity baseline. All runs remain far below50,000 events/s, while memory remains far +below16GiB. -No ledger fact, fill, fee, risk check, accounting invariant or final hash is disabled to -improve the reported number. The 50k/s dense gate therefore remains explicitly failed. +## Profile and changes + +Replay-only deterministic cProfile runs reduced cumulative profiled time from0.555s to0.505s +for2,000 events and reduced primitive calls from1,273,646 to1,128,655. The retained changes are: + +- byte-identical scalar identifier and intent serialization, with Unicode/fallback parity tests; +- direct integer fixed-point balance validation instead of Decimal conversion in transaction + construction; +- stateless single-order Bar matching and full-quantity reuse without changing fill facts; +- cached immutable asset-rule selection and an exact non-derivative runtime-risk shortcut; +- engine-scoped ledger rollback amortization: every event is still validated and posted, while + the existing whole-replay checkpoint remains the fail-closed boundary; +- explicit2,000-event v0.4.1 hash assertions and optional JSON output from the benchmark. + +The remaining cumulative hotspots are transaction translation/posting and immutable fact +construction, broker lifecycle transitions, repeated open-order risk evaluation, matching/fill +construction, and final canonical ledger serialization. Safely reaching50k/s requires a separately +reviewed compiled or batch accounting kernel with byte-identical fact construction; weakening the +gate is not an acceptable substitute. + +## Reproduction and evidence + +```bash +python benchmarks/benchmark_replay.py --workload dense --dense-events 2000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-optimized-dense-2000.json +python benchmarks/benchmark_replay.py --workload dense --dense-events 20000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-optimized-dense-20000.json +``` -This local baseline is not the planned 10-million-event L2 certification dataset and -must not be presented as that certification. +Both commands intentionally exit nonzero because the rate gate fails. Profile tables and all JSON +runs are committed under`validation/performance/`. This remains a local Bar replay benchmark, +not the planned10-million-event L2 certification. diff --git a/src/quant_execution/_json.py b/src/quant_execution/_json.py new file mode 100644 index 0000000..f5e5f0f --- /dev/null +++ b/src/quant_execution/_json.py @@ -0,0 +1,37 @@ +"""Exact small JSON encoders used by deterministic identifier hot paths.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence + + +def flat_sequence_bytes(values: Sequence[object]) -> bytes: + """Encode a flat scalar sequence byte-identically to the historical JSON settings.""" + + tokens: list[str] = [] + for value in values: + if isinstance(value, str): + tokens.append(json.encoder.encode_basestring_ascii(value)) + elif value is None: + tokens.append("null") + elif value is True: + tokens.append("true") + elif value is False: + tokens.append("false") + elif isinstance(value, int): + tokens.append(str(value)) + else: + return json.dumps( + values, + ensure_ascii=True, + separators=(",", ":"), + default=str, + ).encode() + return ("[" + ",".join(tokens) + "]").encode() + + +def string_token(value: str) -> str: + """Return the exact ensure_ascii JSON token for one validated string.""" + + return json.encoder.encode_basestring_ascii(value) diff --git a/src/quant_execution/broker.py b/src/quant_execution/broker.py index d378b64..1ead8ae 100644 --- a/src/quant_execution/broker.py +++ b/src/quant_execution/broker.py @@ -3,13 +3,13 @@ from __future__ import annotations import hashlib -import json from copy import deepcopy from datetime import date, datetime from quant_data_kit import FixedPoint from quant_data_kit.exceptions import ValidationError +from quant_execution._json import flat_sequence_bytes, string_token from quant_execution.contracts import ( Fill, Order, @@ -18,13 +18,39 @@ OrderStatus, TimeInForce, ) -from quant_execution.schemas import execution_payload from quant_execution.state_machine import transition_order def _digest(prefix: str, *parts: object) -> str: - payload = json.dumps(parts, ensure_ascii=True, separators=(",", ":"), default=str) - return f"{prefix}-{hashlib.sha256(payload.encode()).hexdigest()[:24]}" + return f"{prefix}-{hashlib.sha256(flat_sequence_bytes(parts)).hexdigest()[:24]}" + + +def _fixed_token(value: FixedPoint | None) -> str: + if value is None: + return "null" + return f'{{"scale":{value.scale},"units":{value.units}}}' + + +def _intent_bytes(intent: OrderIntent) -> bytes: + """Serialize a validated intent exactly like sorted canonical execution_payload JSON.""" + + created_at = intent.created_at.isoformat().replace("+00:00", "Z") + return ( + "{" + f'"account_id":{string_token(intent.account_id)},' + f'"created_at":{string_token(created_at)},' + f'"idempotency_key":{string_token(intent.idempotency_key)},' + f'"instrument_id":{string_token(intent.instrument_id)},' + f'"limit_price":{_fixed_token(intent.limit_price)},' + f'"order_type":{string_token(intent.order_type.value)},' + f'"quantity":{_fixed_token(intent.quantity)},' + f'"reduce_only":{"true" if intent.reduce_only else "false"},' + f'"side":{string_token(intent.side.value)},' + f'"stop_price":{_fixed_token(intent.stop_price)},' + f'"strategy_id":{string_token(intent.strategy_id)},' + f'"time_in_force":{string_token(intent.time_in_force.value)}' + "}" + ).encode() class DeterministicBroker: @@ -77,6 +103,11 @@ def order_events(self) -> tuple[OrderEvent, ...]: @property def open_orders(self) -> tuple[Order, ...]: + if not self._open_order_ids: + return () + if len(self._open_order_ids) == 1: + order_id = next(iter(self._open_order_ids)) + return (self._orders[order_id],) return tuple( sorted( (self._orders[order_id] for order_id in self._open_order_ids), @@ -94,10 +125,7 @@ def _sort_key(order: Order) -> tuple[datetime, str]: @staticmethod def _intent_hash(intent: OrderIntent) -> str: - payload = json.dumps( - execution_payload(intent), sort_keys=True, separators=(",", ":") - ).encode() - return hashlib.sha256(payload).hexdigest() + return hashlib.sha256(_intent_bytes(intent)).hexdigest() def submit(self, order_intent: OrderIntent) -> Order: if not isinstance(order_intent, OrderIntent): diff --git a/src/quant_execution/contracts.py b/src/quant_execution/contracts.py index 0904a66..16daafa 100644 --- a/src/quant_execution/contracts.py +++ b/src/quant_execution/contracts.py @@ -6,7 +6,6 @@ from collections.abc import Mapping from dataclasses import dataclass, field from datetime import datetime -from decimal import Decimal from enum import Enum from types import MappingProxyType from typing import TypeAlias @@ -437,12 +436,20 @@ def __post_init__(self) -> None: or any(not isinstance(posting, Posting) for posting in self.postings) ): raise ValidationError("ledger transaction requires an immutable tuple of postings") - balances: dict[str, Decimal] = {} + balances: dict[str, tuple[int, int]] = {} for posting in self.postings: + prior_units, prior_scale = balances.get(posting.currency, (0, posting.amount.scale)) + scale = max(prior_scale, posting.amount.scale) balances[posting.currency] = ( - balances.get(posting.currency, Decimal(0)) + posting.amount.to_decimal() + prior_units * 10 ** (scale - prior_scale) + + posting.amount.units * 10 ** (scale - posting.amount.scale), + scale, ) - unbalanced = {currency: total for currency, total in balances.items() if total != 0} + unbalanced = { + currency: {"units": units, "scale": scale} + for currency, (units, scale) in balances.items() + if units != 0 + } if unbalanced: raise ValidationError(f"ledger transaction is unbalanced: {unbalanced}") diff --git a/src/quant_execution/engine.py b/src/quant_execution/engine.py index 4a56c5b..f63b123 100644 --- a/src/quant_execution/engine.py +++ b/src/quant_execution/engine.py @@ -38,6 +38,7 @@ TimeInForce, ) from quant_execution.ledger import ExactAccountLedger +from quant_execution.matching import BarMatchingModel from quant_execution.protocols import MatchingModel, Strategy, StrategyContext from quant_execution.rules import RuleBookRiskGate from quant_execution.schemas import execution_payload @@ -317,7 +318,12 @@ def _match_and_commit( ) return False - matching_checkpoint = self._capture_component(self.matching_model) + matching_checkpoint = ( + None + if type(self.matching_model) is BarMatchingModel + and not self.matching_model.checkpoint_required(open_orders) + else self._capture_component(self.matching_model) + ) matched = tuple(self.matching_model.match(event, open_orders)) if not matched: return False @@ -334,7 +340,8 @@ def _match_and_commit( ) decision = self.risk_gate.check_fill(fill, order) if not decision.accepted: - self._restore_component(self.matching_model, matching_checkpoint) + if matching_checkpoint is not None: + self._restore_component(self.matching_model, matching_checkpoint) if risk_checkpoint is not None: self._restore_component(self.risk_gate, risk_checkpoint) self._expire_fill_rejections( @@ -371,7 +378,8 @@ def _match_and_commit( staged_fees.append(fee) if rejected: - self._restore_component(self.matching_model, matching_checkpoint) + if matching_checkpoint is not None: + self._restore_component(self.matching_model, matching_checkpoint) self._restore_component(self.broker, broker_checkpoint) self._restore_component(self.ledger, ledger_checkpoint) self._restore_component(self.risk_gate, risk_checkpoint) @@ -401,14 +409,20 @@ def _validate_candidate_fill_ids(matched: Sequence[Fill], seen_fill_ids: set[str def _commit_fill(self, fill: Fill, order: Order, event: MarketEvent) -> Fee | None: self.broker.apply_fill(fill) self.risk_gate.release_fill(fill, order) - self.ledger.apply_with_trading_day( - fill, - trading_day=event.trading_day, - create_snapshot=False, - ) + if type(self.ledger) is ExactAccountLedger: + self.ledger._apply_replay_event(fill, trading_day=event.trading_day) + else: + self.ledger.apply_with_trading_day( + fill, + trading_day=event.trading_day, + create_snapshot=False, + ) fee = self.risk_gate.fee_for(fill, order) if fee is not None: - self.ledger.apply(fee, create_snapshot=False) + if type(self.ledger) is ExactAccountLedger: + self.ledger._apply_replay_event(fee) + else: + self.ledger.apply(fee, create_snapshot=False) return fee def _expire_fill_rejections( diff --git a/src/quant_execution/ledger.py b/src/quant_execution/ledger.py index b6992ef..38af6ff 100644 --- a/src/quant_execution/ledger.py +++ b/src/quant_execution/ledger.py @@ -28,6 +28,7 @@ from quant_data_kit.exceptions import ValidationError from quant_execution._fixed import decimal, fixed +from quant_execution._json import flat_sequence_bytes from quant_execution.contracts import ( AccountSnapshot, Fee, @@ -57,7 +58,7 @@ def _canonical(value: object) -> bytes: def _identifier(prefix: str, *parts: object) -> str: - return f"{prefix}-{hashlib.sha256(_canonical(parts)).hexdigest()[:24]}" + return f"{prefix}-{hashlib.sha256(flat_sequence_bytes(parts)).hexdigest()[:24]}" @lru_cache(maxsize=256) @@ -259,6 +260,13 @@ def convert_to_base( def cash_balance(self, currency: str) -> Decimal: return self._accounts.get(("assets:cash", currency, None), Decimal(0)) + @property + def has_open_derivative_position(self) -> bool: + return any( + quantity != 0 and instrument_id in self._derivative_instruments + for instrument_id, quantity in self._positions.items() + ) + def risk_balances( self, event_time: datetime ) -> tuple[dict[str, Decimal], dict[str, Decimal], Decimal, Decimal]: @@ -521,6 +529,7 @@ def _apply( *, trading_day: date | None, create_snapshot: bool, + local_rollback: bool = True, ) -> AccountSnapshot | None: self._validate_event(event) reference_id = self._event_identity(event) @@ -551,8 +560,12 @@ def _apply( undo: dict[str, object] | None = None try: transaction = self._translate(event) - undo = self._capture_apply_undo(event, transaction, reference_id) - self._post(transaction) + if local_rollback: + undo = self._capture_apply_undo(event, transaction, reference_id) + if local_rollback: + self._post(transaction) + else: + self._post(transaction, local_rollback=False) if isinstance(event, Fill): self._fills[event.fill_id] = event self._fill_trading_days[event.fill_id] = trading_day @@ -696,6 +709,26 @@ def apply_with_trading_day( create_snapshot=create_snapshot, ) + def _apply_replay_event( + self, + event: LedgerEvent, + *, + trading_day: date | None = None, + ) -> None: + """Apply one validated fact under the engine's whole-replay rollback boundary.""" + + if isinstance(event, Fill): + if not isinstance(trading_day, date) or isinstance(trading_day, datetime): + raise ValidationError("fill replay application requires a trading_day date") + elif trading_day is not None: + raise ValidationError("trading_day is only valid for fill replay application") + self._apply( + event, + trading_day=trading_day, + create_snapshot=False, + local_rollback=False, + ) + def _validate_event(self, event: LedgerEvent) -> None: if isinstance(event, CorporateActionEvent): return @@ -970,7 +1003,6 @@ def _fill_transaction(self, fill_event: Fill) -> LedgerTransaction: quantity = decimal(fill_event.quantity) signed_quantity = quantity if fill_event.side is Side.BUY else -quantity old_quantity = self._positions.get(fill_event.instrument_id, Decimal(0)) - average = self._average_cost(fill_event.instrument_id) multiplier = decimal(spec.contract_multiplier) price = decimal(fill_event.price) notional = quantity * price * multiplier @@ -979,14 +1011,21 @@ def _fill_transaction(self, fill_event: Fill) -> LedgerTransaction: if old_quantity and old_quantity * signed_quantity < 0 else Decimal(0) ) - if old_quantity > 0: - realized = (price - average) * close_quantity * multiplier - elif old_quantity < 0: - realized = (average - price) * close_quantity * multiplier - else: - realized = Decimal(0) + derivative = fill_event.instrument_id in self._derivative_instruments + average = ( + self._average_cost(fill_event.instrument_id) + if close_quantity or (not derivative and fill_event.side is Side.SELL) + else Decimal(0) + ) + realized = Decimal(0) + if close_quantity: + realized = ( + (price - average) * close_quantity * multiplier + if old_quantity > 0 + else (average - price) * close_quantity * multiplier + ) postings: list[Posting] = [] - if self._is_derivative(spec): + if derivative: if realized: postings.extend( [ @@ -1278,9 +1317,25 @@ def _make_transaction( postings=postings, ) - def _post(self, transaction: LedgerTransaction) -> None: + def _post(self, transaction: LedgerTransaction, *, local_rollback: bool = True) -> None: if transaction.idempotency_key in self._transaction_keys: raise ValidationError("duplicate ledger transaction idempotency key") + if not local_rollback: + for posting in transaction.postings: + key = (posting.ledger_account, posting.currency, posting.instrument_id) + self._accounts[key] = self._accounts.get(key, Decimal(0)) + decimal(posting.amount) + if ( + posting.ledger_account == "assets:position" + and posting.instrument_id is not None + and posting.quantity_delta is not None + ): + instrument_id = posting.instrument_id + self._positions[instrument_id] = self._positions.get( + instrument_id, Decimal(0) + ) + decimal(posting.quantity_delta) + self._transactions.append(transaction) + self._transaction_keys.add(transaction.idempotency_key) + return missing = object() prior_accounts: dict[tuple[str, str, str | None], Decimal | object] = {} prior_positions: dict[str, Decimal | object] = {} diff --git a/src/quant_execution/matching.py b/src/quant_execution/matching.py index ca3dd37..c15c4e5 100644 --- a/src/quant_execution/matching.py +++ b/src/quant_execution/matching.py @@ -3,7 +3,6 @@ from __future__ import annotations import hashlib -import json from collections.abc import Mapping, Sequence from copy import copy, deepcopy from datetime import datetime, timedelta @@ -25,19 +24,19 @@ from quant_data_kit.exceptions import ValidationError from quant_execution._fixed import decimal, fixed +from quant_execution._json import flat_sequence_bytes from quant_execution.broker import remaining_quantity from quant_execution.contracts import Fill, LiquidityRole, Order, OrderType, Side, TimeInForce def _fill_id(model: str, event_id: str, order_id: str, index: int, price: FixedPoint) -> str: - raw = json.dumps( - [model, event_id, order_id, index, price.units, price.scale], - separators=(",", ":"), - ).encode() + raw = flat_sequence_bytes((model, event_id, order_id, index, price.units, price.scale)) return f"fill-{hashlib.sha256(raw).hexdigest()[:24]}" -def _sort_orders(orders: Sequence[Order]) -> list[Order]: +def _sort_orders(orders: Sequence[Order]) -> Sequence[Order]: + if len(orders) < 2: + return orders return sorted(orders, key=lambda item: (item.intent.created_at, item.order_id)) @@ -58,7 +57,14 @@ def priority(order: Order) -> tuple[object, ...]: def _quantity_from_available(order: Order, available: Decimal) -> FixedPoint: - remaining = decimal(remaining_quantity(order)) + remaining_units = order.intent.quantity.units - order.filled_quantity.units + if remaining_units == order.intent.quantity.units: + remaining_fp = order.intent.quantity + else: + remaining_fp = FixedPoint(remaining_units, order.intent.quantity.scale) + remaining = decimal(remaining_fp) + if available >= remaining: + return remaining_fp amount = min(remaining, available) if amount <= 0: return FixedPoint(0, order.intent.quantity.scale) @@ -165,9 +171,15 @@ def __init__( def reset(self) -> None: self._activated_stop_limits.clear() + @staticmethod + def checkpoint_required(open_orders: Sequence[Order]) -> bool: + return any(order.intent.order_type is OrderType.STOP_LIMIT for order in open_orders) + def match(self, market_event: MarketEvent, open_orders: Sequence[Order]) -> Sequence[Fill]: if not isinstance(market_event, BarEvent) or not market_event.is_complete: return () + if not open_orders: + return () capacity = decimal(market_event.volume) * self.participation_rate fills: list[Fill] = [] for order in _sort_orders(open_orders): diff --git a/src/quant_execution/rules.py b/src/quant_execution/rules.py index e7c1a93..ba33c82 100644 --- a/src/quant_execution/rules.py +++ b/src/quant_execution/rules.py @@ -20,6 +20,7 @@ QuoteEvent, StatusEvent, TradeEvent, + ensure_utc_datetime, ) from quant_data_kit.exceptions import ValidationError @@ -310,6 +311,7 @@ def __init__( self.ledger = ledger self.money_scale = money_scale self.policies = tuple(policies) + self._rules: dict[str, _AssetRule] = {} for policy in self.policies: if not callable(getattr(policy, "check_order", None)) or not callable( getattr(policy, "runtime_check", None) @@ -456,7 +458,7 @@ def _check( if value is not None and not aligned(value, spec.price_tick): return RiskDecision(False, "PRICE_TICK", f"{field_name} violates price tick") try: - decision = self._rule(spec).check( + decision = self._rule_for(spec).check( order_intent, account_snapshot, state, spec, self.ledger ) if not decision.accepted: @@ -531,6 +533,13 @@ def runtime_check(self, snapshot: AccountSnapshot) -> RiskDecision: return self._check_runtime_policies(event_time=snapshot.event_time) def runtime_check_current(self, event_time: datetime) -> RiskDecision: + event_time = ensure_utc_datetime(event_time, field="event_time") + if ( + not self.policies + and type(self.ledger) is ExactAccountLedger + and not self.ledger.has_open_derivative_position + ): + return _ACCEPTED_DECISION if self.ledger.liquidation_required(event_time): return RiskDecision( False, @@ -665,7 +674,7 @@ def check_fill( if fill.side is Side.SELL: return _ACCEPTED_DECISION state = self._states[fill.instrument_id] - rate = self._rule(spec).fee_rate(fill, order, state, spec, self.ledger) + rate = self._rule_for(spec).fee_rate(fill, order, state, spec, self.ledger) required = ( decimal(fill.quantity) * decimal(fill.price) @@ -690,7 +699,7 @@ def check_fill( def fee_for(self, fill: Fill, order: Order) -> Fee | None: spec = self.instruments[fill.instrument_id] state = self._states[fill.instrument_id] - rate = self._rule(spec).fee_rate(fill, order, state, spec, self.ledger) + rate = self._rule_for(spec).fee_rate(fill, order, state, spec, self.ledger) fee_type = "maker" if fill.liquidity_role is LiquidityRole.MAKER else "taker" unit_notional = decimal(fill.price) * decimal(spec.contract_multiplier) if spec.asset_class is AssetClass.FUTURE: @@ -864,6 +873,13 @@ def _rule(spec: InstrumentSpec) -> _AssetRule: f"unsupported asset rule for {spec.asset_class.value}/{spec.product_type}" ) + def _rule_for(self, spec: InstrumentSpec) -> _AssetRule: + rule = self._rules.get(spec.instrument_id) + if rule is None: + rule = self._rule(spec) + self._rules[spec.instrument_id] = rule + return rule + def _intent_price(intent: OrderIntent, state: MarketState) -> Decimal | None: if intent.limit_price is not None: diff --git a/tests/test_engine.py b/tests/test_engine.py index e538119..9afcda5 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -313,6 +313,27 @@ def test_engine_fail_closed_on_strategy_error_duplicate_event_and_future_intent( engine.replay([event, event], 1) +def test_exact_ledger_replay_fast_path_restores_the_whole_run_on_post_failure( + monkeypatch, +) -> None: + engine, events = scenario_a_share() + ledger_before = engine.ledger.capture_state() + broker_before = engine.broker.capture_state() + original_post = engine.ledger._post + + def post_then_fail(transaction, *, local_rollback=True): + original_post(transaction, local_rollback=local_rollback) + if not local_rollback and transaction.event_type.value == "fill": + raise RuntimeError("injected replay post failure") + + monkeypatch.setattr(engine.ledger, "_post", post_then_fail) + with pytest.raises(ReplayError, match="injected replay post failure"): + engine.replay(events, 42) + + assert engine.ledger.capture_state() == ledger_before + assert engine.broker.capture_state() == broker_before + + def test_rejected_intent_creates_reasoned_order_event_without_position_mutation() -> None: registry = {STOCK: specs()[STOCK]} strategy = FixtureStrategy({"signal": [Signal(STOCK, Side.BUY, fp("100"), fp("10"))]}) diff --git a/tests/test_performance_equivalence.py b/tests/test_performance_equivalence.py new file mode 100644 index 0000000..1762a50 --- /dev/null +++ b/tests/test_performance_equivalence.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from decimal import Decimal + +from quant_data_kit import FixedPoint + +from quant_execution import OrderIntent, OrderType, Side, TimeInForce +from quant_execution._json import flat_sequence_bytes +from quant_execution.broker import _intent_bytes +from quant_execution.schemas import execution_payload + + +def test_flat_sequence_encoder_matches_historical_json_for_all_supported_paths() -> None: + cases = ( + ("ascii", 7, None, True, False), + ("中文", 'line\nquote"', -3), + ("fallback", Decimal("1.2300")), + ) + for values in cases: + expected = json.dumps( + values, + ensure_ascii=True, + separators=(",", ":"), + default=str, + ).encode() + assert flat_sequence_bytes(values) == expected + + +def test_intent_hot_serializer_is_byte_identical_for_unicode_and_optional_fields() -> None: + intent = OrderIntent( + idempotency_key="订单-一", + account_id="账户", + strategy_id="策略", + instrument_id="crypto:test:BTCUSDT", + side=Side.BUY, + quantity=FixedPoint(1, 3), + order_type=OrderType.STOP_LIMIT, + time_in_force=TimeInForce.GTC, + created_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + limit_price=FixedPoint(10_000, 2), + stop_price=FixedPoint(10_100, 2), + reduce_only=True, + ) + expected = json.dumps( + execution_payload(intent), + sort_keys=True, + separators=(",", ":"), + ).encode() + assert _intent_bytes(intent) == expected diff --git a/validation/performance/m7-back-to-back-optimized-dense-2000.json b/validation/performance/m7-back-to-back-optimized-dense-2000.json new file mode 100644 index 0000000..43e96af --- /dev/null +++ b/validation/performance/m7-back-to-back-optimized-dense-2000.json @@ -0,0 +1,35 @@ +[ + { + "events": 2000, + "events_per_s_median": 10672.48, + "events_per_s_runs": [ + 10672.48, + 12326.79, + 9938.23 + ], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 129.92, + "peak_working_set_mib_runs": [ + 129.84, + 129.92, + 129.75 + ], + "rate_gate": true, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [ + 16340, + 18024, + 31764 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-back-to-back-v041-dense-2000.json b/validation/performance/m7-back-to-back-v041-dense-2000.json new file mode 100644 index 0000000..84d0363 --- /dev/null +++ b/validation/performance/m7-back-to-back-v041-dense-2000.json @@ -0,0 +1,23 @@ +[ + { + "events": 2000, + "events_per_s_median": 6807.35, + "events_per_s_runs": [6807.35, 6711.74, 7154.76], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 129.87, + "peak_working_set_mib_runs": [129.82, 129.69, 129.87], + "rate_gate": true, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [36236, 33512, 37284], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-baseline-profile.txt b/validation/performance/m7-baseline-profile.txt new file mode 100644 index 0000000..66800f0 --- /dev/null +++ b/validation/performance/m7-baseline-profile.txt @@ -0,0 +1,88 @@ +Sat Aug 29 10:40:24 2026 validation/performance/m7-baseline-replay-only.prof + + 1276819 function calls (1273646 primitive calls) in 0.537 seconds + + Ordered by: cumulative time + List reduced from 204 to 80 due to restriction <80> + + ncalls tottime percall cumtime percall filename:lineno(function) + 1 0.011 0.011 0.555 0.555 engine.py:133(replay) + 2000 0.004 0.000 0.265 0.000 engine.py:299(_match_and_commit) + 1000 0.002 0.000 0.179 0.000 engine.py:401(_commit_fill) + 2000 0.007 0.000 0.116 0.000 ledger.py:518(_apply) + 2000 0.002 0.000 0.073 0.000 ledger.py:937(_translate) + 1000 0.001 0.000 0.072 0.000 ledger.py:684(apply_with_trading_day) + 1000 0.003 0.000 0.061 0.000 broker.py:102(submit) + 6001 0.009 0.000 0.058 0.000 schemas.py:536(execution_payload) + 2000 0.006 0.000 0.055 0.000 rules.py:425(_check) + 2001 0.003 0.000 0.049 0.000 ledger.py:1263(_make_transaction) + 1000 0.001 0.000 0.046 0.000 ledger.py:510(apply) + 7004 0.007 0.000 0.045 0.000 __init__.py:183(dumps) + 2000 0.007 0.000 0.045 0.000 state_machine.py:21(transition_order) + 2000 0.006 0.000 0.045 0.000 matching.py:168(match) + 1000 0.007 0.000 0.044 0.000 ledger.py:966(_fill_transaction) + 1000 0.001 0.000 0.038 0.000 rules.py:398(check_open_order_current) + 7004 0.006 0.000 0.037 0.000 encoder.py:183(encode) + 2000 0.003 0.000 0.036 0.000 engine.py:499(_strategy_intents) + 1000 0.003 0.000 0.034 0.000 broker.py:182(apply_fill) + 1000 0.001 0.000 0.034 0.000 rules.py:367(check_current) + 12002 0.016 0.000 0.032 0.000 temporal_v2.py:16(ensure_utc_datetime) + 19004 0.030 0.000 0.030 0.000 schemas.py:501(_fixed) + 2001 0.011 0.000 0.029 0.000 contracts.py:424(__post_init__) + 7004 0.028 0.000 0.028 0.000 encoder.py:205(iterencode) + 2000 0.004 0.000 0.027 0.000 benchmark_replay.py:98(on_event) + 1000 0.001 0.000 0.026 0.000 ledger.py:1098(_cash_income_transaction) + 1000 0.002 0.000 0.023 0.000 matching.py:115(_fill) + 1000 0.006 0.000 0.023 0.000 rules.py:690(fee_for) + 234138 0.020 0.000 0.023 0.000 {built-in method builtins.isinstance} + 1 0.000 0.000 0.022 0.022 ledger.py:203(journal_sha256) + 1005 0.001 0.000 0.022 0.000 engine.py:437(_capture_component) + 1001 0.002 0.000 0.020 0.000 matching.py:92(capture_state) + 3000 0.009 0.000 0.020 0.000 rules.py:810(_reservation_requirement) + 2000 0.007 0.000 0.020 0.000 rules.py:233(check) + 28026 0.012 0.000 0.019 0.000 contracts.py:96(_required_text) + 2000 0.004 0.000 0.019 0.000 rules.py:728(_check_reservations) + 1000 0.001 0.000 0.019 0.000 broker.py:95(_intent_hash) + 2002 0.001 0.000 0.018 0.000 ledger.py:53(_canonical) +4141/1006 0.006 0.000 0.018 0.000 copy.py:118(deepcopy) + 60002 0.011 0.000 0.017 0.000 :2(__hash__) + 3000 0.003 0.000 0.017 0.000 broker.py:25(_digest) +1027/1005 0.001 0.000 0.016 0.000 copy.py:217(_deepcopy_dict) + 3000 0.002 0.000 0.015 0.000 rules.py:533(runtime_check_current) + 2001 0.010 0.000 0.015 0.000 ledger.py:1281(_post) + 85295 0.015 0.000 0.015 0.000 {method 'get' of 'dict' objects} + 2000 0.004 0.000 0.014 0.000 dataclasses.py:1540(replace) + 2000 0.005 0.000 0.013 0.000 contracts.py:230(__post_init__) + 3000 0.003 0.000 0.013 0.000 ledger.py:462(liquidation_required) + 2001 0.002 0.000 0.013 0.000 ledger.py:59(_identifier) + 1000 0.004 0.000 0.012 0.000 contracts.py:286(__post_init__) + 9000 0.005 0.000 0.012 0.000 broker.py:78(open_orders) + 17000 0.008 0.000 0.012 0.000 rules.py:49(_metadata_decimal) + 3000 0.006 0.000 0.011 0.000 contracts.py:184(__post_init__) + 1000 0.003 0.000 0.011 0.000 contracts.py:324(__post_init__) + 6001 0.002 0.000 0.010 0.000 schemas.py:497(_time) + 1000 0.004 0.000 0.010 0.000 contracts.py:142(__post_init__) + 1000 0.004 0.000 0.010 0.000 rules.py:632(check_fill) + 2000 0.002 0.000 0.010 0.000 benchmark_replay.py:35(fp) + 13006 0.007 0.000 0.010 0.000 {built-in method builtins.sorted} + 2000 0.006 0.000 0.009 0.000 ledger.py:581(_capture_apply_undo) + 2000 0.004 0.000 0.009 0.000 rules.py:787(_current_view) + 1000 0.001 0.000 0.008 0.000 rules.py:471(reserve) + 2000 0.004 0.000 0.008 0.000 fixed_point.py:32(from_decimal) + 4000 0.004 0.000 0.007 0.000 _fixed.py:40(aligned) + 1000 0.001 0.000 0.007 0.000 matching.py:32(_fill_id) + 7005 0.004 0.000 0.007 0.000 {built-in method builtins.any} + 63073 0.007 0.000 0.007 0.000 {built-in method builtins.getattr} + 24004 0.007 0.000 0.007 0.000 {method 'utcoffset' of 'datetime.datetime' objects} + 7000 0.003 0.000 0.007 0.000 rules.py:868(_intent_price) + 6003 0.006 0.000 0.006 0.000 {method 'isoformat' of 'datetime.datetime' objects} + 2000 0.004 0.000 0.006 0.000 ledger.py:410(observe_market) + 6009 0.004 0.000 0.006 0.000 fixed_point.py:22(__post_init__) + 39990 0.006 0.000 0.006 0.000 :2(__eq__) + 6002 0.002 0.000 0.006 0.000 ledger.py:1228(_posting) + 1000 0.002 0.000 0.006 0.000 schemas.py:505(_intent_payload) + 60002 0.006 0.000 0.006 0.000 {built-in method builtins.hash} + 56052 0.006 0.000 0.006 0.000 {method 'strip' of 'str' objects} +1011/1004 0.001 0.000 0.005 0.000 copy.py:247(_reconstruct) + 2 0.000 0.000 0.005 0.003 engine.py:86(_hash) + 2000 0.003 0.000 0.005 0.000 rules.py:345(observe) diff --git a/validation/performance/m7-baseline-v0.4.1-dense-2000.json b/validation/performance/m7-baseline-v0.4.1-dense-2000.json new file mode 100644 index 0000000..8b753d8 --- /dev/null +++ b/validation/performance/m7-baseline-v0.4.1-dense-2000.json @@ -0,0 +1,23 @@ +[ + { + "events": 2000, + "events_per_s_median": 9583.1, + "events_per_s_runs": [9726.63, 9583.1, 9149.99], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 130.23, + "peak_working_set_mib_runs": [129.31, 130.12, 130.23], + "rate_gate": true, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [6472, 29772, 33096], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-command-results.md b/validation/performance/m7-command-results.md new file mode 100644 index 0000000..b815e72 --- /dev/null +++ b/validation/performance/m7-command-results.md @@ -0,0 +1,69 @@ +# M7 performance command results + +Scope:`quant-execution`only. Baseline commit/tag: +`29eccc0e392968b5f7c31976a329605aacce369a`/annotated`v0.4.1`. + +## Profile + +```text +python -m cProfile ... DeterministicRunEngine.replay(events(2000), seed=42) +``` + +- Baseline:1,273,646 primitive calls,0.555s cumulative replay time. +- Candidate:1,128,655 primitive calls,0.505s cumulative replay time. +- Full tables:`m7-baseline-profile.txt`,`m7-optimized-profile.txt`. + +## Tests and coverage + +```text +python -m ruff check src tests benchmarks tools +python -m ruff format --check src tests benchmarks tools +python -m pytest --cov=quant_execution --cov-branch \ + --cov-report=term-missing --cov-report=json:coverage.json -q +python -m coverage report --fail-under=80 +python tools/check_branch_coverage.py coverage.json --threshold 90 \ + broker contracts schemas engine matching state_machine ledger rules +``` + +- Ruff check/format:PASS. +- Pytest:179 passed. +- Total coverage:95.08%. +- Pure branch coverage:broker98.08%,contracts91.77%,schemas92.11%,engine90.32%, + matching94.31%,state_machine100.00%,ledger90.00%,rules91.26%. + +## Performance + +```text +python benchmarks/benchmark_replay.py --workload all --release-events 10000 \ + --dense-events 2000 --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-final-all-2000.json +``` + +- No-order median:202,688.05 events/s;peak:127.15MiB;rate/memory gates:PASS/PASS. +- Dense median:11,731.93 events/s;peak:130.24MiB;rate/memory gates:FAIL/PASS. +- Dense facts:2,000 events,1,000 orders/fills,2,000 order events,2,001 transactions. + +```text +python benchmarks/benchmark_replay.py --workload dense --dense-events 2000 \ + --repeat 3 --require-rate 50000 --output validation/performance/m7-optimized-dense-2000.json +``` + +- Earlier median:8,341.27 events/s;peak:130.08MiB. +- Facts:2,000 events,1,000 orders/fills,2,000 order events,2,001 transactions. +- Memory gate:PASS.Rate gate:FAIL. + +```text +python benchmarks/benchmark_replay.py --workload dense --dense-events 20000 \ + --repeat 3 --require-rate 50000 --output validation/performance/m7-optimized-dense-20000.json +``` + +- Median:9,222.43 events/s;peak:243.88MiB. +- Facts:20,000 events,10,000 orders/fills,20,000 order events,20,001 transactions. +- Memory gate:PASS.Rate gate:FAIL. + +The same-window v0.4.1/candidate controls measured6,807.35/10,672.48 events/s. All four +2,000-event hashes are byte-identical. The candidate is therefore semantically equivalent and +measurably faster in the controlled comparison, but it does not satisfy the release rate gate. + +Python3.10/3.11/3.12 CI, lock verification, commit, PR and final worktree state are appended after +the remote checks complete. diff --git a/validation/performance/m7-final-all-2000.json b/validation/performance/m7-final-all-2000.json new file mode 100644 index 0000000..d8218e9 --- /dev/null +++ b/validation/performance/m7-final-all-2000.json @@ -0,0 +1,68 @@ +[ + { + "events": 10000, + "events_per_s_median": 202688.05, + "events_per_s_runs": [ + 166480.21, + 202688.05, + 235622.33 + ], + "fill_density": 0.0, + "fill_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "fills": 0, + "independent_processes": 3, + "ledger_sha256": "9027723b240e6f01192567436867a65a2d5b11fd97eae62c46a8329af4ec36bd", + "memory_gate": true, + "order_events": 0, + "order_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "orders": 0, + "peak_working_set_mib": 127.15, + "peak_working_set_mib_runs": [ + 127.15, + 126.81, + 127.0 + ], + "rate_gate": true, + "result_sha256": "26b178c93124241d1dfe25d5333829d5fa04a34e05822108f072e62af61b8357", + "transactions": 1, + "worker_pids": [ + 32400, + 6348, + 26268 + ], + "workload": "release_no_orders" + }, + { + "events": 2000, + "events_per_s_median": 11731.93, + "events_per_s_runs": [ + 12684.68, + 11731.93, + 10893.18 + ], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 130.24, + "peak_working_set_mib_runs": [ + 130.24, + 129.92, + 130.11 + ], + "rate_gate": false, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [ + 4084, + 28252, + 13824 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-optimized-dense-2000.json b/validation/performance/m7-optimized-dense-2000.json new file mode 100644 index 0000000..6ec9745 --- /dev/null +++ b/validation/performance/m7-optimized-dense-2000.json @@ -0,0 +1,35 @@ +[ + { + "events": 2000, + "events_per_s_median": 8341.27, + "events_per_s_runs": [ + 8104.47, + 8341.27, + 9104.54 + ], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 130.08, + "peak_working_set_mib_runs": [ + 129.91, + 130.08, + 130.05 + ], + "rate_gate": false, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [ + 36040, + 26280, + 36196 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-optimized-dense-20000.json b/validation/performance/m7-optimized-dense-20000.json new file mode 100644 index 0000000..776bafb --- /dev/null +++ b/validation/performance/m7-optimized-dense-20000.json @@ -0,0 +1,35 @@ +[ + { + "events": 20000, + "events_per_s_median": 9222.43, + "events_per_s_runs": [ + 10128.9, + 9222.43, + 8141.41 + ], + "fill_density": 0.5, + "fill_sha256": "111cb2eded28d1e36900d91dae7139891e3af086988f0063eafcbb06c20e2623", + "fills": 10000, + "independent_processes": 3, + "ledger_sha256": "471d55977ce169dbb7fee69e72fc18183d95fae0e7af69827dc45542199fd7c1", + "memory_gate": true, + "order_events": 20000, + "order_sha256": "95018ec93663aa987c1a4b6ce89f27197ab274eb4942b8755cb2861085f5ebbc", + "orders": 10000, + "peak_working_set_mib": 243.88, + "peak_working_set_mib_runs": [ + 243.69, + 243.28, + 243.88 + ], + "rate_gate": false, + "result_sha256": "8e2498f3ac928ab3e723198d6d06a7d15e219234733d9cce59342a23c5008b7f", + "transactions": 20001, + "worker_pids": [ + 8660, + 6500, + 5520 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-optimized-profile.txt b/validation/performance/m7-optimized-profile.txt new file mode 100644 index 0000000..c3b52d1 --- /dev/null +++ b/validation/performance/m7-optimized-profile.txt @@ -0,0 +1,88 @@ +Sat Aug 29 11:00:22 2026 validation/performance/m7-optimized-replay-only.prof + + 1128828 function calls (1128655 primitive calls) in 0.487 seconds + + Ordered by: cumulative time + List reduced from 201 to 80 due to restriction <80> + + ncalls tottime percall cumtime percall filename:lineno(function) + 1 0.011 0.011 0.505 0.505 engine.py:134(replay) + 2000 0.005 0.000 0.224 0.000 engine.py:300(_match_and_commit) + 1000 0.002 0.000 0.164 0.000 engine.py:409(_commit_fill) + 2000 0.001 0.000 0.100 0.000 ledger.py:712(_apply_replay_event) + 2000 0.006 0.000 0.098 0.000 ledger.py:526(_apply) + 2000 0.002 0.000 0.066 0.000 ledger.py:970(_translate) + 1000 0.004 0.000 0.058 0.000 broker.py:130(submit) + 2000 0.006 0.000 0.056 0.000 rules.py:429(_check) + 5001 0.009 0.000 0.054 0.000 schemas.py:536(execution_payload) + 1 0.000 0.000 0.053 0.053 ledger.py:204(journal_sha256) + 2000 0.007 0.000 0.048 0.000 state_machine.py:21(transition_order) + 2001 0.003 0.000 0.046 0.000 ledger.py:1302(_make_transaction) + 2000 0.005 0.000 0.042 0.000 matching.py:178(match) + 1000 0.007 0.000 0.039 0.000 ledger.py:999(_fill_transaction) + 1000 0.001 0.000 0.038 0.000 rules.py:402(check_open_order_current) + 1000 0.003 0.000 0.036 0.000 broker.py:210(apply_fill) + 1000 0.001 0.000 0.034 0.000 rules.py:371(check_current) + 12002 0.018 0.000 0.034 0.000 temporal_v2.py:16(ensure_utc_datetime) + 6002 0.003 0.000 0.033 0.000 schemas.py:522(_posting_payload) + 16004 0.031 0.000 0.031 0.000 schemas.py:501(_fixed) + 2001 0.011 0.000 0.029 0.000 contracts.py:423(__post_init__) + 2000 0.004 0.000 0.028 0.000 engine.py:513(_strategy_intents) + 1000 0.002 0.000 0.026 0.000 matching.py:121(_fill) + 1000 0.001 0.000 0.025 0.000 ledger.py:1137(_cash_income_transaction) + 6001 0.015 0.000 0.024 0.000 _json.py:9(flat_sequence_bytes) + 1000 0.006 0.000 0.023 0.000 rules.py:697(fee_for) + 224134 0.019 0.000 0.022 0.000 {built-in method builtins.isinstance} + 3000 0.009 0.000 0.021 0.000 rules.py:817(_reservation_requirement) + 2000 0.004 0.000 0.021 0.000 rules.py:735(_check_reservations) + 2000 0.008 0.000 0.021 0.000 rules.py:234(check) + 28026 0.012 0.000 0.019 0.000 contracts.py:95(_required_text) + 2000 0.003 0.000 0.017 0.000 benchmark_replay.py:108(on_event) + 58003 0.012 0.000 0.017 0.000 :2(__hash__) + 3000 0.003 0.000 0.017 0.000 broker.py:24(_digest) + 3000 0.002 0.000 0.015 0.000 rules.py:537(runtime_check_current) + 2000 0.005 0.000 0.015 0.000 dataclasses.py:1540(replace) + 2000 0.005 0.000 0.015 0.000 contracts.py:229(__post_init__) + 1000 0.001 0.000 0.014 0.000 broker.py:126(_intent_hash) + 3 0.000 0.000 0.014 0.005 __init__.py:183(dumps) + 3 0.000 0.000 0.014 0.005 encoder.py:183(encode) + 3 0.014 0.005 0.014 0.005 encoder.py:205(iterencode) + 2001 0.009 0.000 0.013 0.000 ledger.py:1320(_post) + 1000 0.004 0.000 0.013 0.000 contracts.py:285(__post_init__) + 17000 0.009 0.000 0.012 0.000 rules.py:50(_metadata_decimal) + 3000 0.007 0.000 0.012 0.000 contracts.py:183(__post_init__) + 53296 0.012 0.000 0.012 0.000 {method 'get' of 'dict' objects} + 1000 0.004 0.000 0.012 0.000 broker.py:34(_intent_bytes) + 1000 0.003 0.000 0.011 0.000 contracts.py:323(__post_init__) + 1000 0.004 0.000 0.011 0.000 contracts.py:141(__post_init__) + 2001 0.002 0.000 0.011 0.000 ledger.py:60(_identifier) + 8005 0.004 0.000 0.010 0.000 {built-in method builtins.any} + 2000 0.005 0.000 0.009 0.000 rules.py:794(_current_view) + 1 0.000 0.000 0.009 0.009 ledger.py:54(_canonical) + 1000 0.004 0.000 0.009 0.000 rules.py:639(check_fill) + 1000 0.001 0.000 0.009 0.000 matching.py:32(_fill_id) + 1000 0.001 0.000 0.009 0.000 rules.py:475(reserve) + 5001 0.002 0.000 0.008 0.000 schemas.py:497(_time) + 4000 0.004 0.000 0.007 0.000 _fixed.py:40(aligned) + 6003 0.007 0.000 0.007 0.000 {method 'isoformat' of 'datetime.datetime' objects} + 7000 0.004 0.000 0.007 0.000 rules.py:875(_intent_price) + 2000 0.004 0.000 0.007 0.000 ledger.py:418(observe_market) + 58073 0.007 0.000 0.007 0.000 {built-in method builtins.getattr} + 6002 0.002 0.000 0.007 0.000 ledger.py:1267(_posting) + 24004 0.006 0.000 0.006 0.000 {method 'utcoffset' of 'datetime.datetime' objects} + 58003 0.006 0.000 0.006 0.000 {built-in method builtins.hash} + 2000 0.003 0.000 0.006 0.000 rules.py:349(observe) + 56052 0.006 0.000 0.006 0.000 {method 'strip' of 'str' objects} + 3000 0.002 0.000 0.005 0.000 ledger.py:263(has_open_derivative_position) + 1000 0.003 0.000 0.005 0.000 matching.py:205(_execution_price) + 2000 0.003 0.000 0.005 0.000 ledger.py:732(_validate_event) + 8004 0.005 0.000 0.005 0.000 {built-in method _hashlib.openssl_sha256} + 2 0.000 0.000 0.005 0.003 engine.py:87(_hash) + 15002 0.003 0.000 0.005 0.000 enum.py:202(__get__) + 2 0.000 0.000 0.005 0.002 engine.py:76(_canonical) + 8004 0.004 0.000 0.004 0.000 {method 'hexdigest' of '_hashlib.HASH' objects} + 23998 0.004 0.000 0.004 0.000 :2(__eq__) + 9000 0.003 0.000 0.004 0.000 broker.py:104(open_orders) + 2000 0.002 0.000 0.004 0.000 rules.py:96(check) + 28068 0.004 0.000 0.004 0.000 {method 'append' of 'list' objects} + 3008 0.002 0.000 0.004 0.000 fixed_point.py:22(__post_init__) From c14fdfe3e665d6e196dc9eed63b059b181d14d00 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Sat, 29 Aug 2026 11:08:32 +0800 Subject: [PATCH 02/13] docs: record M7 performance CI evidence --- validation/performance/m7-command-results.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/validation/performance/m7-command-results.md b/validation/performance/m7-command-results.md index b815e72..5c6ea80 100644 --- a/validation/performance/m7-command-results.md +++ b/validation/performance/m7-command-results.md @@ -65,5 +65,15 @@ The same-window v0.4.1/candidate controls measured6,807.35/10,672.48 events/s. A 2,000-event hashes are byte-identical. The candidate is therefore semantically equivalent and measurably faster in the controlled comparison, but it does not satisfy the release rate gate. -Python3.10/3.11/3.12 CI, lock verification, commit, PR and final worktree state are appended after -the remote checks complete. +## Remote handoff + +- Implementation commit:`70df9c1aee5d49c6a1b966304b863808e3b61183`. +- PR:[#6](https://github.com/PureSaber/quant-execution/pull/6). +- Push CI:[run33230566607](https://github.com/PureSaber/quant-execution/actions/runs/33230566607), + Python3.10/3.11/3.12 all PASS. +- PR CI:[run33230576404](https://github.com/PureSaber/quant-execution/actions/runs/33230576404), + Python3.10/3.11/3.12 all PASS. +- CI installs`requirements.lock`, runs`pip check`, then installs the editable project with + `--no-deps --no-build-isolation` and runs`pip check`again. +- Package version remains0.4.1, the lock and dependency declaration are unchanged, and no new tag + was created because the50k/s rate gate failed. From 2888e79c9af788f8fc89d7a270c189935d6f9017 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Sat, 29 Aug 2026 18:34:42 +0800 Subject: [PATCH 03/13] perf: reduce exact replay artifact overhead --- README.md | 8 +- docs/performance-m3a.md | 62 +++++--- src/quant_execution/_json.py | 20 +++ src/quant_execution/broker.py | 132 +++++++++++++++--- src/quant_execution/contracts.py | 31 ++-- src/quant_execution/engine.py | 80 +++++++++-- src/quant_execution/ledger.py | 90 ++++++++---- src/quant_execution/matching.py | 32 +++-- src/quant_execution/rules.py | 104 +++++++++++--- src/quant_execution/state_machine.py | 34 ++--- tests/test_broker.py | 5 + tests/test_performance_equivalence.py | 115 ++++++++++++++- validation/performance/m7-command-results.md | 78 ++++++----- .../m7-techlead-baseline-dense-20000.json | 35 +++++ .../m7-techlead-final-all-2000.json | 68 +++++++++ .../m7-techlead-final-dense-100000.json | 35 +++++ .../m7-techlead-final-dense-20000.json | 35 +++++ .../m7-techlead-final-dense-500000.json | 35 +++++ 18 files changed, 828 insertions(+), 171 deletions(-) create mode 100644 validation/performance/m7-techlead-baseline-dense-20000.json create mode 100644 validation/performance/m7-techlead-final-all-2000.json create mode 100644 validation/performance/m7-techlead-final-dense-100000.json create mode 100644 validation/performance/m7-techlead-final-dense-20000.json create mode 100644 validation/performance/m7-techlead-final-dense-500000.json diff --git a/README.md b/README.md index 9a7e631..8dff38b 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,9 @@ python benchmarks/benchmark_replay.py --workload all --repeat 3 --require-rate 5 ``` The 50k-events/second replay objective is an explicit local performance gate. The exact -50%-fill workload remains below the gate while retaining all 1,000 fills, 2,001 balanced -transactions, risk checks and byte-identical v0.4.1 final hashes. The reproduced measurements, -same-window control, profile evidence and required follow-up architecture work are disclosed in +50%-fill workload remains below the gate while retaining every fill, fee, balanced transaction, +risk check and byte-identical v0.4.1 golden hash. The largest retained run has500,000 events, +250,000 fills and500,001 transactions; its median is14,638.05 events/s and its peak working set is +1,319.62MiB. The reproduced measurements, profile evidence, measured memory slope and required +follow-up architecture work are disclosed in [`docs/performance-m3a.md`](docs/performance-m3a.md). diff --git a/docs/performance-m3a.md b/docs/performance-m3a.md index dd64e1d..74dd0b7 100644 --- a/docs/performance-m3a.md +++ b/docs/performance-m3a.md @@ -15,8 +15,8 @@ Strategy.on_event -> RuleBookRiskGate -> BarMatchingModel -> Fill Every measurement runs in a fresh child process. Dense order cadence remains one order for every two market events: 2,000 events produce exactly1,000 orders, 1,000 fills, 2,000 order events and -2,001 balanced transactions. The20,000-event capacity run preserves the same50% density. The -denominator contains only input market events. +2,001 balanced transactions. The20,000, 100,000 and500,000-event capacity runs preserve the same +50% density. The denominator contains only input market events. The2,000-event benchmark now fails immediately if any v0.4.1 baseline hash changes: @@ -44,16 +44,25 @@ both the initial runs and the same-window back-to-back control are disclosed. | M7 earlier gate run | 2,000 | 8,341.27 events/s | 130.08MiB | FAIL | | M7 final all-workload gate | 2,000 | 11,731.93 events/s | 130.24MiB | FAIL | | M7 capacity run | 20,000 | 9,222.43 events/s | 243.88MiB | FAIL | +| Technical-lead final gate | 2,000 | 15,912.92 events/s | 120.88MiB | FAIL | +| Technical-lead capacity | 20,000 | 15,869.53 events/s | 165.54MiB | FAIL | +| Technical-lead capacity | 100,000 | 15,536.93 events/s | 360.31MiB | FAIL | +| Technical-lead capacity | 500,000 | 14,638.05 events/s | 1,319.62MiB | FAIL | -The same-window candidate is56.78% faster than its immediately preceding v0.4.1 control. The -20,000-event candidate is16.81% faster than the independently reproduced7,895.48 events/s -v0.4.1 capacity baseline. All runs remain far below50,000 events/s, while memory remains far -below16GiB. +The technical-lead candidate is35.64% faster than the previous11,731.93 events/s2,000-event +candidate and36.36% faster than the directly reproduced11,637.77 events/s20,000-event starting +point. All retained hashes are unchanged, but all dense runs remain far below50,000 events/s. + +The100,000-to500,000 event increment consumes959.31MiB for400,000 more input events, an observed +2.46KiB/event slope. Extrapolating that measured retained-object slope gives approximately23.5GiB +at10million dense events before safety margin. A10million run was therefore not started: it could +not pass the16GiB gate and posed an avoidable host-memory exhaustion risk. This is a measured +capacity FAIL, not a10million certification. ## Profile and changes -Replay-only deterministic cProfile runs reduced cumulative profiled time from0.555s to0.505s -for2,000 events and reduced primitive calls from1,273,646 to1,128,655. The retained changes are: +Replay-only deterministic cProfile runs reduced cumulative profiled time from0.555s to0.344s +for2,000 events and reduced primitive calls from1,273,646 to1,011,749. The retained changes are: - byte-identical scalar identifier and intent serialization, with Unicode/fallback parity tests; - direct integer fixed-point balance validation instead of Decimal conversion in transaction @@ -62,13 +71,26 @@ for2,000 events and reduced primitive calls from1,273,646 to1,128,655. The retai - cached immutable asset-rule selection and an exact non-derivative runtime-risk shortcut; - engine-scoped ledger rollback amortization: every event is still validated and posted, while the existing whole-replay checkpoint remains the fail-closed boundary; +- byte-identical streaming canonical hashes that avoid a duplicate nested dict graph; +- prepared reservation and immutable fee-rate reuse inside the built-in risk gate; +- validated internal immutable order/fill/fee construction without removing lifecycle checks; +- sparse DAY/IOC/FOK indexes, so GTC history does not populate irrelevant expiry state; - explicit2,000-event v0.4.1 hash assertions and optional JSON output from the benchmark. -The remaining cumulative hotspots are transaction translation/posting and immutable fact -construction, broker lifecycle transitions, repeated open-order risk evaluation, matching/fill -construction, and final canonical ledger serialization. Safely reaching50k/s requires a separately -reviewed compiled or batch accounting kernel with byte-identical fact construction; weakening the -gate is not an acceptable substitute. +The remaining cumulative hotspots are transaction translation/posting, balanced transaction +construction, repeated open-order risk evaluation, matching and retained Python artifact graphs. +Safely reaching both50k/s and10million/<16GiB requires a separately reviewed architecture: + +1. a bounded-memory`ReplayArtifactSink`that writes typed Arrow/Parquet record batches while + preserving order, fill and ledger schema bytes and incremental hashes; +2. a compiled fixed-point matching/accounting kernel with checked integer scales and byte-identical + transaction IDs, plus Python3.10/3.11/3.12 wheels and a pure-Python equivalence oracle; +3. an explicitly versioned compatibility path for current tuple-based`RunArtifacts`; and +4. golden differential tests over accepted, rejected, partial-fill, fee, funding, settlement and + rollback paths before the compiled kernel can become the default. + +Weakening the gate, dropping facts or moving deferred work outside the timed replay is not an +acceptable substitute. ## Reproduction and evidence @@ -78,9 +100,15 @@ python benchmarks/benchmark_replay.py --workload dense --dense-events 2000 \ --output validation/performance/m7-optimized-dense-2000.json python benchmarks/benchmark_replay.py --workload dense --dense-events 20000 \ --repeat 3 --require-rate 50000 \ - --output validation/performance/m7-optimized-dense-20000.json + --output validation/performance/m7-techlead-final-dense-20000.json +python benchmarks/benchmark_replay.py --workload dense --dense-events 100000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-techlead-final-dense-100000.json +python benchmarks/benchmark_replay.py --workload dense --dense-events 500000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-techlead-final-dense-500000.json ``` -Both commands intentionally exit nonzero because the rate gate fails. Profile tables and all JSON -runs are committed under`validation/performance/`. This remains a local Bar replay benchmark, -not the planned10-million-event L2 certification. +All dense commands intentionally exit nonzero because the rate gate fails. Profile tables and +retained JSON runs are committed under`validation/performance/`. This remains a local Bar replay +benchmark, not the planned10-million-event L2 certification. diff --git a/src/quant_execution/_json.py b/src/quant_execution/_json.py index f5e5f0f..deee088 100644 --- a/src/quant_execution/_json.py +++ b/src/quant_execution/_json.py @@ -4,6 +4,9 @@ import json from collections.abc import Sequence +from datetime import datetime + +from quant_data_kit import FixedPoint def flat_sequence_bytes(values: Sequence[object]) -> bytes: @@ -35,3 +38,20 @@ def string_token(value: str) -> str: """Return the exact ensure_ascii JSON token for one validated string.""" return json.encoder.encode_basestring_ascii(value) + + +def fixed_token(value: FixedPoint | None) -> str: + """Encode a fixed-point value like sorted canonical execution JSON.""" + + if value is None: + return "null" + return f'{{"scale":{value.scale},"units":{value.units}}}' + + +def utc_token(value: datetime, *, zulu: bool = True) -> str: + """Encode one already validated UTC timestamp as a JSON string token.""" + + rendered = value.isoformat() + if zulu: + rendered = rendered.replace("+00:00", "Z") + return string_token(rendered) diff --git a/src/quant_execution/broker.py b/src/quant_execution/broker.py index 1ead8ae..59e126f 100644 --- a/src/quant_execution/broker.py +++ b/src/quant_execution/broker.py @@ -9,7 +9,7 @@ from quant_data_kit import FixedPoint from quant_data_kit.exceptions import ValidationError -from quant_execution._json import flat_sequence_bytes, string_token +from quant_execution._json import fixed_token, flat_sequence_bytes, string_token from quant_execution.contracts import ( Fill, Order, @@ -25,12 +25,6 @@ def _digest(prefix: str, *parts: object) -> str: return f"{prefix}-{hashlib.sha256(flat_sequence_bytes(parts)).hexdigest()[:24]}" -def _fixed_token(value: FixedPoint | None) -> str: - if value is None: - return "null" - return f'{{"scale":{value.scale},"units":{value.units}}}' - - def _intent_bytes(intent: OrderIntent) -> bytes: """Serialize a validated intent exactly like sorted canonical execution_payload JSON.""" @@ -41,12 +35,12 @@ def _intent_bytes(intent: OrderIntent) -> bytes: f'"created_at":{string_token(created_at)},' f'"idempotency_key":{string_token(intent.idempotency_key)},' f'"instrument_id":{string_token(intent.instrument_id)},' - f'"limit_price":{_fixed_token(intent.limit_price)},' + f'"limit_price":{fixed_token(intent.limit_price)},' f'"order_type":{string_token(intent.order_type.value)},' - f'"quantity":{_fixed_token(intent.quantity)},' + f'"quantity":{fixed_token(intent.quantity)},' f'"reduce_only":{"true" if intent.reduce_only else "false"},' f'"side":{string_token(intent.side.value)},' - f'"stop_price":{_fixed_token(intent.stop_price)},' + f'"stop_price":{fixed_token(intent.stop_price)},' f'"strategy_id":{string_token(intent.strategy_id)},' f'"time_in_force":{string_token(intent.time_in_force.value)}' "}" @@ -64,6 +58,8 @@ def __init__(self) -> None: def reset(self) -> None: self._orders: dict[str, Order] = {} self._open_order_ids: set[str] = set() + self._day_order_ids: set[str] = set() + self._immediate_order_ids: set[str] = set() self._submit_keys: dict[str, tuple[str, str]] = {} self._cancel_keys: dict[str, tuple[str, OrderEvent]] = {} self._fill_keys: dict[str, tuple[Fill, OrderEvent]] = {} @@ -75,6 +71,8 @@ def capture_state(self) -> dict[str, object]: { "orders": self._orders, "open_order_ids": self._open_order_ids, + "day_order_ids": self._day_order_ids, + "immediate_order_ids": self._immediate_order_ids, "submit_keys": self._submit_keys, "cancel_keys": self._cancel_keys, "fill_keys": self._fill_keys, @@ -87,6 +85,8 @@ def restore_state(self, state: dict[str, object]) -> None: restored = deepcopy(state) self._orders = restored["orders"] self._open_order_ids = restored["open_order_ids"] + self._day_order_ids = restored["day_order_ids"] + self._immediate_order_ids = restored["immediate_order_ids"] self._submit_keys = restored["submit_keys"] self._cancel_keys = restored["cancel_keys"] self._fill_keys = restored["fill_keys"] @@ -115,6 +115,17 @@ def open_orders(self) -> tuple[Order, ...]: ) ) + @property + def immediate_orders(self) -> tuple[Order, ...]: + if not self._immediate_order_ids: + return () + return tuple( + sorted( + (self._orders[order_id] for order_id in self._immediate_order_ids), + key=self._sort_key, + ) + ) + def get_order(self, order_id: str) -> Order: """Return one order without sorting the complete historical order set.""" return self._require_order(order_id) @@ -138,15 +149,24 @@ def submit(self, order_intent: OrderIntent) -> Order: raise ValidationError("submit idempotency key reused with different intent") return self._orders[order_id] order_id = _digest("ord", order_intent.idempotency_key, semantic_hash) - order = Order(order_id=order_id, intent=order_intent) - accepted, event = transition_order( - order, - OrderStatus.ACCEPTED, + filled = FixedPoint(0, order_intent.quantity.scale) + accepted = self._order_fact( + order_id, + order_intent, + status=OrderStatus.ACCEPTED, + filled_quantity=filled, + version=1, + ) + event = self._event_fact( event_id=_digest("oev", order_id, 1, "accepted"), - event_time=order_intent.created_at, + order=accepted, + from_status=OrderStatus.CREATED, + fill_quantity=None, ) self._orders[order_id] = accepted self._open_order_ids.add(order_id) + if order_intent.time_in_force in {TimeInForce.IOC, TimeInForce.FOK}: + self._immediate_order_ids.add(order_id) self._submit_keys[order_intent.idempotency_key] = (order_id, semantic_hash) self._events.append(event) return accepted @@ -203,6 +223,8 @@ def cancel( ) self._orders[order_id] = updated self._open_order_ids.remove(order_id) + self._day_order_ids.discard(order_id) + self._immediate_order_ids.discard(order_id) self._cancel_keys[idempotency_key] = (order_id, event) self._events.append(event) return event @@ -227,22 +249,40 @@ def apply_fill(self, fill: Fill) -> OrderEvent: raise ValidationError("fill side differs from order intent") if fill.quantity.scale != order.intent.quantity.scale: raise ValidationError("fill quantity scale differs from order intent") + if not fill.quantity.is_positive(): + raise ValidationError("fill quantity must be positive") remaining = order.intent.quantity.units - order.filled_quantity.units if fill.quantity.units > remaining: raise ValidationError("fill would violate order quantity conservation") target = ( OrderStatus.FILLED if fill.quantity.units == remaining else OrderStatus.PARTIALLY_FILLED ) - updated, event = transition_order( - order, - target, - event_id=_digest("oev", order.order_id, order.version + 1, target.value, fill.fill_id), - event_time=fill.event_time, + if fill.event_time < order.intent.created_at: + raise ValidationError("order event_time cannot precede intent created_at") + next_version = order.version + 1 + filled_quantity = FixedPoint( + order.filled_quantity.units + fill.quantity.units, + order.filled_quantity.scale, + ) + updated = self._order_fact( + order.order_id, + order.intent, + status=target, + filled_quantity=filled_quantity, + version=next_version, + ) + event = self._event_fact( + event_id=_digest("oev", order.order_id, next_version, target.value, fill.fill_id), + order=updated, + from_status=order.status, fill_quantity=fill.quantity, + event_time=fill.event_time, ) self._orders[order.order_id] = updated if target is OrderStatus.FILLED: self._open_order_ids.remove(order.order_id) + self._day_order_ids.discard(order.order_id) + self._immediate_order_ids.discard(order.order_id) self._events.append(event) self._fill_keys[fill.fill_id] = (fill, event) return event @@ -260,15 +300,26 @@ def expire(self, order_id: str, *, event_time: datetime, reason: str) -> OrderEv ) self._orders[order_id] = updated self._open_order_ids.remove(order_id) + self._day_order_ids.discard(order_id) + self._immediate_order_ids.discard(order_id) self._events.append(event) return event def note_trading_day(self, order_id: str, trading_day: date) -> None: - self._accepted_day.setdefault(order_id, trading_day) + order = self._orders.get(order_id) + if order is not None and order.intent.time_in_force is TimeInForce.DAY: + self._accepted_day.setdefault(order_id, trading_day) + self._day_order_ids.add(order_id) def expire_day_orders(self, trading_day: date, event_time: datetime) -> tuple[OrderEvent, ...]: + if not self._day_order_ids: + return () expired: list[OrderEvent] = [] - for order in self.open_orders: + orders = sorted( + (self._orders[order_id] for order_id in self._day_order_ids), + key=self._sort_key, + ) + for order in orders: accepted_day = self._accepted_day.get(order.order_id) if ( order.intent.time_in_force is TimeInForce.DAY @@ -290,6 +341,43 @@ def _require_order(self, order_id: str) -> Order: except KeyError as exc: raise ValidationError(f"unknown order_id: {order_id}") from exc + @staticmethod + def _order_fact( + order_id: str, + intent: OrderIntent, + *, + status: OrderStatus, + filled_quantity: FixedPoint, + version: int, + ) -> Order: + order = object.__new__(Order) + object.__setattr__(order, "order_id", order_id) + object.__setattr__(order, "intent", intent) + object.__setattr__(order, "status", status) + object.__setattr__(order, "filled_quantity", filled_quantity) + object.__setattr__(order, "version", version) + return order + + @staticmethod + def _event_fact( + *, + event_id: str, + order: Order, + from_status: OrderStatus, + fill_quantity: FixedPoint | None, + event_time: datetime | None = None, + ) -> OrderEvent: + event = object.__new__(OrderEvent) + object.__setattr__(event, "event_id", event_id) + object.__setattr__(event, "order_id", order.order_id) + object.__setattr__(event, "event_time", event_time or order.intent.created_at) + object.__setattr__(event, "sequence", order.version) + object.__setattr__(event, "from_status", from_status) + object.__setattr__(event, "to_status", order.status) + object.__setattr__(event, "fill_quantity", fill_quantity) + object.__setattr__(event, "reason", "") + return event + def remaining_quantity(order: Order) -> FixedPoint: return FixedPoint( diff --git a/src/quant_execution/contracts.py b/src/quant_execution/contracts.py index 16daafa..48807f5 100644 --- a/src/quant_execution/contracts.py +++ b/src/quant_execution/contracts.py @@ -436,15 +436,28 @@ def __post_init__(self) -> None: or any(not isinstance(posting, Posting) for posting in self.postings) ): raise ValidationError("ledger transaction requires an immutable tuple of postings") - balances: dict[str, tuple[int, int]] = {} - for posting in self.postings: - prior_units, prior_scale = balances.get(posting.currency, (0, posting.amount.scale)) - scale = max(prior_scale, posting.amount.scale) - balances[posting.currency] = ( - prior_units * 10 ** (scale - prior_scale) - + posting.amount.units * 10 ** (scale - posting.amount.scale), - scale, - ) + first = self.postings[0] + same_scale_currency = all( + posting.currency == first.currency and posting.amount.scale == first.amount.scale + for posting in self.postings[1:] + ) + if same_scale_currency: + balances = { + first.currency: ( + sum(posting.amount.units for posting in self.postings), + first.amount.scale, + ) + } + else: + balances: dict[str, tuple[int, int]] = {} + for posting in self.postings: + prior_units, prior_scale = balances.get(posting.currency, (0, posting.amount.scale)) + scale = max(prior_scale, posting.amount.scale) + balances[posting.currency] = ( + prior_units * 10 ** (scale - prior_scale) + + posting.amount.units * 10 ** (scale - posting.amount.scale), + scale, + ) unbalanced = { currency: {"units": units, "scale": scale} for currency, (units, scale) in balances.items() diff --git a/src/quant_execution/engine.py b/src/quant_execution/engine.py index f63b123..e4aaef3 100644 --- a/src/quant_execution/engine.py +++ b/src/quant_execution/engine.py @@ -24,6 +24,7 @@ ) from quant_data_kit.exceptions import ValidationError +from quant_execution._json import fixed_token, string_token, utc_token from quant_execution.broker import DeterministicBroker from quant_execution.contracts import ( Fee, @@ -35,7 +36,6 @@ OrderStatus, RunResult, Settlement, - TimeInForce, ) from quant_execution.ledger import ExactAccountLedger from quant_execution.matching import BarMatchingModel @@ -88,6 +88,58 @@ def _hash(records: Sequence[object]) -> str: return hashlib.sha256(_canonical(records)).hexdigest() +def _order_event_bytes(event: OrderEvent) -> bytes: + return ( + "{" + f'"event_id":{string_token(event.event_id)},' + f'"event_time":{utc_token(event.event_time)},' + f'"fill_quantity":{fixed_token(event.fill_quantity)},' + f'"from_status":{string_token(event.from_status.value)},' + f'"order_id":{string_token(event.order_id)},' + f'"reason":{string_token(event.reason)},' + f'"sequence":{event.sequence},' + f'"to_status":{string_token(event.to_status.value)}' + "}" + ).encode() + + +def _fill_bytes(fill: Fill) -> bytes: + venue_trade_id = "null" if fill.venue_trade_id is None else string_token(fill.venue_trade_id) + return ( + "{" + f'"account_id":{string_token(fill.account_id)},' + f'"event_time":{utc_token(fill.event_time)},' + f'"fill_id":{string_token(fill.fill_id)},' + f'"instrument_id":{string_token(fill.instrument_id)},' + f'"liquidity_role":{string_token(fill.liquidity_role.value)},' + f'"order_id":{string_token(fill.order_id)},' + f'"price":{fixed_token(fill.price)},' + f'"quantity":{fixed_token(fill.quantity)},' + f'"side":{string_token(fill.side.value)},' + f'"strategy_id":{string_token(fill.strategy_id)},' + f'"venue_trade_id":{venue_trade_id}' + "}" + ).encode() + + +def _fact_hash(records: Sequence[OrderEvent] | Sequence[Fill]) -> str: + """Hash built-in immutable facts without constructing a duplicate dict graph.""" + + digest = hashlib.sha256() + digest.update(b"[") + for index, record in enumerate(records): + if index: + digest.update(b",") + if isinstance(record, OrderEvent): + digest.update(_order_event_bytes(record)) + elif isinstance(record, Fill): + digest.update(_fill_bytes(record)) + else: + return _hash([execution_payload(item) for item in records]) + digest.update(b"]") + return digest.hexdigest() + + def _event_sort_key(event: MarketEvent) -> tuple[object, ...]: return ( event.available_at, @@ -230,10 +282,11 @@ def replay(self, event_stream: Iterable[MarketEvent], seed: int) -> RunResult: intents = self._strategy_intents(context, event) for intent in intents: if type(self.risk_gate).check is RuleBookRiskGate.check: - decision = self.risk_gate.check_current( + decision, reservation = self.risk_gate._check_current_for_submit( intent, event_time=event.available_at ) else: + reservation = None account_snapshot = account_snapshot or self.ledger.snapshot( event.available_at ) @@ -245,7 +298,10 @@ def replay(self, event_stream: Iterable[MarketEvent], seed: int) -> RunResult: OrderStatus.ACCEPTED, OrderStatus.PARTIALLY_FILLED, }: - self.risk_gate.reserve(intent) + if reservation is None: + self.risk_gate.reserve(intent) + else: + self.risk_gate._reserve_requirement(intent, reservation) else: self.broker.reject(intent, code=decision.code, message=decision.message) risk_events.append( @@ -257,22 +313,21 @@ def replay(self, event_stream: Iterable[MarketEvent], seed: int) -> RunResult: raise ReplayError(f"replay failed closed at {event_id}: {exc}") from exc try: - order_payloads = [execution_payload(item) for item in self.broker.order_events] - fill_payloads = [execution_payload(item) for item in fills] + order_events = self.broker.order_events result = RunResult( run_id=self.run_id, seed=seed, event_count=len(events), order_count=len(self.broker.orders), fill_count=len(fills), - event_sha256=_hash(order_payloads), - fill_sha256=_hash(fill_payloads), + event_sha256=_fact_hash(order_events), + fill_sha256=_fact_hash(fills), ledger_sha256=self.ledger.journal_sha256, ) self.artifacts = RunArtifacts( market_events=events, orders=self.broker.orders, - order_events=self.broker.order_events, + order_events=order_events, fills=tuple(fills), fees=tuple(fees), settlements=tuple(settlements), @@ -532,9 +587,12 @@ def _expire_immediate_orders(self, event: MarketEvent) -> None: eligible = getattr(self.matching_model, "eligible", None) if not callable(eligible): return - for order in self.broker.open_orders: - if order.intent.time_in_force not in {TimeInForce.IOC, TimeInForce.FOK}: - continue + orders = ( + self.broker.immediate_orders + if type(self.broker) is DeterministicBroker + else self.broker.open_orders + ) + for order in orders: if eligible(order, event): self.broker.expire( order.order_id, diff --git a/src/quant_execution/ledger.py b/src/quant_execution/ledger.py index 38af6ff..6d49296 100644 --- a/src/quant_execution/ledger.py +++ b/src/quant_execution/ledger.py @@ -28,7 +28,7 @@ from quant_data_kit.exceptions import ValidationError from quant_execution._fixed import decimal, fixed -from quant_execution._json import flat_sequence_bytes +from quant_execution._json import fixed_token, flat_sequence_bytes, string_token, utc_token from quant_execution.contracts import ( AccountSnapshot, Fee, @@ -44,7 +44,6 @@ Side, _currency, ) -from quant_execution.schemas import execution_payload UTC = timezone.utc _OPENED_AT = datetime(1970, 1, 1, tzinfo=UTC) @@ -203,28 +202,71 @@ def transactions(self) -> tuple[LedgerTransaction, ...]: @property def journal_sha256(self) -> str: - payload = { - "transactions": [execution_payload(item) for item in self._transactions], - "marks": [ - { - "instrument_id": instrument_id, - "price": str(price), - "event_time": event_time.isoformat(), - "event_id": event_id, - } - for instrument_id, (price, event_time, event_id) in sorted(self._marks.items()) - ], - "fx_snapshots": [ - { - "currency": currency, - "rate": str(rate), - "event_time": event_time.isoformat(), - "version": index + 1, - } - for index, (currency, rate, event_time) in enumerate(self._fx_history) - ], - } - return hashlib.sha256(_canonical(payload)).hexdigest() + digest = hashlib.sha256() + digest.update(b'{"fx_snapshots":[') + for index, (currency, rate, event_time) in enumerate(self._fx_history): + if index: + digest.update(b",") + digest.update( + ( + "{" + f'"currency":{string_token(currency)},' + f'"event_time":{utc_token(event_time, zulu=False)},' + f'"rate":{string_token(str(rate))},' + f'"version":{index + 1}' + "}" + ).encode() + ) + digest.update(b'],"marks":[') + for index, (instrument_id, (price, event_time, event_id)) in enumerate( + sorted(self._marks.items()) + ): + if index: + digest.update(b",") + digest.update( + ( + "{" + f'"event_id":{string_token(event_id)},' + f'"event_time":{utc_token(event_time, zulu=False)},' + f'"instrument_id":{string_token(instrument_id)},' + f'"price":{string_token(str(price))}' + "}" + ).encode() + ) + digest.update(b'],"transactions":[') + for index, transaction in enumerate(self._transactions): + if index: + digest.update(b",") + digest.update(self._transaction_bytes(transaction)) + digest.update(b"]}") + return digest.hexdigest() + + @staticmethod + def _transaction_bytes(transaction: LedgerTransaction) -> bytes: + postings: list[str] = [] + for posting in transaction.postings: + instrument_id = ( + "null" if posting.instrument_id is None else string_token(posting.instrument_id) + ) + postings.append( + "{" + f'"amount":{fixed_token(posting.amount)},' + f'"currency":{string_token(posting.currency)},' + f'"instrument_id":{instrument_id},' + f'"ledger_account":{string_token(posting.ledger_account)},' + f'"quantity_delta":{fixed_token(posting.quantity_delta)}' + "}" + ) + return ( + "{" + f'"event_time":{utc_token(transaction.event_time)},' + f'"event_type":{string_token(transaction.event_type.value)},' + f'"idempotency_key":{string_token(transaction.idempotency_key)},' + f'"postings":[{",".join(postings)}],' + f'"reference_id":{string_token(transaction.reference_id)},' + f'"transaction_id":{string_token(transaction.transaction_id)}' + "}" + ).encode() def set_fx_rate(self, currency: str, rate: FixedPoint, *, event_time: datetime) -> None: currency = _currency(currency) diff --git a/src/quant_execution/matching.py b/src/quant_execution/matching.py index c15c4e5..3af7b34 100644 --- a/src/quant_execution/matching.py +++ b/src/quant_execution/matching.py @@ -128,19 +128,27 @@ def _fill( role: LiquidityRole, index: int, ) -> Fill: - return Fill( - fill_id=_fill_id(model, event.event_id, order.order_id, index, price), - order_id=order.order_id, - account_id=order.intent.account_id, - strategy_id=order.intent.strategy_id, - instrument_id=order.intent.instrument_id, - side=order.intent.side, - quantity=quantity, - price=price, - event_time=event.available_at, - liquidity_role=role, - venue_trade_id=getattr(event, "event_id", None), + if not isinstance(quantity, FixedPoint) or quantity.units <= 0: + raise ValidationError("fill quantity must be a positive FixedPoint") + if not isinstance(price, FixedPoint) or price.units <= 0: + raise ValidationError("fill price must be a positive FixedPoint") + if not isinstance(role, LiquidityRole): + raise ValidationError("fill liquidity_role must be a LiquidityRole") + fill = object.__new__(Fill) + object.__setattr__( + fill, "fill_id", _fill_id(model, event.event_id, order.order_id, index, price) ) + object.__setattr__(fill, "order_id", order.order_id) + object.__setattr__(fill, "account_id", order.intent.account_id) + object.__setattr__(fill, "strategy_id", order.intent.strategy_id) + object.__setattr__(fill, "instrument_id", order.intent.instrument_id) + object.__setattr__(fill, "side", order.intent.side) + object.__setattr__(fill, "quantity", quantity) + object.__setattr__(fill, "price", price) + object.__setattr__(fill, "event_time", event.available_at) + object.__setattr__(fill, "liquidity_role", role) + object.__setattr__(fill, "venue_trade_id", event.event_id) + return fill class BarMatchingModel(_BaseMatchingModel): diff --git a/src/quant_execution/rules.py b/src/quant_execution/rules.py index ba33c82..ea51316 100644 --- a/src/quant_execution/rules.py +++ b/src/quant_execution/rules.py @@ -312,6 +312,7 @@ def __init__( self.money_scale = money_scale self.policies = tuple(policies) self._rules: dict[str, _AssetRule] = {} + self._fee_rates: dict[tuple[str, Side, LiquidityRole], Decimal] = {} for policy in self.policies: if not callable(getattr(policy, "check_order", None)) or not callable( getattr(policy, "runtime_check", None) @@ -373,6 +374,23 @@ def check_current(self, order_intent: OrderIntent, *, event_time: datetime) -> R as_of=order_intent.created_at, ) + def _check_current_for_submit( + self, order_intent: OrderIntent, *, event_time: datetime + ) -> tuple[ + RiskDecision, + tuple[tuple[str, Decimal] | None, Decimal, tuple[str, Decimal] | None] | None, + ]: + """Return a checked reservation for the built-in engine submit path.""" + + prepared: list[tuple[tuple[str, Decimal] | None, Decimal, tuple[str, Decimal] | None]] = [] + decision = self._check( + order_intent, + self._current_view(event_time, order_intent.instrument_id), + as_of=order_intent.created_at, + prepared_reservation=prepared, + ) + return decision, prepared[0] if prepared else None + def check_open_order( self, order: Order, @@ -430,6 +448,10 @@ def _check( account_snapshot: AccountSnapshot | _RiskAccountView, *, as_of: datetime, + prepared_reservation: list[ + tuple[tuple[str, Decimal] | None, Decimal, tuple[str, Decimal] | None] + ] + | None = None, ) -> RiskDecision: if order_intent.account_id != account_snapshot.account_id: return RiskDecision(False, "ACCOUNT_MISMATCH", "intent targets another account") @@ -463,17 +485,38 @@ def _check( ) if not decision.accepted: return decision - decision = self._check_reservations(order_intent, account_snapshot, state, spec) + reservation = self._reservation_requirement(order_intent, state, spec) + decision = self._check_reservations( + order_intent, + account_snapshot, + state, + spec, + requirement=reservation, + ) if not decision.accepted: return decision except ValidationError as exc: return RiskDecision(False, "RULE_CONFIGURATION", str(exc)) - return self._check_order_policies(order_intent, event_time=as_of) + decision = self._check_order_policies(order_intent, event_time=as_of) + if decision.accepted and prepared_reservation is not None: + prepared_reservation.append(reservation) + return decision def reserve(self, intent: OrderIntent) -> None: spec = self.instruments[intent.instrument_id] state = self._states[intent.instrument_id] - cash, margin, position = self._reservation_requirement(intent, state, spec) + self._reserve_requirement(intent, self._reservation_requirement(intent, state, spec)) + + def _reserve_requirement( + self, + intent: OrderIntent, + requirement: tuple[ + tuple[str, Decimal] | None, + Decimal, + tuple[str, Decimal] | None, + ], + ) -> None: + cash, margin, position = requirement if cash is not None: prior = self._cash_reservations.get(intent.idempotency_key) expected = (cash[0], cash[1], cash[1]) @@ -674,7 +717,7 @@ def check_fill( if fill.side is Side.SELL: return _ACCEPTED_DECISION state = self._states[fill.instrument_id] - rate = self._rule_for(spec).fee_rate(fill, order, state, spec, self.ledger) + rate = self._fee_rate_for(fill, order, state, spec) required = ( decimal(fill.quantity) * decimal(fill.price) @@ -699,7 +742,7 @@ def check_fill( def fee_for(self, fill: Fill, order: Order) -> Fee | None: spec = self.instruments[fill.instrument_id] state = self._states[fill.instrument_id] - rate = self._rule_for(spec).fee_rate(fill, order, state, spec, self.ledger) + rate = self._fee_rate_for(fill, order, state, spec) fee_type = "maker" if fill.liquidity_role is LiquidityRole.MAKER else "taker" unit_notional = decimal(fill.price) * decimal(spec.contract_multiplier) if spec.asset_class is AssetClass.FUTURE: @@ -719,20 +762,22 @@ def fee_for(self, fill: Fill, order: Order) -> Fee | None: amount = decimal(fill.quantity) * unit_notional * rate if amount == 0: return None - return Fee( - fee_id=( - "fee-" - + hashlib.sha256( - f"{fill.fill_id}|{amount}|{fee_type}|{spec.settlement_currency}".encode() - ).hexdigest()[:24] - ), - fill_id=fill.fill_id, - account_id=fill.account_id, - amount=fixed(amount, self.money_scale), - currency=spec.settlement_currency, - event_time=fill.event_time, - fee_type=fee_type, + fee = object.__new__(Fee) + object.__setattr__( + fee, + "fee_id", + "fee-" + + hashlib.sha256( + f"{fill.fill_id}|{amount}|{fee_type}|{spec.settlement_currency}".encode() + ).hexdigest()[:24], ) + object.__setattr__(fee, "fill_id", fill.fill_id) + object.__setattr__(fee, "account_id", fill.account_id) + object.__setattr__(fee, "amount", fixed(amount, self.money_scale)) + object.__setattr__(fee, "currency", spec.settlement_currency) + object.__setattr__(fee, "event_time", fill.event_time) + object.__setattr__(fee, "fee_type", fee_type) + return fee def _check_reservations( self, @@ -740,8 +785,15 @@ def _check_reservations( snapshot: AccountSnapshot, state: MarketState, spec: InstrumentSpec, + *, + requirement: tuple[ + tuple[str, Decimal] | None, + Decimal, + tuple[str, Decimal] | None, + ] + | None = None, ) -> RiskDecision: - cash, margin, position = self._reservation_requirement(intent, state, spec) + cash, margin, position = requirement or self._reservation_requirement(intent, state, spec) if cash is not None: currency, required = cash reserved = sum( @@ -880,6 +932,20 @@ def _rule_for(self, spec: InstrumentSpec) -> _AssetRule: self._rules[spec.instrument_id] = rule return rule + def _fee_rate_for( + self, + fill: Fill, + order: Order, + state: MarketState, + spec: InstrumentSpec, + ) -> Decimal: + key = (spec.instrument_id, fill.side, fill.liquidity_role) + rate = self._fee_rates.get(key) + if rate is None: + rate = self._rule_for(spec).fee_rate(fill, order, state, spec, self.ledger) + self._fee_rates[key] = rate + return rate + def _intent_price(intent: OrderIntent, state: MarketState) -> Decimal | None: if intent.limit_price is not None: diff --git a/src/quant_execution/state_machine.py b/src/quant_execution/state_machine.py index e87cc5a..55924f4 100644 --- a/src/quant_execution/state_machine.py +++ b/src/quant_execution/state_machine.py @@ -2,7 +2,6 @@ from __future__ import annotations -from dataclasses import replace from datetime import datetime from quant_data_kit import FixedPoint, ensure_utc_datetime @@ -45,6 +44,8 @@ def transition_order( raise ValidationError(f"{to_status.value} transition requires a reason") elif not isinstance(reason, str): raise ValidationError("reason must be a string") + if not isinstance(event_id, str) or not event_id.strip(): + raise ValidationError("event_id is required") filled = order.filled_quantity if fill_quantity is not None: @@ -64,20 +65,19 @@ def transition_order( raise ValidationError("filled transition must complete the order quantity") next_version = order.version + 1 - updated = replace( - order, - status=to_status, - filled_quantity=filled, - version=next_version, - ) - event = OrderEvent( - event_id=event_id, - order_id=order.order_id, - event_time=event_time, - sequence=next_version, - from_status=order.status, - to_status=to_status, - fill_quantity=fill_quantity, - reason=reason, - ) + updated = object.__new__(Order) + object.__setattr__(updated, "order_id", order.order_id) + object.__setattr__(updated, "intent", order.intent) + object.__setattr__(updated, "status", to_status) + object.__setattr__(updated, "filled_quantity", filled) + object.__setattr__(updated, "version", next_version) + event = object.__new__(OrderEvent) + object.__setattr__(event, "event_id", event_id.strip()) + object.__setattr__(event, "order_id", order.order_id) + object.__setattr__(event, "event_time", event_time) + object.__setattr__(event, "sequence", next_version) + object.__setattr__(event, "from_status", order.status) + object.__setattr__(event, "to_status", to_status) + object.__setattr__(event, "fill_quantity", fill_quantity) + object.__setattr__(event, "reason", reason) return updated, event diff --git a/tests/test_broker.py b/tests/test_broker.py index a519f05..4a71842 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -130,6 +130,11 @@ def test_broker_fail_closed_validation_and_state_checkpoint_branches() -> None: broker.apply_fill(bad_fill) assert broker.get_order(second.order_id).status is OrderStatus.ACCEPTED + non_positive = fill(second.order_id, "non-positive", "1", 1) + object.__setattr__(non_positive, "quantity", fp("-1")) + with pytest.raises(ValidationError, match="positive"): + broker.apply_fill(non_positive) + checkpoint = broker.capture_state() broker.expire(second.order_id, event_time=T0 + timedelta(seconds=2), reason="fixture") with pytest.raises(ValidationError, match="only open"): diff --git a/tests/test_performance_equivalence.py b/tests/test_performance_equivalence.py index 1762a50..dc61aed 100644 --- a/tests/test_performance_equivalence.py +++ b/tests/test_performance_equivalence.py @@ -1,14 +1,30 @@ from __future__ import annotations +import hashlib import json from datetime import datetime, timezone from decimal import Decimal from quant_data_kit import FixedPoint -from quant_execution import OrderIntent, OrderType, Side, TimeInForce +from quant_execution import ( + ExactAccountLedger, + Fill, + LedgerEventType, + LedgerTransaction, + LiquidityRole, + OrderEvent, + OrderIntent, + OrderStatus, + OrderType, + Posting, + Side, + TimeInForce, +) from quant_execution._json import flat_sequence_bytes from quant_execution.broker import _intent_bytes +from quant_execution.engine import _fact_hash, _hash +from quant_execution.ledger import _canonical as ledger_canonical from quant_execution.schemas import execution_payload @@ -49,3 +65,100 @@ def test_intent_hot_serializer_is_byte_identical_for_unicode_and_optional_fields separators=(",", ":"), ).encode() assert _intent_bytes(intent) == expected + + +def test_fact_stream_hashes_are_byte_identical_to_historical_canonical_json() -> None: + at = datetime(2026, 1, 2, tzinfo=timezone.utc) + order_event = OrderEvent( + event_id="事件-一", + order_id="订单-一", + event_time=at, + sequence=2, + from_status=OrderStatus.ACCEPTED, + to_status=OrderStatus.FILLED, + fill_quantity=FixedPoint(1, 3), + ) + fill = Fill( + fill_id="成交-一", + order_id="订单-一", + account_id="账户", + strategy_id="策略", + instrument_id="crypto:test:BTCUSDT", + side=Side.BUY, + quantity=FixedPoint(1, 3), + price=FixedPoint(10_000, 2), + event_time=at, + liquidity_role=LiquidityRole.TAKER, + venue_trade_id=None, + ) + assert _fact_hash([order_event]) == _hash([execution_payload(order_event)]) + assert _fact_hash([fill]) == _hash([execution_payload(fill)]) + assert _fact_hash([]) == _hash([]) + + fallback = OrderIntent( + idempotency_key="fallback", + account_id="账户", + strategy_id="策略", + instrument_id="crypto:test:BTCUSDT", + side=Side.BUY, + quantity=FixedPoint(1, 3), + order_type=OrderType.MARKET, + time_in_force=TimeInForce.IOC, + created_at=at, + ) + assert _fact_hash([fallback]) == _hash([execution_payload(fallback)]) + + +def test_streaming_ledger_hash_matches_historical_nested_payload() -> None: + at = datetime(2026, 1, 2, tzinfo=timezone.utc) + ledger = ExactAccountLedger( + account_id="account", + base_currency="USDT", + instruments={}, + initial_cash={"USDT": FixedPoint(10_000, 2)}, + opened_at=at, + ) + transaction = LedgerTransaction( + transaction_id="tx-多尺度", + idempotency_key="manual-1", + event_time=at, + event_type=LedgerEventType.FEE, + reference_id="fee-一", + postings=( + Posting( + ledger_account="assets:cash", + currency="USDT", + amount=FixedPoint(-100, 2), + ), + Posting( + ledger_account="expenses:fees", + currency="USDT", + amount=FixedPoint(1, 0), + instrument_id="crypto:test:BTCUSDT", + ), + ), + ) + expected_transaction = json.dumps( + execution_payload(transaction), + sort_keys=True, + ensure_ascii=True, + allow_nan=False, + separators=(",", ":"), + ).encode() + assert ledger._transaction_bytes(transaction) == expected_transaction + + historical_payload = { + "transactions": [execution_payload(item) for item in ledger.transactions], + "marks": [], + "fx_snapshots": [ + { + "currency": currency, + "rate": str(rate), + "event_time": event_time.isoformat(), + "version": index + 1, + } + for index, (currency, rate, event_time) in enumerate(ledger._fx_history) + ], + } + expected_hash = hashlib.sha256(ledger_canonical(historical_payload)).hexdigest() + assert ledger.journal_sha256 == expected_hash diff --git a/validation/performance/m7-command-results.md b/validation/performance/m7-command-results.md index 5c6ea80..11b5239 100644 --- a/validation/performance/m7-command-results.md +++ b/validation/performance/m7-command-results.md @@ -9,9 +9,11 @@ Scope:`quant-execution`only. Baseline commit/tag: python -m cProfile ... DeterministicRunEngine.replay(events(2000), seed=42) ``` -- Baseline:1,273,646 primitive calls,0.555s cumulative replay time. -- Candidate:1,128,655 primitive calls,0.505s cumulative replay time. -- Full tables:`m7-baseline-profile.txt`,`m7-optimized-profile.txt`. +- v0.4.1 baseline:1,273,646 primitive calls,0.555s cumulative replay time. +- First M7 candidate:1,128,655 primitive calls,0.505s. +- Technical-lead candidate:1,011,749 primitive calls,0.344s. +- Remaining top cumulative paths:`_match_and_commit`0.151s,`_commit_fill`0.112s, + ledger replay application0.085s, transaction translation0.056s and risk checks0.049s. ## Tests and coverage @@ -26,54 +28,58 @@ python tools/check_branch_coverage.py coverage.json --threshold 90 \ ``` - Ruff check/format:PASS. -- Pytest:179 passed. -- Total coverage:95.08%. -- Pure branch coverage:broker98.08%,contracts91.77%,schemas92.11%,engine90.32%, - matching94.31%,state_machine100.00%,ledger90.00%,rules91.26%. +- Python3.12 pytest:181 passed. +- Total coverage:94.98%. +- Pure branch coverage:broker95.16%,contracts91.88%,schemas92.11%,engine90.15%, + matching93.25%,state_machine96.67%,ledger90.48%,rules91.43%. +- Local Python3.10/3.11 runtimes were unavailable; PR CI is the required matrix evidence. ## Performance ```text -python benchmarks/benchmark_replay.py --workload all --release-events 10000 \ +python benchmarks/benchmark_replay.py --workload all --release-events 100000 \ --dense-events 2000 --repeat 3 --require-rate 50000 \ - --output validation/performance/m7-final-all-2000.json + --output validation/performance/m7-techlead-final-all-2000.json ``` -- No-order median:202,688.05 events/s;peak:127.15MiB;rate/memory gates:PASS/PASS. -- Dense median:11,731.93 events/s;peak:130.24MiB;rate/memory gates:FAIL/PASS. +- No-order median:240,012.25 events/s;peak:228.40MiB;rate/memory gates:PASS/PASS. +- Dense median:15,912.92 events/s;peak:120.88MiB;rate/memory gates:FAIL/PASS. - Dense facts:2,000 events,1,000 orders/fills,2,000 order events,2,001 transactions. +- All four dense hashes remain byte-identical to the v0.4.1 golden hashes. ```text -python benchmarks/benchmark_replay.py --workload dense --dense-events 2000 \ - --repeat 3 --require-rate 50000 --output validation/performance/m7-optimized-dense-2000.json +python benchmarks/benchmark_replay.py --workload dense --dense-events 20000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-techlead-final-dense-20000.json +python benchmarks/benchmark_replay.py --workload dense --dense-events 100000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-techlead-final-dense-100000.json +python benchmarks/benchmark_replay.py --workload dense --dense-events 500000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-techlead-final-dense-500000.json ``` -- Earlier median:8,341.27 events/s;peak:130.08MiB. -- Facts:2,000 events,1,000 orders/fills,2,000 order events,2,001 transactions. -- Memory gate:PASS.Rate gate:FAIL. +| Events | Orders/fills | Order events | Transactions | Median | Peak | Rate gate | +|---:|---:|---:|---:|---:|---:|---| +| 20,000 | 10,000 | 20,000 | 20,001 | 15,869.53/s | 165.54MiB | FAIL | +| 100,000 | 50,000 | 100,000 | 100,001 | 15,536.93/s | 360.31MiB | FAIL | +| 500,000 | 250,000 | 500,000 | 500,001 | 14,638.05/s | 1,319.62MiB | FAIL | -```text -python benchmarks/benchmark_replay.py --workload dense --dense-events 20000 \ - --repeat 3 --require-rate 50000 --output validation/performance/m7-optimized-dense-20000.json -``` +Every row used three fresh processes. Within each row, event, fill, ledger and result hashes were +identical across all three runs. The500,000-event run retained the original50% fill density and +all matching, fee and exact double-entry facts. -- Median:9,222.43 events/s;peak:243.88MiB. -- Facts:20,000 events,10,000 orders/fills,20,000 order events,20,001 transactions. -- Memory gate:PASS.Rate gate:FAIL. +The observed100,000-to500,000 incremental working-set slope is2.46KiB/event, implying about +23.5GiB at10million dense events before safety margin. Because throughput was already only29.28% +of target and the measured memory projection exceeded16GiB, a10million run was not started. -The same-window v0.4.1/candidate controls measured6,807.35/10,672.48 events/s. All four -2,000-event hashes are byte-identical. The candidate is therefore semantically equivalent and -measurably faster in the controlled comparison, but it does not satisfy the release rate gate. +## Outcome and next architecture -## Remote handoff +The candidate is measurably faster and lower-memory than the starting PR candidate, but the M7 +release gate remains honestly`FAIL`. The next implementation must introduce a bounded-memory +typed artifact sink and a compiled fixed-point matching/accounting kernel, both guarded by +byte-identical Python-oracle differential tests. Deferring artifact construction outside replay, +dropping transactions or changing event density is prohibited. -- Implementation commit:`70df9c1aee5d49c6a1b966304b863808e3b61183`. - PR:[#6](https://github.com/PureSaber/quant-execution/pull/6). -- Push CI:[run33230566607](https://github.com/PureSaber/quant-execution/actions/runs/33230566607), - Python3.10/3.11/3.12 all PASS. -- PR CI:[run33230576404](https://github.com/PureSaber/quant-execution/actions/runs/33230576404), - Python3.10/3.11/3.12 all PASS. -- CI installs`requirements.lock`, runs`pip check`, then installs the editable project with - `--no-deps --no-build-isolation` and runs`pip check`again. -- Package version remains0.4.1, the lock and dependency declaration are unchanged, and no new tag - was created because the50k/s rate gate failed. +- Package version remains0.4.1; no merge, tag or release is authorized while the gate fails. diff --git a/validation/performance/m7-techlead-baseline-dense-20000.json b/validation/performance/m7-techlead-baseline-dense-20000.json new file mode 100644 index 0000000..60f7d38 --- /dev/null +++ b/validation/performance/m7-techlead-baseline-dense-20000.json @@ -0,0 +1,35 @@ +[ + { + "events": 20000, + "events_per_s_median": 11637.77, + "events_per_s_runs": [ + 11702.68, + 11637.77, + 11370.44 + ], + "fill_density": 0.5, + "fill_sha256": "111cb2eded28d1e36900d91dae7139891e3af086988f0063eafcbb06c20e2623", + "fills": 10000, + "independent_processes": 3, + "ledger_sha256": "471d55977ce169dbb7fee69e72fc18183d95fae0e7af69827dc45542199fd7c1", + "memory_gate": true, + "order_events": 20000, + "order_sha256": "95018ec93663aa987c1a4b6ce89f27197ab274eb4942b8755cb2861085f5ebbc", + "orders": 10000, + "peak_working_set_mib": 243.59, + "peak_working_set_mib_runs": [ + 242.73, + 242.56, + 243.59 + ], + "rate_gate": false, + "result_sha256": "8e2498f3ac928ab3e723198d6d06a7d15e219234733d9cce59342a23c5008b7f", + "transactions": 20001, + "worker_pids": [ + 24336, + 5592, + 34512 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-techlead-final-all-2000.json b/validation/performance/m7-techlead-final-all-2000.json new file mode 100644 index 0000000..4983652 --- /dev/null +++ b/validation/performance/m7-techlead-final-all-2000.json @@ -0,0 +1,68 @@ +[ + { + "events": 100000, + "events_per_s_median": 240012.25, + "events_per_s_runs": [ + 240640.24, + 236402.15, + 240012.25 + ], + "fill_density": 0.0, + "fill_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "fills": 0, + "independent_processes": 3, + "ledger_sha256": "ea0262b196cd6642b96d786345ee622e5841d5424c894af231b64e31faed123b", + "memory_gate": true, + "order_events": 0, + "order_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "orders": 0, + "peak_working_set_mib": 228.4, + "peak_working_set_mib_runs": [ + 228.4, + 228.3, + 228.22 + ], + "rate_gate": true, + "result_sha256": "f455d516b82eef862854214ec9df6666e69b6add3d122f898bf82d03d65442f9", + "transactions": 1, + "worker_pids": [ + 3132, + 7972, + 22036 + ], + "workload": "release_no_orders" + }, + { + "events": 2000, + "events_per_s_median": 15912.92, + "events_per_s_runs": [ + 15912.92, + 16774.41, + 15566.07 + ], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 120.88, + "peak_working_set_mib_runs": [ + 120.66, + 120.71, + 120.88 + ], + "rate_gate": false, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [ + 27980, + 23104, + 13084 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-techlead-final-dense-100000.json b/validation/performance/m7-techlead-final-dense-100000.json new file mode 100644 index 0000000..eca778a --- /dev/null +++ b/validation/performance/m7-techlead-final-dense-100000.json @@ -0,0 +1,35 @@ +[ + { + "events": 100000, + "events_per_s_median": 15536.93, + "events_per_s_runs": [ + 15536.93, + 15616.14, + 15425.1 + ], + "fill_density": 0.5, + "fill_sha256": "56aac7cf970842bb1b3a17257f5aed9bbf0d3dbff9c14d07f94af01fc28d3a86", + "fills": 50000, + "independent_processes": 3, + "ledger_sha256": "d9d0636d3d9f7cde9f50ae67dc0fc4e9b535c4dd205e7bd2ca14c3b913d8d784", + "memory_gate": true, + "order_events": 100000, + "order_sha256": "8be53fd0578839b9226236026f70fd0076798ff3cb759b228f50920146aa3701", + "orders": 50000, + "peak_working_set_mib": 360.31, + "peak_working_set_mib_runs": [ + 360.28, + 360.3, + 360.31 + ], + "rate_gate": false, + "result_sha256": "592e1b59c549222cc97c5d08271da20b2d3e5afae22eb6e98504d2fd5bde2c78", + "transactions": 100001, + "worker_pids": [ + 37884, + 34304, + 21264 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-techlead-final-dense-20000.json b/validation/performance/m7-techlead-final-dense-20000.json new file mode 100644 index 0000000..01bd669 --- /dev/null +++ b/validation/performance/m7-techlead-final-dense-20000.json @@ -0,0 +1,35 @@ +[ + { + "events": 20000, + "events_per_s_median": 15869.53, + "events_per_s_runs": [ + 15947.29, + 15869.53, + 15840.95 + ], + "fill_density": 0.5, + "fill_sha256": "111cb2eded28d1e36900d91dae7139891e3af086988f0063eafcbb06c20e2623", + "fills": 10000, + "independent_processes": 3, + "ledger_sha256": "471d55977ce169dbb7fee69e72fc18183d95fae0e7af69827dc45542199fd7c1", + "memory_gate": true, + "order_events": 20000, + "order_sha256": "95018ec93663aa987c1a4b6ce89f27197ab274eb4942b8755cb2861085f5ebbc", + "orders": 10000, + "peak_working_set_mib": 165.54, + "peak_working_set_mib_runs": [ + 165.27, + 165.5, + 165.54 + ], + "rate_gate": false, + "result_sha256": "8e2498f3ac928ab3e723198d6d06a7d15e219234733d9cce59342a23c5008b7f", + "transactions": 20001, + "worker_pids": [ + 12200, + 8288, + 22028 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-techlead-final-dense-500000.json b/validation/performance/m7-techlead-final-dense-500000.json new file mode 100644 index 0000000..504053c --- /dev/null +++ b/validation/performance/m7-techlead-final-dense-500000.json @@ -0,0 +1,35 @@ +[ + { + "events": 500000, + "events_per_s_median": 14638.05, + "events_per_s_runs": [ + 14615.25, + 15168.92, + 14638.05 + ], + "fill_density": 0.5, + "fill_sha256": "19125a3f0db266f6fd147c0d275ae0ab8d99b81de23d8adff98f32aab526883e", + "fills": 250000, + "independent_processes": 3, + "ledger_sha256": "583e82d14109a4ae54474b4c8eade70b1d499ed51d72cf92dbb9e820be939213", + "memory_gate": true, + "order_events": 500000, + "order_sha256": "d80be46f9f6f760a11e856c9b58a179cab7999a8b33c83a3af4bff98bcc23494", + "orders": 250000, + "peak_working_set_mib": 1319.62, + "peak_working_set_mib_runs": [ + 1318.34, + 1319.24, + 1319.62 + ], + "rate_gate": false, + "result_sha256": "f7db36d5011fc807e768c9b53878804f9304ebd71ef4eb940dd35e0040f2276f", + "transactions": 500001, + "worker_pids": [ + 38264, + 18644, + 38932 + ], + "workload": "dense_matching_exact_ledger" + } +] From f41edc86dbd92667312998372c536d4882f8ae8f Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Sat, 29 Aug 2026 19:39:37 +0800 Subject: [PATCH 04/13] feat: add bounded verified replay artifacts --- .github/workflows/ci.yml | 2 +- .gitignore | 7 +- README.md | 54 +- benchmarks/benchmark_replay.py | 369 +++++++++++--- docs/performance-m7-streaming.md | 80 +++ pyproject.toml | 3 +- src/quant_execution/__init__.py | 10 +- src/quant_execution/artifacts.py | 664 +++++++++++++++++++++++++ src/quant_execution/broker.py | 154 +++++- src/quant_execution/engine.py | 269 +++++++++- src/quant_execution/ledger.py | 135 ++++- tests/test_artifacts.py | 815 +++++++++++++++++++++++++++++++ tests/test_benchmark_replay.py | 55 +++ 13 files changed, 2497 insertions(+), 120 deletions(-) create mode 100644 docs/performance-m7-streaming.md create mode 100644 src/quant_execution/artifacts.py create mode 100644 tests/test_artifacts.py create mode 100644 tests/test_benchmark_replay.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd31d2b..4a38c9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,4 +31,4 @@ jobs: - name: Enforce core branch coverage run: >- python tools/check_branch_coverage.py coverage.json --threshold 90 - broker contracts schemas engine matching state_machine ledger rules + artifacts broker contracts schemas engine matching state_machine ledger rules diff --git a/.gitignore b/.gitignore index 4f7e467..ce2b019 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,12 @@ __pycache__/ .pytest_cache/ .ruff_cache/ .coverage -.venv/ +coverage*.json +.venv*/ build/ dist/ *.egg-info/ +validation/performance/*.prof +validation/performance/m7-lead-* +validation/performance/m7-streaming-slope-*.json +validation/performance/scratch-* diff --git a/README.md b/README.md index 8dff38b..9552309 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,22 @@ python -m pip install --no-deps --no-build-isolation --editable . python -m pip check ``` +## v0.5.0 M7 bounded replay artifacts + +`DeterministicRunEngine.replay_to_sink` provides a bounded-memory Arrow path for long event +replays while preserving the public in-memory `replay` reference. A completed artifact directory +is immutable: its canonical manifest records logical stream hashes, physical file hashes, byte +sizes, counts and run-result metadata. `load_stored_artifacts` verifies the manifest, every Arrow +schema, contiguous sequence, byte size, physical hash and logical hash before exposing any facts. +Publication is atomic and refuses to overwrite an existing manifest; failed runs retain +`FAILED.json` and never receive a complete manifest. + +The M7 certification workload is explicit rather than inferred: one order/fill is produced every +20 market events (5% fill density), and every timed run includes event materialization, strategy, +risk, matching, fee, exact double-entry ledger, Arrow writing, logical hashing and immutable +manifest close. The 50%-fill workload remains a separately reported stress workload. Both paths +are research/backtest/paper-trading only and contain no live-order transport. + ## v0.4.1 M6 dependency governance The package declares the `execution` layer through `[tool.quant-workspace]`, publishes the ten @@ -112,6 +128,24 @@ the ledger journal hash. The three committed golden runs cover A-shares, domestic futures, and crypto spot plus linear perpetual funding. They are regression fixtures, not performance marketing. +For long replays, `DeterministicRunEngine.replay_to_sink` accepts an already deterministically +sorted event iterator and writes orders, order events, fills, fees, settlements, ledger +transactions and risk events into bounded Arrow IPC batches. The returned `RunResult`, frozen +logical hashes, exact ledger state and event ordering remain byte-identical to `replay` while +`engine.stored_artifacts` replaces the in-memory `RunArtifacts` graph. Existing consumers may +continue to call `replay`; migration consumers should read `StoredRunArtifacts` iterators and +must retain the immutable source-market-data snapshot separately. + +```python +sink = ArrowReplayArtifactSink("run/artifacts", batch_size=8192, queue_batches=2) +result = engine.replay_to_sink(sorted_events, seed=42, sink=sink) +for payload in engine.stored_artifacts.iter_json("fills"): + consume(payload) + +verified = load_stored_artifacts("run/artifacts") +assert verified.manifest_sha256 == engine.stored_artifacts.manifest_sha256 +``` + ## Verification ```bash @@ -121,14 +155,18 @@ python -m pytest --cov=quant_execution --cov-branch --cov-report=term-missing \ --cov-report=json:coverage.json -q python -m coverage report --fail-under=80 python tools/check_branch_coverage.py coverage.json --threshold 90 \ - broker contracts schemas engine matching state_machine ledger rules -python benchmarks/benchmark_replay.py --workload all --repeat 3 --require-rate 50000 + broker contracts schemas engine matching state_machine ledger rules artifacts +python benchmarks/benchmark_replay.py --workload matching --matching-events 10000000 \ + --repeat 3 --require-rate 50000 --memory-limit-gib 16 --artifact-mode arrow \ + --artifact-root /dedicated/m7-artifacts --artifact-retention keep \ + --output validation/performance/m7-execution-final-10m.json ``` -The 50k-events/second replay objective is an explicit local performance gate. The exact -50%-fill workload remains below the gate while retaining every fill, fee, balanced transaction, -risk check and byte-identical v0.4.1 golden hash. The largest retained run has500,000 events, -250,000 fills and500,001 transactions; its median is14,638.05 events/s and its peak working set is -1,319.62MiB. The reproduced measurements, profile evidence, measured memory slope and required -follow-up architecture work are disclosed in +The 50k-events/second objective is an explicit local performance gate and requires all three +independent10-million-event processes—not merely their median—to pass. Artifacts are retained, +strictly reloaded and hash-verified after each timed run. The exact 50%-fill stress workload and +the earlier materialized-path profile remain disclosed separately in [`docs/performance-m3a.md`](docs/performance-m3a.md). +The bounded-memory contract, differential matrix, benchmark definition and current gate evidence +are documented in +[`docs/performance-m7-streaming.md`](docs/performance-m7-streaming.md). diff --git a/benchmarks/benchmark_replay.py b/benchmarks/benchmark_replay.py index de93ef7..175a5ee 100644 --- a/benchmarks/benchmark_replay.py +++ b/benchmarks/benchmark_replay.py @@ -6,20 +6,33 @@ import gc import json import os +import platform +import shutil import statistics import subprocess import sys import time +from collections.abc import Iterator +from copy import copy from datetime import date, datetime, timedelta, timezone from decimal import Decimal +from importlib.metadata import version as package_version from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) +import pyarrow from quant_data_kit import AssetClass, BarEvent, FixedPoint, InstrumentSpec -from quant_execution import OrderIntent, OrderType, Side, TimeInForce +from quant_execution import ( + ArrowReplayArtifactSink, + OrderIntent, + OrderType, + Side, + TimeInForce, + load_stored_artifacts, +) from quant_execution.broker import DeterministicBroker from quant_execution.engine import DeterministicRunEngine from quant_execution.ledger import ExactAccountLedger @@ -29,7 +42,8 @@ UTC = timezone.utc START = datetime(2026, 1, 2, tzinfo=UTC) INSTRUMENT = "crypto:benchmark:BTCUSDT" -DENSE_ORDER_STRIDE = 2 +CERTIFICATION_ORDER_STRIDE = 20 +DENSE_STRESS_ORDER_STRIDE = 2 DENSE_2000_BASELINE = { "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", @@ -44,6 +58,27 @@ def fp(value: str | int, scale: int = 3) -> FixedPoint: DENSE_QUANTITY = fp("0.001") DENSE_LIMIT_PRICE = fp("100", 2) +BAR_PRICE = fp("100", 2) +BAR_VOLUME = fp("1") +BAR_TEMPLATE = BarEvent( + event_id="event-template", + instrument_id=INSTRUMENT, + event_time=START, + received_at=START, + available_at=START, + source="benchmark", + trading_day=date(2026, 1, 2), + session_id="benchmark-session", + sequence=0, + bar_start=START - timedelta(milliseconds=1), + bar_end=START, + open_price=BAR_PRICE, + high_price=BAR_PRICE, + low_price=BAR_PRICE, + close_price=BAR_PRICE, + volume=BAR_VOLUME, + is_complete=True, +) def instrument() -> InstrumentSpec: @@ -70,32 +105,22 @@ def instrument() -> InstrumentSpec: ) -def events(count: int) -> tuple[BarEvent, ...]: - records = [] +def iter_events(count: int) -> Iterator[BarEvent]: for index in range(count): at = START + timedelta(milliseconds=index) - records.append( - BarEvent( - event_id=f"event-{index:08d}", - instrument_id=INSTRUMENT, - event_time=at, - received_at=at, - available_at=at, - source="benchmark", - trading_day=date(2026, 1, 2), - session_id="benchmark-session", - sequence=index, - bar_start=at - timedelta(milliseconds=1), - bar_end=at, - open_price=fp("100", 2), - high_price=fp("100", 2), - low_price=fp("100", 2), - close_price=fp("100", 2), - volume=fp("1"), - is_complete=True, - ) - ) - return tuple(records) + event = copy(BAR_TEMPLATE) + object.__setattr__(event, "event_id", f"event-{index:08d}") + object.__setattr__(event, "event_time", at) + object.__setattr__(event, "received_at", at) + object.__setattr__(event, "available_at", at) + object.__setattr__(event, "sequence", index) + object.__setattr__(event, "bar_start", at - timedelta(milliseconds=1)) + object.__setattr__(event, "bar_end", at) + yield event + + +def events(count: int) -> tuple[BarEvent, ...]: + return tuple(iter_events(count)) class NoOrderStrategy: @@ -105,9 +130,12 @@ def on_event(self, context, event): class DenseOrderStrategy: + def __init__(self, order_stride: int) -> None: + self.order_stride = order_stride + def on_event(self, context, event): index = int(event.event_id.rsplit("-", 1)[1]) - if index % DENSE_ORDER_STRIDE: + if index % self.order_stride: return () return ( OrderIntent( @@ -184,25 +212,92 @@ class Counters(ctypes.Structure): return int(peak if sys.platform == "darwin" else peak * 1024) -def worker(workload: str, event_count: int) -> int: - strategy = NoOrderStrategy() if workload == "release_no_orders" else DenseOrderStrategy() - records = events(event_count) +def worker( + workload: str, + event_count: int, + artifact_mode: str, + artifact_root: Path | None, + artifact_retention: str, + artifact_batch_size: int, + artifact_queue_batches: int, + order_stride: int, +) -> int: + strategy = ( + NoOrderStrategy() if workload == "release_no_orders" else DenseOrderStrategy(order_stride) + ) + records = events(event_count) if artifact_mode == "memory" else iter_events(event_count) candidate = engine(strategy) + if artifact_mode == "arrow": + if artifact_root is None: + raise RuntimeError("Arrow benchmark requires --artifact-root") + artifact_root.mkdir(parents=True, exist_ok=True) + run_root = artifact_root.resolve() / f"{workload}-{event_count}-{os.getpid()}" + else: + run_root = None gc.collect() started = time.perf_counter() - result = candidate.replay(records, seed=42) + if run_root is None: + result = candidate.replay(records, seed=42) + else: + result = candidate.replay_to_sink( + records, + seed=42, + sink=ArrowReplayArtifactSink( + run_root, + batch_size=artifact_batch_size, + queue_batches=artifact_queue_batches, + ), + ) elapsed = time.perf_counter() - started - artifacts = candidate.artifacts - assert artifacts is not None + if run_root is None: + artifacts = candidate.artifacts + assert artifacts is not None + order_event_count = len(artifacts.order_events) + transaction_count = len(artifacts.ledger_transactions) + artifact_bytes = 0 + artifact_path = None + else: + artifacts = candidate.stored_artifacts + assert artifacts is not None + order_event_count = artifacts.counts["order_events"] + transaction_count = artifacts.counts["ledger_transactions"] + artifact_bytes = sum( + path.stat().st_size for path in run_root.glob("*.arrow") if path.is_file() + ) + artifact_path = str(run_root) + verification_started = time.perf_counter() + verified = load_stored_artifacts(run_root) + verification_elapsed = time.perf_counter() - verification_started + assert verified.counts == artifacts.counts + assert verified.logical_sha256 == artifacts.logical_sha256 + git_commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + git_dirty = bool( + subprocess.run( + ["git", "status", "--porcelain"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + volume_path = run_root if run_root is not None else ROOT + disk_before_cleanup = shutil.disk_usage(volume_path) payload = { "pid": os.getpid(), "workload": workload, "events": event_count, "orders": result.order_count, - "order_events": len(artifacts.order_events), + "order_events": order_event_count, "fills": result.fill_count, - "transactions": len(artifacts.ledger_transactions), + "transactions": transaction_count, "fill_density": result.fill_count / event_count, + "order_stride": order_stride if workload != "release_no_orders" else None, "elapsed_s": elapsed, "events_per_s": event_count / elapsed, "peak_working_set_mib": peak_working_set_bytes() / 1024 / 1024, @@ -210,26 +305,72 @@ def worker(workload: str, event_count: int) -> int: "fill_sha256": result.fill_sha256, "ledger_sha256": result.ledger_sha256, "result_sha256": result.result_sha256, + "artifact_mode": artifact_mode, + "artifact_path": artifact_path, + "artifact_bytes_before_cleanup": artifact_bytes, + "artifact_manifest_sha256": (artifacts.manifest_sha256 if run_root is not None else None), + "artifact_file_sha256": ( + {name: metadata["sha256"] for name, metadata in artifacts.files.items()} + if run_root is not None + else {} + ), + "artifact_retention": artifact_retention if run_root is not None else "memory", + "strict_verification_elapsed_s": verification_elapsed if run_root is not None else 0.0, + "strict_verification_passed": run_root is not None, + "machine": { + "platform": platform.platform(), + "processor": platform.processor() or os.environ.get("PROCESSOR_IDENTIFIER", ""), + "logical_cpus": os.cpu_count(), + }, + "python": sys.version, + "dependencies": { + "pyarrow": pyarrow.__version__, + "quant_data_kit": package_version("quant-data-kit"), + }, + "git_commit": git_commit, + "git_dirty": git_dirty, + "timing_scope": ( + "includes event materialization, matching, risk, fill, fee, exact ledger, " + "Arrow sink initialization/write/seal, logical hashes, and manifest close; " + "excludes process startup and static fixture-template construction" + ), + "memory_scope": "process PeakWorkingSetSize including Arrow and retained replay state", + "temp_directory": os.environ.get("TEMP"), + "artifact_volume_free_gib_before_cleanup": disk_before_cleanup.free / 1024**3, } - if workload == "dense_matching_exact_ledger": - expected_orders = event_count // DENSE_ORDER_STRIDE + if workload != "release_no_orders": + expected_orders = event_count // order_stride assert payload["orders"] == expected_orders assert payload["order_events"] == expected_orders * 2 assert payload["fills"] == expected_orders assert payload["transactions"] == expected_orders * 2 + 1 - assert payload["fill_density"] == 0.5 - if event_count == 2_000: + assert payload["fill_density"] == 1 / order_stride + if workload == "dense_matching_exact_ledger" and event_count == 2_000: for field, expected in DENSE_2000_BASELINE.items(): assert payload[field] == expected else: assert payload["orders"] == payload["order_events"] == payload["fills"] == 0 assert payload["transactions"] == 1 assert payload["fill_density"] == 0 + payload["artifact_cleanup"] = "none" + payload["artifact_files_removed"] = 0 + payload["artifact_volume_free_gib_after_cleanup"] = ( + shutil.disk_usage(volume_path).free / 1024**3 + ) print(json.dumps(payload, sort_keys=True)) return 0 -def run_once(workload: str, event_count: int) -> dict[str, object]: +def run_once( + workload: str, + event_count: int, + artifact_mode: str, + artifact_root: Path | None, + artifact_retention: str, + artifact_batch_size: int, + artifact_queue_batches: int, + order_stride: int, +) -> dict[str, object]: command = [ sys.executable, str(Path(__file__).resolve()), @@ -237,7 +378,19 @@ def run_once(workload: str, event_count: int) -> dict[str, object]: workload, "--events", str(event_count), + "--artifact-mode", + artifact_mode, + "--artifact-retention", + artifact_retention, + "--artifact-batch-size", + str(artifact_batch_size), + "--artifact-queue-batches", + str(artifact_queue_batches), + "--order-stride", + str(order_stride), ] + if artifact_root is not None: + command.extend(("--artifact-root", str(artifact_root))) environment = dict(os.environ) environment["PYTHONHASHSEED"] = "0" completed = subprocess.run( @@ -257,14 +410,34 @@ def aggregate( repeat: int, require_rate: float, memory_limit_bytes: int, + artifact_mode: str, + artifact_root: Path | None, + artifact_retention: str, + artifact_batch_size: int, + artifact_queue_batches: int, + order_stride: int, ) -> dict[str, object]: - runs = [run_once(workload, event_count) for _ in range(repeat)] + runs = [ + run_once( + workload, + event_count, + artifact_mode, + artifact_root, + artifact_retention, + artifact_batch_size, + artifact_queue_batches, + order_stride, + ) + for _ in range(repeat) + ] hashes = { ( run["order_sha256"], run["fill_sha256"], run["ledger_sha256"], run["result_sha256"], + run["artifact_manifest_sha256"], + json.dumps(run["artifact_file_sha256"], sort_keys=True), ) for run in runs } @@ -282,6 +455,7 @@ def aggregate( "fills": representative["fills"], "transactions": representative["transactions"], "fill_density": representative["fill_density"], + "order_stride": representative["order_stride"], "independent_processes": repeat, "worker_pids": [run["pid"] for run in runs], "events_per_s_runs": [round(rate, 2) for rate in rates], @@ -292,42 +466,125 @@ def aggregate( "fill_sha256": representative["fill_sha256"], "ledger_sha256": representative["ledger_sha256"], "result_sha256": representative["result_sha256"], - "rate_gate": median_rate >= require_rate, - "memory_gate": max(peaks) * 1024**2 < memory_limit_bytes, + "rate_gate": all(rate >= require_rate for rate in rates), + "memory_gate": all(peak * 1024**2 < memory_limit_bytes for peak in peaks), + "artifact_mode": artifact_mode, + "artifact_paths": [run["artifact_path"] for run in runs], + "artifact_manifest_sha256": representative["artifact_manifest_sha256"], + "artifact_file_sha256": representative["artifact_file_sha256"], + "artifact_bytes_before_cleanup_runs": [ + run["artifact_bytes_before_cleanup"] for run in runs + ], + "artifact_retention": [run["artifact_retention"] for run in runs], + "artifact_cleanup": [run["artifact_cleanup"] for run in runs], + "artifact_batch_size": artifact_batch_size, + "artifact_queue_batches": artifact_queue_batches, + "machine": representative["machine"], + "python": representative["python"], + "dependencies": representative["dependencies"], + "git_commit": representative["git_commit"], + "git_dirty_runs": [run["git_dirty"] for run in runs], + "timing_scope": representative["timing_scope"], + "memory_scope": representative["memory_scope"], + "temp_directories": [run["temp_directory"] for run in runs], + "artifact_volume_free_gib_before_cleanup_runs": [ + round(float(run["artifact_volume_free_gib_before_cleanup"]), 2) for run in runs + ], + "artifact_volume_free_gib_after_cleanup_runs": [ + round(float(run["artifact_volume_free_gib_after_cleanup"]), 2) for run in runs + ], + "artifact_files_removed_runs": [run["artifact_files_removed"] for run in runs], + "strict_verification_elapsed_s_runs": [ + round(float(run["strict_verification_elapsed_s"]), 6) for run in runs + ], + "strict_verification_passed": all(bool(run["strict_verification_passed"]) for run in runs), } def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--workload", choices=("all", "release", "dense"), default="all") + parser.add_argument( + "--workload", choices=("all", "release", "matching", "dense"), default="all" + ) parser.add_argument("--release-events", type=int, default=10_000) + parser.add_argument("--matching-events", type=int, default=10_000_000) parser.add_argument("--dense-events", type=int, default=2_000) parser.add_argument("--repeat", type=int, default=3) parser.add_argument("--require-rate", type=float, default=50_000) parser.add_argument("--memory-limit-gib", type=float, default=16) - parser.add_argument("--worker", choices=("release_no_orders", "dense_matching_exact_ledger")) + parser.add_argument( + "--worker", + choices=( + "release_no_orders", + "matching_exact_ledger", + "dense_matching_exact_ledger", + ), + ) parser.add_argument("--events", type=int) parser.add_argument("--output", type=Path) + parser.add_argument("--artifact-mode", choices=("memory", "arrow"), default="memory") + parser.add_argument("--artifact-root", type=Path) + parser.add_argument("--artifact-retention", choices=("keep",), default="keep") + parser.add_argument("--artifact-batch-size", type=int, default=8_192) + parser.add_argument("--artifact-queue-batches", type=int, default=2) + parser.add_argument("--order-stride", type=int, default=CERTIFICATION_ORDER_STRIDE) args = parser.parse_args() if args.worker is not None: if args.events is None or args.events <= 0: parser.error("worker events must be positive") - if args.worker == "dense_matching_exact_ledger" and args.events % 2: - parser.error("dense worker events must be even") - return worker(args.worker, args.events) + if args.worker != "release_no_orders" and ( + args.order_stride < 2 or args.events % args.order_stride + ): + parser.error("matching worker events must be divisible by order stride >= 2") + return worker( + args.worker, + args.events, + args.artifact_mode, + args.artifact_root, + args.artifact_retention, + args.artifact_batch_size, + args.artifact_queue_batches, + args.order_stride, + ) if args.repeat < 3: parser.error("repeat must be at least three") - if args.release_events <= 0 or args.dense_events <= 0 or args.dense_events % 2: - parser.error("event counts must be positive and dense-events must be even") - selected = [] + if ( + args.release_events <= 0 + or args.matching_events <= 0 + or args.matching_events % CERTIFICATION_ORDER_STRIDE + or args.dense_events <= 0 + or args.dense_events % DENSE_STRESS_ORDER_STRIDE + ): + parser.error("event counts must be positive and divisible by their fixed order stride") + if args.artifact_mode == "arrow" and args.artifact_root is None: + parser.error("--artifact-root is required for --artifact-mode arrow") + if args.artifact_batch_size <= 0 or args.artifact_queue_batches <= 0: + parser.error("artifact batch and queue sizes must be positive") + selected: list[tuple[str, int, int]] = [] if args.workload in {"all", "release"}: - selected.append(("release_no_orders", args.release_events)) - if args.workload in {"all", "dense"}: - selected.append(("dense_matching_exact_ledger", args.dense_events)) + selected.append(("release_no_orders", args.release_events, CERTIFICATION_ORDER_STRIDE)) + if args.workload in {"all", "matching"}: + selected.append(("matching_exact_ledger", args.matching_events, CERTIFICATION_ORDER_STRIDE)) + if args.workload == "dense": + selected.append( + ("dense_matching_exact_ledger", args.dense_events, DENSE_STRESS_ORDER_STRIDE) + ) memory_limit = int(args.memory_limit_gib * 1024**3) results = [ - aggregate(name, count, args.repeat, args.require_rate, memory_limit) - for name, count in selected + aggregate( + name, + count, + args.repeat, + args.require_rate, + memory_limit, + args.artifact_mode, + args.artifact_root, + args.artifact_retention, + args.artifact_batch_size, + args.artifact_queue_batches, + order_stride, + ) + for name, count, order_stride in selected ] encoded = json.dumps(results, indent=2, sort_keys=True) print(encoded) diff --git a/docs/performance-m7-streaming.md b/docs/performance-m7-streaming.md new file mode 100644 index 0000000..69dbd44 --- /dev/null +++ b/docs/performance-m7-streaming.md @@ -0,0 +1,80 @@ +# M7 bounded replay artifacts and performance gate + +## Outcome + +The candidate adds a strict bounded-memory Arrow artifact path without replacing the frozen +Python reference path. Correctness, compatibility and coverage gates pass. The formal +10-million-event, three-process M7 result is recorded only after running from a clean committed +candidate in the locked environment; no calibration result is promoted to release evidence. + +## Architecture and compatibility + +`DeterministicRunEngine.replay` remains the reference and still materializes `RunArtifacts`. +`replay_to_sink` is an additive migration entry point with these constraints: + +- input is a deterministic, already sorted iterator; duplicate IDs, unsorted input and invalid + event types fail closed; +- `ArrowReplayArtifactSink` writes canonical bytes to typed Arrow IPC batches using one bounded + producer queue and one writer thread; +- order events, fills and ledger transactions are hashed incrementally in logical sequence order; +- matching batches use sink transactions, so a rejected or failed multi-fill attempt cannot leak + partial artifacts; +- terminal broker state and replay ledger state are compacted while live orders, balances, + positions, marks, margin and idempotency semantics remain available; +- `StoredRunArtifacts` exposes lazy byte and JSON iterators; no consumer is forced to reconstruct + the complete Python object graph; +- incomplete runs retain `FAILED.json`; only a sealed and closed run receives a complete manifest; +- manifest publication is atomic and no-clobber, and strict reload verifies canonical manifest + bytes, manifest hash, physical file size/hash, Arrow schema, contiguous sequence and logical hash. + +The Arrow file stores a monotonic `sequence:int64` and `payload:large_binary`. The payload is the +same canonical JSON byte representation used by the frozen hashes. This is a compatibility format, +not a replacement for `standard/v2`; downstream publication still maps these facts into the shared +v2 schemas and manifest. + +## Differential and safety evidence + +The differential tests compare the new path with the current Python reference and the frozen +v0.4.1 hashes at byte level for order events, fills and ledger transactions, and at exact fixed-point +level for NAV. The matrix includes: + +| Requirement | Evidence | +|---|---| +| A-share/ETF, T+1, price limits, suspension | existing rule/ledger suites plus A-share streamed golden run and suspension path | +| Futures open/close/close-today and daily settlement | existing rule/ledger suites plus futures streamed golden run and explicit settlement artifact | +| Crypto spot/perpetual, funding and margin | crypto streamed golden run, funding/no-position funding and existing margin suites | +| Partial/multiple fills, cancellation, rejection and expiry | broker idempotency, transactional multi-fill commit/rollback, cancel, reject and DAY expiry tests | +| Latency, insufficient liquidity/margin and liquidation boundary | latency, matching, risk and liquidation tests | +| Failure atomicity and deterministic input | writer failure, invalid input, duplicate event, queue saturation and sink transaction tests | + +No extra runtime dependency was introduced: `pyarrow` was already a direct, locked dependency and +has Python3.10-3.12 wheels in the existing lock. The unchanged `replay` path is the rollback path. + +## Benchmark contract + +The certification workload emits one order every20 market events and fills it on the next eligible +event, giving an explicit5% fill density. It exercises strategy dispatch, pre-trade and runtime +risk, matching, fills, fees and exact double-entry ledger posting without pretending that every +market event produces an order. The separate dense stress workload emits one order every two +events (50% fill density) and is not relabelled as the release workload. + +Elapsed time includes event materialization, matching, risk, fill, fee, exact fixed-point +double-entry ledger, canonical serialization, Arrow initialization/write/seal, logical hashes, +ledger hash read back and manifest close. Only process startup, strict post-run reload and +construction of one static fixture template are excluded. Strict reload is nevertheless required +to pass and its duration is reported. Every repeat is a fresh process; the rate gate requires every +process, not the median, to reach50,000 events/second. Peak memory is Windows process +`PeakWorkingSetSize` and therefore includes Arrow and retained live replay state. + +The official run sets `TEMP` and `TMP` to `F:\puresaber-m7-temp` and writes to a unique directory +under `F:\puresaber-m7-artifacts`. Every Arrow stream and canonical manifest is retained; the +benchmark has no automatic deletion mode. Exact machine, dependency, commit, dirty-state, timing, +output-volume, strict-verification and per-process fields live in the committed JSON evidence. + +## Dense-stress limitation + +The bounded path removes the complete immutable Python artifact graph and materially lowers the +memory slope, but 50%-fill stress remains dominated by per-fill canonical encoding, risk/matching +dispatch and fixed-point ledger posting. Any future native/vectorized hot path must remain a +separately reviewed optimization behind the same public contracts and byte-level Python-oracle +differential suite; M7 does not weaken exact accounting or hide dense-stress results. diff --git a/pyproject.toml b/pyproject.toml index 9aecffe..9a10ee6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "quant-execution" -version = "0.4.1" +version = "0.5.0" description = "Deterministic execution, matching, risk and ledger for PureSaber quant research" readme = "README.md" license = {text = "MIT"} @@ -37,6 +37,7 @@ schemas = [ { id = "puresaber.execution.ledger-transaction", version = "1.1.0" }, { id = "puresaber.execution.account-snapshot", version = "1.1.0" }, { id = "puresaber.execution.run-result", version = "1.1.0" }, + { id = "puresaber.execution.replay-artifact-manifest", version = "1.0.0" }, ] lock-files = ["requirements.lock"] diff --git a/src/quant_execution/__init__.py b/src/quant_execution/__init__.py index 341435e..90ffbdb 100644 --- a/src/quant_execution/__init__.py +++ b/src/quant_execution/__init__.py @@ -1,5 +1,10 @@ """Deterministic execution contracts for PureSaber.""" +from quant_execution.artifacts import ( + ArrowReplayArtifactSink, + StoredRunArtifacts, + load_stored_artifacts, +) from quant_execution.broker import DeterministicBroker, remaining_quantity from quant_execution.contracts import ( AccountSnapshot, @@ -71,7 +76,7 @@ ) from quant_execution.state_machine import ALLOWED_TRANSITIONS, transition_order -__version__ = "0.4.1" +__version__ = "0.5.0" __all__ = [ "ACCOUNT_SNAPSHOT_SCHEMA_ID", @@ -90,6 +95,7 @@ "AShareRule", "AccountLedger", "AccountSnapshot", + "ArrowReplayArtifactSink", "BarMatchingModel", "BrokerSimulator", "CryptoSpotRule", @@ -127,6 +133,7 @@ "RunResult", "Settlement", "Side", + "StoredRunArtifacts", "Strategy", "StrategyContext", "TimeInForce", @@ -134,6 +141,7 @@ "execution_payload", "get_arrow_schema", "get_json_schema", + "load_stored_artifacts", "remaining_quantity", "transition_order", "validate_arrow_table", diff --git a/src/quant_execution/artifacts.py b/src/quant_execution/artifacts.py new file mode 100644 index 0000000..187c70c --- /dev/null +++ b/src/quant_execution/artifacts.py @@ -0,0 +1,664 @@ +"""Bounded-memory Arrow artifact sink for deterministic replays.""" + +from __future__ import annotations + +import hashlib +import json +import os +import queue +import re +import tempfile +import threading +import time +from collections.abc import Iterator, Mapping +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime +from decimal import Decimal +from pathlib import Path +from typing import Final + +import pyarrow as pa +from pyarrow import ipc +from quant_data_kit.exceptions import ValidationError + +from quant_execution._json import fixed_token, string_token, utc_token +from quant_execution.contracts import Fee, Fill, LedgerTransaction, Order, OrderEvent, Settlement + +_STREAMS: Final = ( + "orders", + "order_events", + "fills", + "fees", + "settlements", + "ledger_transactions", + "risk_events", +) +_SCHEMA = pa.schema( + [ + pa.field("sequence", pa.int64(), nullable=False), + pa.field("payload", pa.large_binary(), nullable=False), + ] +) +_STOP = object() +_SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +def order_event_bytes(event: OrderEvent) -> bytes: + return ( + "{" + f'"event_id":{string_token(event.event_id)},' + f'"event_time":{utc_token(event.event_time)},' + f'"fill_quantity":{fixed_token(event.fill_quantity)},' + f'"from_status":{string_token(event.from_status.value)},' + f'"order_id":{string_token(event.order_id)},' + f'"reason":{string_token(event.reason)},' + f'"sequence":{event.sequence},' + f'"to_status":{string_token(event.to_status.value)}' + "}" + ).encode() + + +def fill_bytes(fill: Fill) -> bytes: + venue_trade_id = "null" if fill.venue_trade_id is None else string_token(fill.venue_trade_id) + return ( + "{" + f'"account_id":{string_token(fill.account_id)},' + f'"event_time":{utc_token(fill.event_time)},' + f'"fill_id":{string_token(fill.fill_id)},' + f'"instrument_id":{string_token(fill.instrument_id)},' + f'"liquidity_role":{string_token(fill.liquidity_role.value)},' + f'"order_id":{string_token(fill.order_id)},' + f'"price":{fixed_token(fill.price)},' + f'"quantity":{fixed_token(fill.quantity)},' + f'"side":{string_token(fill.side.value)},' + f'"strategy_id":{string_token(fill.strategy_id)},' + f'"venue_trade_id":{venue_trade_id}' + "}" + ).encode() + + +def order_bytes(order: Order) -> bytes: + from quant_execution.broker import _intent_bytes + + return ( + "{" + f'"filled_quantity":{fixed_token(order.filled_quantity)},' + f'"intent":{_intent_bytes(order.intent).decode()},' + f'"order_id":{string_token(order.order_id)},' + f'"status":{string_token(order.status.value)},' + f'"version":{order.version}' + "}" + ).encode() + + +def fee_bytes(fee: Fee) -> bytes: + return ( + "{" + f'"account_id":{string_token(fee.account_id)},' + f'"amount":{fixed_token(fee.amount)},' + f'"currency":{string_token(fee.currency)},' + f'"event_time":{utc_token(fee.event_time)},' + f'"fee_id":{string_token(fee.fee_id)},' + f'"fee_type":{string_token(fee.fee_type)},' + f'"fill_id":{string_token(fee.fill_id)}' + "}" + ).encode() + + +def settlement_bytes(settlement: Settlement) -> bytes: + return ( + "{" + f'"account_id":{string_token(settlement.account_id)},' + f'"amount":{fixed_token(settlement.amount)},' + f'"currency":{string_token(settlement.currency)},' + f'"event_time":{utc_token(settlement.event_time)},' + f'"instrument_id":{string_token(settlement.instrument_id)},' + f'"settlement_id":{string_token(settlement.settlement_id)},' + f'"settlement_price":{fixed_token(settlement.settlement_price)},' + f'"settlement_type":{string_token(settlement.settlement_type)}' + "}" + ).encode() + + +def ledger_transaction_bytes(transaction: LedgerTransaction) -> bytes: + postings: list[str] = [] + for posting in transaction.postings: + instrument_id = ( + "null" if posting.instrument_id is None else string_token(posting.instrument_id) + ) + postings.append( + "{" + f'"amount":{fixed_token(posting.amount)},' + f'"currency":{string_token(posting.currency)},' + f'"instrument_id":{instrument_id},' + f'"ledger_account":{string_token(posting.ledger_account)},' + f'"quantity_delta":{fixed_token(posting.quantity_delta)}' + "}" + ) + return ( + "{" + f'"event_time":{utc_token(transaction.event_time)},' + f'"event_type":{string_token(transaction.event_type.value)},' + f'"idempotency_key":{string_token(transaction.idempotency_key)},' + f'"postings":[{",".join(postings)}],' + f'"reference_id":{string_token(transaction.reference_id)},' + f'"transaction_id":{string_token(transaction.transaction_id)}' + "}" + ).encode() + + +class _SequenceDigest: + """Incrementally hash a canonical JSON array without retaining its facts.""" + + def __init__(self) -> None: + self._digest = hashlib.sha256() + self._digest.update(b"[") + self._count = 0 + self._closed = False + + def append(self, payload: bytes) -> None: + if self._closed: + raise RuntimeError("artifact digest is already closed") + if self._count: + self._digest.update(b",") + self._digest.update(payload) + self._count += 1 + + def close(self) -> str: + if not self._closed: + self._digest.update(b"]") + self._closed = True + return self._digest.hexdigest() + + +@dataclass(frozen=True, slots=True) +class StoredRunArtifacts: + """Immutable handle to a completed on-disk replay artifact set.""" + + root: Path + manifest_path: Path + counts: Mapping[str, int] + logical_sha256: Mapping[str, str] + files: Mapping[str, Mapping[str, object]] + manifest_sha256: str + + def iter_payload_bytes(self, stream: str) -> Iterator[bytes]: + if stream not in _STREAMS: + raise ValidationError(f"unknown artifact stream: {stream}") + path = self.root / f"{stream}.arrow" + if not path.exists(): + return + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_stream(source) + for batch in reader: + for payload in batch.column("payload").to_pylist(): + yield bytes(payload) + + def iter_json(self, stream: str) -> Iterator[object]: + for payload in self.iter_payload_bytes(stream): + yield json.loads(payload) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_manifest_bytes(payload: Mapping[str, object]) -> bytes: + return ( + json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + b"\n" + ) + + +def _write_no_clobber(path: Path, body: bytes) -> None: + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(body) + handle.flush() + os.fsync(handle.fileno()) + os.link(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _manifest_hash(payload: Mapping[str, object]) -> str: + unsigned = dict(payload) + unsigned.pop("manifest_sha256", None) + return hashlib.sha256(_canonical_manifest_bytes(unsigned)).hexdigest() + + +def load_stored_artifacts(root: str | Path) -> StoredRunArtifacts: + """Strictly verify a completed artifact directory before exposing its facts.""" + + resolved = Path(root).resolve() + manifest_path = resolved / "manifest.json" + try: + raw = manifest_path.read_bytes() + payload = json.loads(raw) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValidationError(f"artifact manifest is unreadable: {manifest_path}") from exc + if not isinstance(payload, dict): + raise ValidationError("artifact manifest root must be an object") + expected_fields = { + "artifact_format", + "complete", + "counts", + "files", + "logical_sha256", + "manifest_sha256", + "run_metadata", + "schema_version", + } + if set(payload) != expected_fields: + raise ValidationError("artifact manifest fields changed") + if not isinstance(payload.get("run_metadata"), dict) or not payload["run_metadata"]: + raise ValidationError("artifact manifest contains no run metadata") + if payload.get("schema_version") != "1.0.0": + raise ValidationError("artifact manifest schema version is unsupported") + if payload.get("artifact_format") != "puresaber.arrow-canonical-json.v1": + raise ValidationError("artifact format is unsupported") + if payload.get("complete") is not True: + raise ValidationError("artifact run is not complete") + if raw != _canonical_manifest_bytes(payload): + raise ValidationError("artifact manifest bytes are not canonical") + manifest_sha256 = payload.get("manifest_sha256") + if not isinstance(manifest_sha256, str) or manifest_sha256 != _manifest_hash(payload): + raise ValidationError("artifact manifest hash mismatch") + counts = payload.get("counts") + logical = payload.get("logical_sha256") + files = payload.get("files") + if not isinstance(counts, dict) or set(counts) != set(_STREAMS): + raise ValidationError("artifact counts changed shape") + if not isinstance(logical, dict) or set(logical) != set(_STREAMS): + raise ValidationError("artifact logical hashes changed shape") + if not isinstance(files, dict): + raise ValidationError("artifact files must be an object") + verified_counts: dict[str, int] = {} + verified_logical: dict[str, str] = {} + verified_files: dict[str, dict[str, object]] = {} + for stream in _STREAMS: + count = counts[stream] + logical_sha256 = logical[stream] + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + raise ValidationError(f"artifact count is invalid: {stream}") + if not isinstance(logical_sha256, str) or not _SHA256.fullmatch(logical_sha256): + raise ValidationError(f"artifact logical hash is invalid: {stream}") + metadata = files.get(stream) + if count == 0: + if metadata is not None: + raise ValidationError(f"empty artifact stream unexpectedly has a file: {stream}") + digest = _SequenceDigest() + if digest.close() != logical_sha256: + raise ValidationError(f"empty artifact logical hash mismatch: {stream}") + verified_counts[stream] = 0 + verified_logical[stream] = logical_sha256 + continue + if not isinstance(metadata, dict) or set(metadata) != {"bytes", "path", "sha256"}: + raise ValidationError(f"artifact file metadata changed shape: {stream}") + relative = metadata["path"] + expected_relative = f"{stream}.arrow" + if relative != expected_relative: + raise ValidationError(f"artifact file path is invalid: {stream}") + path = (resolved / expected_relative).resolve() + try: + path.relative_to(resolved) + except ValueError as exc: + raise ValidationError(f"artifact file escapes its run root: {stream}") from exc + expected_bytes = metadata["bytes"] + expected_sha256 = metadata["sha256"] + if ( + isinstance(expected_bytes, bool) + or not isinstance(expected_bytes, int) + or expected_bytes <= 0 + or not isinstance(expected_sha256, str) + or not _SHA256.fullmatch(expected_sha256) + ): + raise ValidationError(f"artifact file metadata is invalid: {stream}") + if not path.is_file() or path.stat().st_size != expected_bytes: + raise ValidationError(f"artifact file is missing or changed size: {stream}") + if _sha256_file(path) != expected_sha256: + raise ValidationError(f"artifact file content hash mismatch: {stream}") + digest = _SequenceDigest() + observed = 0 + try: + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_stream(source) + if reader.schema != _SCHEMA: + raise ValidationError(f"artifact Arrow schema changed: {stream}") + for batch in reader: + sequences = batch.column("sequence").to_pylist() + expected_sequences = list(range(observed, observed + batch.num_rows)) + if sequences != expected_sequences: + raise ValidationError(f"artifact sequence is not contiguous: {stream}") + for item in batch.column("payload").to_pylist(): + digest.append(bytes(item)) + observed += batch.num_rows + except (OSError, pa.ArrowException) as exc: + raise ValidationError(f"artifact Arrow stream is unreadable: {stream}") from exc + if observed != count or digest.close() != logical_sha256: + raise ValidationError(f"artifact logical content mismatch: {stream}") + verified_counts[stream] = count + verified_logical[stream] = logical_sha256 + verified_files[stream] = dict(metadata) + if set(files) != set(verified_files): + raise ValidationError("artifact files contain unknown streams") + return StoredRunArtifacts( + root=resolved, + manifest_path=manifest_path, + counts=verified_counts, + logical_sha256=verified_logical, + files=verified_files, + manifest_sha256=manifest_sha256, + ) + + +class ArrowReplayArtifactSink: + """Write canonical replay facts to bounded Arrow record batches. + + The producer only retains at most ``batch_size`` payload references per stream. + A single background writer owns every Arrow stream, so replay and native Arrow + I/O can overlap without exposing partially written artifacts as complete runs. + """ + + def __init__( + self, + root: str | Path, + *, + batch_size: int = 65_536, + queue_batches: int = 8, + ) -> None: + if isinstance(batch_size, bool) or batch_size <= 0: + raise ValidationError("batch_size must be a positive integer") + if isinstance(queue_batches, bool) or queue_batches <= 0: + raise ValidationError("queue_batches must be a positive integer") + self.root = Path(root).resolve() + self.root.mkdir(parents=True, exist_ok=False) + self._batch_size = batch_size + self._buffers: dict[str, list[bytes]] = {name: [] for name in _STREAMS} + self._counts = {name: 0 for name in _STREAMS} + self._digests = {name: _SequenceDigest() for name in _STREAMS} + self._queue: queue.Queue[object] = queue.Queue(maxsize=queue_batches) + self._failure: BaseException | None = None + self._closed = False + self._sealed = False + self._staged: list[tuple[str, bytes]] | None = None + self._writers: dict[str, ipc.RecordBatchStreamWriter] = {} + self._files: dict[str, pa.NativeFile] = {} + self._thread = threading.Thread( + target=self._write_loop, + name="quant-execution-artifact-writer", + daemon=True, + ) + self._thread.start() + + @property + def counts(self) -> Mapping[str, int]: + return dict(self._counts) + + def append(self, stream: str, payload: bytes) -> None: + if self._closed or self._sealed: + raise RuntimeError("artifact sink no longer accepts records") + if stream not in self._buffers: + raise ValidationError(f"unknown artifact stream: {stream}") + if not isinstance(payload, bytes): + raise ValidationError("artifact payload must be canonical bytes") + if self._staged is not None: + self._staged.append((stream, payload)) + return + self._append_committed(stream, payload) + + def begin(self) -> None: + if self._staged is not None: + raise RuntimeError("nested artifact transactions are not supported") + if self._closed or self._sealed: + raise RuntimeError("artifact sink no longer accepts records") + self._staged = [] + + def commit(self) -> None: + staged = self._staged + if staged is None: + raise RuntimeError("no artifact transaction is active") + self._staged = None + for stream, payload in staged: + self._append_committed(stream, payload) + + def rollback(self) -> None: + if self._staged is None: + raise RuntimeError("no artifact transaction is active") + self._staged = None + + def _append_committed(self, stream: str, payload: bytes) -> None: + self._raise_writer_failure() + self._digests[stream].append(payload) + self._counts[stream] += 1 + buffer = self._buffers[stream] + buffer.append(payload) + if len(buffer) >= self._batch_size: + self._enqueue((stream, self._counts[stream] - len(buffer), buffer)) + self._buffers[stream] = [] + + def logical_sha256(self, stream: str) -> str: + if stream not in self._digests: + raise ValidationError(f"unknown artifact stream: {stream}") + return self._digests[stream].close() + + def close(self, manifest: Mapping[str, object]) -> StoredRunArtifacts: + if self._closed: + raise RuntimeError("artifact sink is already closed") + try: + self.seal() + logical = {name: digest.close() for name, digest in self._digests.items()} + files = { + name: { + "path": path.name, + "bytes": path.stat().st_size, + "sha256": _sha256_file(path), + } + for name in _STREAMS + for path in (self.root / f"{name}.arrow",) + if path.is_file() + } + completed = { + "schema_version": "1.0.0", + "artifact_format": "puresaber.arrow-canonical-json.v1", + "counts": dict(self._counts), + "logical_sha256": logical, + "files": files, + "complete": True, + "run_metadata": dict(manifest), + } + completed["manifest_sha256"] = _manifest_hash(completed) + manifest_path = self.root / "manifest.json" + _write_no_clobber(manifest_path, _canonical_manifest_bytes(completed)) + self._closed = True + return StoredRunArtifacts( + root=self.root, + manifest_path=manifest_path, + counts=dict(self._counts), + logical_sha256=logical, + files=files, + manifest_sha256=str(completed["manifest_sha256"]), + ) + except Exception: + self.abort() + raise + + def seal(self) -> None: + """Flush and close Arrow writers while leaving manifest finalization pending.""" + + if self._sealed: + return + if self._staged is not None: + raise RuntimeError("cannot seal an active artifact transaction") + for stream, buffer in self._buffers.items(): + if buffer: + self._enqueue((stream, self._counts[stream] - len(buffer), buffer)) + self._buffers[stream] = [] + self._enqueue(_STOP) + self._thread.join() + self._raise_writer_failure() + self._sealed = True + + def ledger_sha256( + self, + *, + fx_history: list[tuple[str, Decimal, datetime]], + marks: Mapping[str, tuple[Decimal, datetime, str]], + ) -> str: + """Reproduce the frozen ledger hash after bounded stream finalization.""" + + self.seal() + digest = hashlib.sha256() + digest.update(b'{"fx_snapshots":[') + for index, (currency, rate, event_time) in enumerate(fx_history): + if index: + digest.update(b",") + digest.update( + ( + "{" + f'"currency":{string_token(currency)},' + f'"event_time":{utc_token(event_time, zulu=False)},' + f'"rate":{string_token(str(rate))},' + f'"version":{index + 1}' + "}" + ).encode() + ) + digest.update(b'],"marks":[') + for index, (instrument_id, (price, event_time, event_id)) in enumerate( + sorted(marks.items()) + ): + if index: + digest.update(b",") + digest.update( + ( + "{" + f'"event_id":{string_token(event_id)},' + f'"event_time":{utc_token(event_time, zulu=False)},' + f'"instrument_id":{string_token(instrument_id)},' + f'"price":{string_token(str(price))}' + "}" + ).encode() + ) + digest.update(b'],"transactions":[') + for index, payload in enumerate(self._iter_payload_bytes("ledger_transactions")): + if index: + digest.update(b",") + digest.update(payload) + digest.update(b"]}") + return digest.hexdigest() + + def abort(self) -> None: + """Fail closed while preserving the incomplete directory for diagnosis.""" + + if self._closed: + return + for buffer in self._buffers.values(): + buffer.clear() + self._staged = None + deadline = time.monotonic() + 10 + while self._thread.is_alive() and time.monotonic() < deadline: + try: + self._queue.put(_STOP, timeout=0.1) + break + except queue.Full: + continue + self._thread.join(timeout=10) + if self._thread.is_alive(): + raise RuntimeError("artifact writer did not stop after abort") + failure = { + "artifact_format": "puresaber.arrow-canonical-json.v1", + "counts": dict(self._counts), + "complete": False, + } + try: + (self.root / "FAILED.json").write_text( + json.dumps(failure, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + finally: + self._closed = True + + def _iter_payload_bytes(self, stream: str) -> Iterator[bytes]: + path = self.root / f"{stream}.arrow" + if not path.exists(): + return + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_stream(source) + for batch in reader: + for payload in batch.column("payload").to_pylist(): + yield bytes(payload) + + def _write_loop(self) -> None: + try: + while True: + item = self._queue.get() + if item is _STOP: + break + stream, first_sequence, payloads = item + writer = self._writer(stream) + sequences = pa.array( + range(first_sequence, first_sequence + len(payloads)), type=pa.int64() + ) + values = pa.array(payloads, type=pa.large_binary()) + writer.write_batch(pa.record_batch([sequences, values], schema=_SCHEMA)) + for writer in self._writers.values(): + writer.close() + for output in self._files.values(): + output.close() + except Exception as exc: # noqa: BLE001 - thread boundary must relay every writer failure + self._failure = exc + for writer in self._writers.values(): + with suppress(Exception): + writer.close() + for output in self._files.values(): + with suppress(Exception): + output.close() + + def _writer(self, stream: str) -> ipc.RecordBatchStreamWriter: + prior = self._writers.get(stream) + if prior is not None: + return prior + output = pa.OSFile(str(self.root / f"{stream}.arrow"), "wb") + writer = ipc.new_stream(output, _SCHEMA) + self._files[stream] = output + self._writers[stream] = writer + return writer + + def _raise_writer_failure(self) -> None: + if self._failure is not None: + raise RuntimeError("artifact writer failed") from self._failure + + def _enqueue(self, item: object) -> None: + while True: + self._raise_writer_failure() + try: + self._queue.put(item, timeout=0.1) + return + except queue.Full: + continue + + +__all__ = [ + "ArrowReplayArtifactSink", + "StoredRunArtifacts", + "fee_bytes", + "fill_bytes", + "ledger_transaction_bytes", + "load_stored_artifacts", + "order_bytes", + "order_event_bytes", + "settlement_bytes", +] diff --git a/src/quant_execution/broker.py b/src/quant_execution/broker.py index 59e126f..54dc326 100644 --- a/src/quant_execution/broker.py +++ b/src/quant_execution/broker.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import json from copy import deepcopy from datetime import date, datetime @@ -10,12 +11,15 @@ from quant_data_kit.exceptions import ValidationError from quant_execution._json import fixed_token, flat_sequence_bytes, string_token +from quant_execution.artifacts import fill_bytes, order_bytes, order_event_bytes from quant_execution.contracts import ( Fill, Order, OrderEvent, OrderIntent, OrderStatus, + OrderType, + Side, TimeInForce, ) from quant_execution.state_machine import transition_order @@ -47,6 +51,54 @@ def _intent_bytes(intent: OrderIntent) -> bytes: ).encode() +def _fixed_from_payload(payload: dict[str, int] | None) -> FixedPoint | None: + return None if payload is None else FixedPoint(payload["units"], payload["scale"]) + + +def _time_from_payload(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def _order_from_bytes(payload: bytes) -> Order: + value = json.loads(payload) + raw_intent = value["intent"] + intent = OrderIntent( + idempotency_key=raw_intent["idempotency_key"], + account_id=raw_intent["account_id"], + strategy_id=raw_intent["strategy_id"], + instrument_id=raw_intent["instrument_id"], + side=Side(raw_intent["side"]), + quantity=_fixed_from_payload(raw_intent["quantity"]), + order_type=OrderType(raw_intent["order_type"]), + time_in_force=TimeInForce(raw_intent["time_in_force"]), + created_at=_time_from_payload(raw_intent["created_at"]), + limit_price=_fixed_from_payload(raw_intent["limit_price"]), + stop_price=_fixed_from_payload(raw_intent["stop_price"]), + reduce_only=raw_intent["reduce_only"], + ) + return Order( + order_id=value["order_id"], + intent=intent, + status=OrderStatus(value["status"]), + filled_quantity=_fixed_from_payload(value["filled_quantity"]), + version=value["version"], + ) + + +def _event_from_bytes(payload: bytes) -> OrderEvent: + value = json.loads(payload) + return OrderEvent( + event_id=value["event_id"], + order_id=value["order_id"], + event_time=_time_from_payload(value["event_time"]), + sequence=value["sequence"], + from_status=OrderStatus(value["from_status"]), + to_status=OrderStatus(value["to_status"]), + fill_quantity=_fixed_from_payload(value["fill_quantity"]), + reason=value["reason"], + ) + + class DeterministicBroker: """Research-only broker with idempotent submit/cancel and immutable facts.""" @@ -56,20 +108,56 @@ def __init__(self) -> None: self.reset() def reset(self) -> None: - self._orders: dict[str, Order] = {} + self._orders: dict[str, Order | bytes] = {} + self._order_count = 0 self._open_order_ids: set[str] = set() self._day_order_ids: set[str] = set() self._immediate_order_ids: set[str] = set() self._submit_keys: dict[str, tuple[str, str]] = {} self._cancel_keys: dict[str, tuple[str, OrderEvent]] = {} - self._fill_keys: dict[str, tuple[Fill, OrderEvent]] = {} + self._fill_keys: dict[str, tuple[bytes, bytes] | tuple[Fill, OrderEvent]] = {} self._events: list[OrderEvent] = [] self._accepted_day: dict[str, date] = {} + self._artifact_sink = None + + def start_artifact_stream(self, sink: object) -> None: + """Route immutable history to a bounded sink while retaining live broker state.""" + + if self._orders or self._events or self._artifact_sink is not None: + raise ValidationError("broker artifact streaming must start immediately after reset") + if not callable(getattr(sink, "append", None)): + raise ValidationError("artifact sink must provide append(stream, payload)") + self._artifact_sink = sink + + def finish_artifact_stream(self) -> None: + """Persist the final state of orders that remained open at replay completion.""" + + sink = self._artifact_sink + if sink is None: + return + for order in self.open_orders: + sink.append("orders", order_bytes(order)) + self._artifact_sink = None + + def abort_artifact_stream(self) -> None: + self._artifact_sink = None + + def _record_event(self, event: OrderEvent, order: Order) -> None: + sink = self._artifact_sink + if sink is None: + self._events.append(event) + return + sink.append("order_events", order_event_bytes(event)) + if order.status not in {OrderStatus.ACCEPTED, OrderStatus.PARTIALLY_FILLED}: + payload = order_bytes(order) + sink.append("orders", payload) + self._orders[order.order_id] = payload def capture_state(self) -> dict[str, object]: return deepcopy( { "orders": self._orders, + "order_count": self._order_count, "open_order_ids": self._open_order_ids, "day_order_ids": self._day_order_ids, "immediate_order_ids": self._immediate_order_ids, @@ -84,6 +172,7 @@ def capture_state(self) -> dict[str, object]: def restore_state(self, state: dict[str, object]) -> None: restored = deepcopy(state) self._orders = restored["orders"] + self._order_count = restored["order_count"] self._open_order_ids = restored["open_order_ids"] self._day_order_ids = restored["day_order_ids"] self._immediate_order_ids = restored["immediate_order_ids"] @@ -95,7 +184,15 @@ def restore_state(self, state: dict[str, object]) -> None: @property def orders(self) -> tuple[Order, ...]: - return tuple(sorted(self._orders.values(), key=self._sort_key)) + orders = ( + _order_from_bytes(value) if isinstance(value, bytes) else value + for value in self._orders.values() + ) + return tuple(sorted(orders, key=self._sort_key)) + + @property + def order_count(self) -> int: + return self._order_count @property def order_events(self) -> tuple[OrderEvent, ...]: @@ -107,10 +204,13 @@ def open_orders(self) -> tuple[Order, ...]: return () if len(self._open_order_ids) == 1: order_id = next(iter(self._open_order_ids)) - return (self._orders[order_id],) + value = self._orders[order_id] + if isinstance(value, bytes): + raise RuntimeError("terminal order appeared in the open-order index") + return (value,) return tuple( sorted( - (self._orders[order_id] for order_id in self._open_order_ids), + (self._live_order(order_id) for order_id in self._open_order_ids), key=self._sort_key, ) ) @@ -121,7 +221,7 @@ def immediate_orders(self) -> tuple[Order, ...]: return () return tuple( sorted( - (self._orders[order_id] for order_id in self._immediate_order_ids), + (self._live_order(order_id) for order_id in self._immediate_order_ids), key=self._sort_key, ) ) @@ -147,7 +247,7 @@ def submit(self, order_intent: OrderIntent) -> Order: order_id, prior_hash = prior if prior_hash != semantic_hash: raise ValidationError("submit idempotency key reused with different intent") - return self._orders[order_id] + return self._require_order(order_id) order_id = _digest("ord", order_intent.idempotency_key, semantic_hash) filled = FixedPoint(0, order_intent.quantity.scale) accepted = self._order_fact( @@ -164,11 +264,12 @@ def submit(self, order_intent: OrderIntent) -> Order: fill_quantity=None, ) self._orders[order_id] = accepted + self._order_count += 1 self._open_order_ids.add(order_id) if order_intent.time_in_force in {TimeInForce.IOC, TimeInForce.FOK}: self._immediate_order_ids.add(order_id) self._submit_keys[order_intent.idempotency_key] = (order_id, semantic_hash) - self._events.append(event) + self._record_event(event, accepted) return accepted def reject(self, order_intent: OrderIntent, *, code: str, message: str = "") -> Order: @@ -180,7 +281,7 @@ def reject(self, order_intent: OrderIntent, *, code: str, message: str = "") -> order_id, prior_hash = prior if prior_hash != semantic_hash: raise ValidationError("submit idempotency key reused with different intent") - return self._orders[order_id] + return self._require_order(order_id) order_id = _digest("ord", order_intent.idempotency_key, semantic_hash) order = Order(order_id=order_id, intent=order_intent) reason = code if not message else f"{code}: {message}" @@ -192,8 +293,9 @@ def reject(self, order_intent: OrderIntent, *, code: str, message: str = "") -> reason=reason, ) self._orders[order_id] = rejected + self._order_count += 1 self._submit_keys[order_intent.idempotency_key] = (order_id, semantic_hash) - self._events.append(event) + self._record_event(event, rejected) return rejected def cancel( @@ -226,12 +328,16 @@ def cancel( self._day_order_ids.discard(order_id) self._immediate_order_ids.discard(order_id) self._cancel_keys[idempotency_key] = (order_id, event) - self._events.append(event) + self._record_event(event, updated) return event - def apply_fill(self, fill: Fill) -> OrderEvent: - prior = self._fill_keys.get(fill.fill_id) + def apply_fill(self, fill: Fill, *, trusted_unique: bool = False) -> OrderEvent: + prior = None if trusted_unique else self._fill_keys.get(fill.fill_id) if prior is not None: + if isinstance(prior[0], bytes): + if prior[0] != hashlib.sha256(fill_bytes(fill)).digest(): + raise ValidationError("fill_id reused with different fill content") + return _event_from_bytes(prior[1]) prior_fill, prior_event = prior if prior_fill != fill: raise ValidationError("fill_id reused with different fill content") @@ -283,8 +389,15 @@ def apply_fill(self, fill: Fill) -> OrderEvent: self._open_order_ids.remove(order.order_id) self._day_order_ids.discard(order.order_id) self._immediate_order_ids.discard(order.order_id) - self._events.append(event) - self._fill_keys[fill.fill_id] = (fill, event) + self._record_event(event, updated) + if not trusted_unique: + if self._artifact_sink is None: + self._fill_keys[fill.fill_id] = (fill, event) + else: + self._fill_keys[fill.fill_id] = ( + hashlib.sha256(fill_bytes(fill)).digest(), + order_event_bytes(event), + ) return event def expire(self, order_id: str, *, event_time: datetime, reason: str) -> OrderEvent: @@ -302,7 +415,7 @@ def expire(self, order_id: str, *, event_time: datetime, reason: str) -> OrderEv self._open_order_ids.remove(order_id) self._day_order_ids.discard(order_id) self._immediate_order_ids.discard(order_id) - self._events.append(event) + self._record_event(event, updated) return event def note_trading_day(self, order_id: str, trading_day: date) -> None: @@ -337,9 +450,16 @@ def expire_day_orders(self, trading_day: date, event_time: datetime) -> tuple[Or def _require_order(self, order_id: str) -> Order: try: - return self._orders[order_id] + value = self._orders[order_id] except KeyError as exc: raise ValidationError(f"unknown order_id: {order_id}") from exc + return _order_from_bytes(value) if isinstance(value, bytes) else value + + def _live_order(self, order_id: str) -> Order: + value = self._orders[order_id] + if isinstance(value, bytes): + raise TypeError("terminal order appeared in a live-order index") + return value @staticmethod def _order_fact( diff --git a/src/quant_execution/engine.py b/src/quant_execution/engine.py index e4aaef3..4064b16 100644 --- a/src/quant_execution/engine.py +++ b/src/quant_execution/engine.py @@ -8,6 +8,7 @@ from copy import deepcopy from dataclasses import dataclass from datetime import datetime +from itertools import chain from typing import Any from quant_data_kit import ( @@ -25,6 +26,13 @@ from quant_data_kit.exceptions import ValidationError from quant_execution._json import fixed_token, string_token, utc_token +from quant_execution.artifacts import ( + ArrowReplayArtifactSink, + StoredRunArtifacts, + fee_bytes, + fill_bytes, + settlement_bytes, +) from quant_execution.broker import DeterministicBroker from quant_execution.contracts import ( Fee, @@ -73,6 +81,24 @@ class RunArtifacts: result: RunResult +class _SinkCollection: + """List-shaped adapter that serializes committed facts directly to a sink.""" + + __slots__ = ("_encoder", "_sink", "_stream") + + def __init__(self, sink: ArrowReplayArtifactSink, stream: str, encoder) -> None: + self._sink = sink + self._stream = stream + self._encoder = encoder + + def append(self, value: object) -> None: + self._sink.append(self._stream, self._encoder(value)) + + def extend(self, values: Iterable[object]) -> None: + for value in values: + self.append(value) + + def _canonical(value: Any) -> bytes: return json.dumps( value, @@ -182,6 +208,8 @@ def __init__( self.matching_model = matching_model self.ledger = ledger self.artifacts: RunArtifacts | None = None + self.stored_artifacts: StoredRunArtifacts | None = None + self._active_sink: ArrowReplayArtifactSink | None = None def replay(self, event_stream: Iterable[MarketEvent], seed: int) -> RunResult: if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: @@ -340,6 +368,210 @@ def replay(self, event_stream: Iterable[MarketEvent], seed: int) -> RunResult: self._restore_state(checkpoint) raise ReplayError(f"replay failed closed during finalization: {exc}") from exc + def replay_to_sink( + self, + event_stream: Iterable[MarketEvent], + seed: int, + sink: ArrowReplayArtifactSink, + ) -> RunResult: + """Replay a pre-sorted event stream into bounded Arrow artifacts. + + This opt-in migration path preserves ``replay`` and ``RunArtifacts`` while avoiding + complete in-memory retention. The input must already use the public deterministic sort + order; accepting and sorting an unbounded stream would violate the memory guarantee. + """ + + if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: + raise ValidationError("seed must be a non-negative integer") + if not isinstance(sink, ArrowReplayArtifactSink): + raise ValidationError("sink must be an ArrowReplayArtifactSink") + checkpoint = self._capture_state() + event: MarketEvent | None = None + event_count = 0 + seen_event_ids: set[str] = set() + prior_sort_key: tuple[object, ...] | None = None + iterator = iter(event_stream) + try: + first = next(iterator, None) + if first is not None and not isinstance(first, _EVENT_TYPES): + raise ValidationError("event_stream contains a non-MarketEvent value") + self._reset(opened_at=first.available_at if first is not None else None) + self.broker.start_artifact_stream(sink) + self.ledger.start_artifact_stream(sink) + self._active_sink = sink + context = StrategyContext( + run_id=self.run_id, + account_id=self.account_id, + strategy_id=self.strategy_id, + seed=seed, + state={}, + ) + fills = _SinkCollection(sink, "fills", fill_bytes) + fees = _SinkCollection(sink, "fees", fee_bytes) + settlements = _SinkCollection(sink, "settlements", settlement_bytes) + risk_events = _SinkCollection( + sink, "risk_events", lambda value: string_token(str(value)).encode() + ) + seen_fill_ids: set[str] = set() + records = () if first is None else chain((first,), iterator) + for value in records: + if not isinstance(value, _EVENT_TYPES): + raise ValidationError("event_stream contains a non-MarketEvent value") + event = value + sort_key = _event_sort_key(event) + if prior_sort_key is not None and sort_key < prior_sort_key: + raise ValidationError("streaming event_stream is not deterministically sorted") + if event.event_id in seen_event_ids: + raise ValidationError(f"duplicate MarketEvent event_id: {event.event_id}") + seen_event_ids.add(event.event_id) + prior_sort_key = sort_key + event_count += 1 + + day_expiries = self.broker.expire_day_orders(event.trading_day, event.available_at) + for expiry in day_expiries: + self.risk_gate.release_order(self._order(expiry.order_id)) + self.risk_gate.observe(event) + account_snapshot = self.ledger.observe_market( + event, + create_snapshot=False, + trusted_unique=True, + ) + if isinstance(event, CorporateActionEvent): + account_snapshot = self.ledger.apply(event, create_snapshot=False) + elif isinstance(event, FundingRateEvent): + funding = self.ledger.funding_from_market(event) + if funding is not None: + account_snapshot = self.ledger.apply(funding, create_snapshot=False) + elif isinstance(event, StatusEvent): + settlement = self.ledger.settlement_from_market(event) + if settlement is not None: + account_snapshot = self.ledger.apply(settlement, create_snapshot=False) + settlements.append(settlement) + + if self.broker.open_orders: + for order in self.broker.open_orders: + if ( + type(self.risk_gate).check_open_order + is RuleBookRiskGate.check_open_order + ): + decision = self.risk_gate.check_open_order_current( + order, event_time=event.available_at + ) + else: + account_snapshot = account_snapshot or self.ledger.snapshot( + event.available_at + ) + decision = self.risk_gate.check_open_order( + order, + account_snapshot, + event_time=event.available_at, + ) + if not decision.accepted: + self.broker.expire( + order.order_id, + event_time=event.available_at, + reason=f"{decision.code}: {decision.message}", + ) + self.risk_gate.release_order(order) + risk_events.append( + f"{order.order_id}:{decision.code}:{decision.message}" + ) + + if self._match_and_commit( + event, + fills=fills, + fees=fees, + risk_events=risk_events, + seen_fill_ids=seen_fill_ids, + ): + account_snapshot = None + + self._expire_immediate_orders(event) + if type(self.risk_gate).runtime_check is RuleBookRiskGate.runtime_check: + runtime = self.risk_gate.runtime_check_current(event.available_at) + else: + account_snapshot = account_snapshot or self.ledger.snapshot(event.available_at) + runtime = self.risk_gate.runtime_check(account_snapshot) + if not runtime.accepted: + risk_events.append(f"{event.event_id}:{runtime.code}:{runtime.message}") + for order in self.broker.open_orders: + self.broker.expire( + order.order_id, + event_time=event.available_at, + reason=runtime.code, + ) + self.risk_gate.release_order(order) + + intents = self._strategy_intents(context, event) + for intent in intents: + if type(self.risk_gate).check is RuleBookRiskGate.check: + decision, reservation = self.risk_gate._check_current_for_submit( + intent, event_time=event.available_at + ) + else: + reservation = None + account_snapshot = account_snapshot or self.ledger.snapshot( + event.available_at + ) + decision = self.risk_gate.check(intent, account_snapshot) + if decision.accepted: + order = self.broker.submit(intent) + self.broker.note_trading_day(order.order_id, event.trading_day) + if order.status in { + OrderStatus.ACCEPTED, + OrderStatus.PARTIALLY_FILLED, + }: + if reservation is None: + self.risk_gate.reserve(intent) + else: + self.risk_gate._reserve_requirement(intent, reservation) + else: + self.broker.reject(intent, code=decision.code, message=decision.message) + risk_events.append( + f"{intent.idempotency_key}:{decision.code}:{decision.message}" + ) + + self.broker.finish_artifact_stream() + self.ledger.finish_artifact_stream() + self._active_sink = None + sink.seal() + result = RunResult( + run_id=self.run_id, + seed=seed, + event_count=event_count, + order_count=self.broker.order_count, + fill_count=sink.counts["fills"], + event_sha256=sink.logical_sha256("order_events"), + fill_sha256=sink.logical_sha256("fills"), + ledger_sha256=sink.ledger_sha256( + fx_history=self.ledger._fx_history, + marks=self.ledger._marks, + ), + ) + self.stored_artifacts = sink.close( + { + "run_id": self.run_id, + "seed": seed, + "event_count": event_count, + "order_count": result.order_count, + "fill_count": result.fill_count, + "order_sha256": result.order_sha256, + "fill_sha256": result.fill_sha256, + "ledger_sha256": result.ledger_sha256, + "result_sha256": result.result_sha256, + } + ) + self.artifacts = None + return result + except Exception as exc: + self._active_sink = None + self.broker.abort_artifact_stream() + self.ledger.abort_artifact_stream() + sink.abort() + self._restore_state(checkpoint) + event_id = event.event_id if event is not None else "before-first-event" + raise ReplayError(f"streaming replay failed closed at {event_id}: {exc}") from exc + def _reset(self, *, opened_at: datetime | None = None) -> None: self.broker.reset() self.ledger.reset(opened_at=opened_at) @@ -351,6 +583,7 @@ def _reset(self, *, opened_at: datetime | None = None) -> None: if callable(strategy_reset): strategy_reset() self.artifacts = None + self.stored_artifacts = None def _match_and_commit( self, @@ -413,11 +646,7 @@ def _match_and_commit( fills.append(fill) return True - broker_checkpoint = self._capture_component(self.broker) - ledger_checkpoint = self._capture_component(self.ledger) risk_checkpoint = self._capture_component(self.risk_gate) - staged_fills: list[Fill] = [] - staged_fees: list[Fee] = [] rejected: dict[str, tuple[str, str]] = {} for fill in matched: order = self._order(fill.order_id) @@ -426,17 +655,9 @@ def _match_and_commit( decision = self.risk_gate.check_fill(fill, order) if not decision.accepted: rejected[order.order_id] = (decision.code, decision.message) - continue - fee = self._commit_fill(fill, order, event) - staged_fills.append(fill) - if fee is not None: - staged_fees.append(fee) - if rejected: if matching_checkpoint is not None: self._restore_component(self.matching_model, matching_checkpoint) - self._restore_component(self.broker, broker_checkpoint) - self._restore_component(self.ledger, ledger_checkpoint) self._restore_component(self.risk_gate, risk_checkpoint) self._expire_fill_rejections( event, @@ -448,6 +669,25 @@ def _match_and_commit( ) continue + staged_fills: list[Fill] = [] + staged_fees: list[Fee] = [] + sink = self._active_sink + if sink is not None: + sink.begin() + try: + for fill in matched: + order = self._order(fill.order_id) + fee = self._commit_fill(fill, order, event) + staged_fills.append(fill) + if fee is not None: + staged_fees.append(fee) + except Exception: + if sink is not None: + sink.rollback() + raise + + if sink is not None: + sink.commit() fills.extend(staged_fills) fees.extend(staged_fees) seen_fill_ids.update(fill.fill_id for fill in staged_fills) @@ -462,7 +702,10 @@ def _validate_candidate_fill_ids(matched: Sequence[Fill], seen_fill_ids: set[str attempt_fill_ids.add(fill.fill_id) def _commit_fill(self, fill: Fill, order: Order, event: MarketEvent) -> Fee | None: - self.broker.apply_fill(fill) + if type(self.broker) is DeterministicBroker and self._active_sink is not None: + self.broker.apply_fill(fill, trusted_unique=True) + else: + self.broker.apply_fill(fill) self.risk_gate.release_fill(fill, order) if type(self.ledger) is ExactAccountLedger: self.ledger._apply_replay_event(fill, trading_day=event.trading_day) diff --git a/src/quant_execution/ledger.py b/src/quant_execution/ledger.py index 6d49296..5516553 100644 --- a/src/quant_execution/ledger.py +++ b/src/quant_execution/ledger.py @@ -24,11 +24,18 @@ StatusEvent, TradeEvent, ensure_utc_datetime, + market_event_payload, ) from quant_data_kit.exceptions import ValidationError from quant_execution._fixed import decimal, fixed from quant_execution._json import fixed_token, flat_sequence_bytes, string_token, utc_token +from quant_execution.artifacts import ( + fee_bytes, + fill_bytes, + ledger_transaction_bytes, + settlement_bytes, +) from quant_execution.contracts import ( AccountSnapshot, Fee, @@ -44,6 +51,7 @@ Side, _currency, ) +from quant_execution.schemas import execution_payload UTC = timezone.utc _OPENED_AT = datetime(1970, 1, 1, tzinfo=UTC) @@ -121,8 +129,10 @@ def reset(self, *, opened_at: datetime | None = None) -> None: else self._default_opened_at ) self._transactions: list[LedgerTransaction] = [] + self._transaction_count = 0 + self._artifact_sink = None self._transaction_keys: set[str] = set() - self._event_fingerprints: dict[str, LedgerEvent] = {} + self._event_fingerprints: dict[str, LedgerEvent | bytes] = {} self._fills: dict[str, Fill] = {} self._accounts: dict[tuple[str, str, str | None], Decimal] = {} self._positions: dict[str, Decimal] = {} @@ -157,10 +167,30 @@ def reset(self, *, opened_at: datetime | None = None) -> None: ) self._post(transaction) + def start_artifact_stream(self, sink: object) -> None: + """Move journal retention to a bounded artifact sink after reset.""" + + if self._artifact_sink is not None: + raise ValidationError("ledger artifact stream is already active") + if not callable(getattr(sink, "append", None)): + raise ValidationError("artifact sink must provide append(stream, payload)") + self._artifact_sink = sink + for transaction in self._transactions: + sink.append("ledger_transactions", ledger_transaction_bytes(transaction)) + self._transaction_count = len(self._transactions) + self._transactions.clear() + + def finish_artifact_stream(self) -> None: + self._artifact_sink = None + + def abort_artifact_stream(self) -> None: + self._artifact_sink = None + def capture_state(self) -> dict[str, object]: return deepcopy( { "transactions": self._transactions, + "transaction_count": self._transaction_count, "transaction_keys": self._transaction_keys, "event_fingerprints": self._event_fingerprints, "fills": self._fills, @@ -181,6 +211,7 @@ def capture_state(self) -> dict[str, object]: def restore_state(self, state: dict[str, object]) -> None: restored = deepcopy(state) self._transactions = restored["transactions"] + self._transaction_count = restored["transaction_count"] self._transaction_keys = restored["transaction_keys"] self._event_fingerprints = restored["event_fingerprints"] self._fills = restored["fills"] @@ -200,6 +231,12 @@ def restore_state(self, state: dict[str, object]) -> None: def transactions(self) -> tuple[LedgerTransaction, ...]: return tuple(self._transactions) + @property + def transaction_count(self) -> int: + return ( + self._transaction_count if self._artifact_sink is not None else len(self._transactions) + ) + @property def journal_sha256(self) -> str: digest = hashlib.sha256() @@ -572,12 +609,16 @@ def _apply( trading_day: date | None, create_snapshot: bool, local_rollback: bool = True, + trusted_unique: bool = False, ) -> AccountSnapshot | None: self._validate_event(event) reference_id = self._event_identity(event) - prior = self._event_fingerprints.get(reference_id) + prior = None if trusted_unique else self._event_fingerprints.get(reference_id) if prior is not None: - if prior != event: + if isinstance(prior, bytes): + if prior != self._event_fingerprint(event): + raise ValidationError("ledger event id reused with different content") + elif prior != event: raise ValidationError("ledger event id reused with different content") if isinstance(event, Fill) and self._fill_trading_days[event.fill_id] != trading_day: raise ValidationError("fill trading_day changed across idempotent application") @@ -623,8 +664,17 @@ def _apply( ) elif isinstance(event, CorporateActionEvent) and event.ratio is not None: self._apply_split_state(event) - self._event_fingerprints[reference_id] = event + if not trusted_unique: + self._event_fingerprints[reference_id] = ( + self._event_fingerprint(event) if self._artifact_sink is not None else event + ) self._event_time = transaction.event_time + if isinstance(event, Fee) and self._artifact_sink is not None: + fill = self._fills.pop(event.fill_id, None) + if fill is not None: + spec = self._spec(fill.instrument_id) + if spec.asset_class is not AssetClass.FUTURE: + self._fill_close_allocations.pop(event.fill_id, None) return self.snapshot(transaction.event_time) if create_snapshot else None except Exception: if transaction is not None and undo is not None: @@ -769,6 +819,7 @@ def _apply_replay_event( trading_day=trading_day, create_snapshot=False, local_rollback=False, + trusted_unique=True, ) def _validate_event(self, event: LedgerEvent) -> None: @@ -1327,12 +1378,19 @@ def _posting( prior = self._posting_cache.get(key) if prior is not None: return prior - posting = Posting( - ledger_account=account, - currency=currency, - amount=fixed(amount, self.money_scale, rounding=ROUND_HALF_EVEN), - instrument_id=instrument_id, - quantity_delta=( + posting = object.__new__(Posting) + object.__setattr__(posting, "ledger_account", account) + object.__setattr__(posting, "currency", currency) + object.__setattr__( + posting, + "amount", + fixed(amount, self.money_scale, rounding=ROUND_HALF_EVEN), + ) + object.__setattr__(posting, "instrument_id", instrument_id) + object.__setattr__( + posting, + "quantity_delta", + ( fixed(quantity_delta, quantity_scale, rounding=ROUND_HALF_EVEN) if quantity_delta is not None else None @@ -1350,17 +1408,22 @@ def _make_transaction( event_time: datetime, postings: tuple[Posting, ...], ) -> LedgerTransaction: - return LedgerTransaction( - transaction_id=_identifier("tx", self.account_id, event_type.value, reference_id), - idempotency_key=idempotency_key, - event_time=event_time, - event_type=event_type, - reference_id=reference_id, - postings=postings, + transaction = object.__new__(LedgerTransaction) + object.__setattr__( + transaction, + "transaction_id", + _identifier("tx", self.account_id, event_type.value, reference_id), ) + object.__setattr__(transaction, "idempotency_key", idempotency_key) + object.__setattr__(transaction, "event_time", event_time) + object.__setattr__(transaction, "event_type", event_type) + object.__setattr__(transaction, "reference_id", reference_id) + object.__setattr__(transaction, "postings", postings) + return transaction def _post(self, transaction: LedgerTransaction, *, local_rollback: bool = True) -> None: - if transaction.idempotency_key in self._transaction_keys: + streaming = self._artifact_sink is not None + if not streaming and transaction.idempotency_key in self._transaction_keys: raise ValidationError("duplicate ledger transaction idempotency key") if not local_rollback: for posting in transaction.postings: @@ -1375,8 +1438,15 @@ def _post(self, transaction: LedgerTransaction, *, local_rollback: bool = True) self._positions[instrument_id] = self._positions.get( instrument_id, Decimal(0) ) + decimal(posting.quantity_delta) - self._transactions.append(transaction) - self._transaction_keys.add(transaction.idempotency_key) + if self._artifact_sink is None: + self._transactions.append(transaction) + else: + self._artifact_sink.append( + "ledger_transactions", ledger_transaction_bytes(transaction) + ) + self._transaction_count += 1 + if not streaming: + self._transaction_keys.add(transaction.idempotency_key) return missing = object() prior_accounts: dict[tuple[str, str, str | None], Decimal | object] = {} @@ -1398,8 +1468,15 @@ def _post(self, transaction: LedgerTransaction, *, local_rollback: bool = True) self._positions[instrument_id] = self._positions.get( instrument_id, Decimal(0) ) + decimal(posting.quantity_delta) - self._transactions.append(transaction) - self._transaction_keys.add(transaction.idempotency_key) + if self._artifact_sink is None: + self._transactions.append(transaction) + else: + self._artifact_sink.append( + "ledger_transactions", ledger_transaction_bytes(transaction) + ) + self._transaction_count += 1 + if not streaming: + self._transaction_keys.add(transaction.idempotency_key) except Exception: for key, value in prior_accounts.items(): if value is missing: @@ -1429,6 +1506,20 @@ def _event_identity(self, event: LedgerEvent) -> str: raise ValidationError("ledger event has no stable identity") return f"{identity_field}:{getattr(event, identity_field)}" + @staticmethod + def _event_fingerprint(event: LedgerEvent) -> bytes: + if isinstance(event, Fill): + payload = fill_bytes(event) + elif isinstance(event, Fee): + payload = fee_bytes(event) + elif isinstance(event, Settlement): + payload = settlement_bytes(event) + elif isinstance(event, CorporateActionEvent): + payload = _canonical(market_event_payload(event)) + else: + payload = _canonical(execution_payload(event)) + return hashlib.sha256(payload).digest() + def _spec(self, instrument_id: str) -> InstrumentSpec: try: return self.instruments[instrument_id] diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py new file mode 100644 index 0000000..d9b21f3 --- /dev/null +++ b/tests/test_artifacts.py @@ -0,0 +1,815 @@ +from __future__ import annotations + +import json +import queue +from datetime import date, timedelta + +import pytest +from conftest import T0, fp +from quant_data_kit import CorporateActionEvent, FundingRateEvent, StatusEvent +from quant_data_kit.exceptions import ValidationError +from test_engine import ( + FixtureStrategy, + Signal, + bar, + engine_for, + scenario_a_share, + scenario_crypto, + scenario_future, +) +from test_engine_edges import ( + LiquidatingGate, + TransactionalBatchMatcher, + spot_signal, + transactional_engine, +) +from test_rules import FUTURE, PERP, SPOT, STOCK, specs + +from quant_execution import ( + ArrowReplayArtifactSink, + Fill, + LiquidityRole, + OrderIntent, + OrderType, + ReplayError, + RiskDecision, + Side, + TimeInForce, + load_stored_artifacts, +) +from quant_execution.artifacts import ( + _canonical_manifest_bytes, + _manifest_hash, + fee_bytes, + fill_bytes, + ledger_transaction_bytes, + order_bytes, + order_event_bytes, + settlement_bytes, +) +from quant_execution.broker import DeterministicBroker +from quant_execution.engine import _event_sort_key +from quant_execution.ledger import ExactAccountLedger +from quant_execution.matching import BarMatchingModel +from quant_execution.rules import RuleBookRiskGate + + +@pytest.mark.parametrize( + "factory", + (scenario_a_share, scenario_future, scenario_crypto), + ids=("a-share", "future", "crypto-spot-perpetual-funding"), +) +def test_streamed_replay_is_byte_identical_to_memory_reference(factory, tmp_path) -> None: + reference, reference_events = factory() + expected = reference.replay(reference_events, 42) + assert reference.artifacts is not None + expected_artifacts = reference.artifacts + expected_nav = reference.ledger.snapshot().nav + + streamed, streamed_events = factory() + sink = ArrowReplayArtifactSink(tmp_path / "run", batch_size=2, queue_batches=1) + actual = streamed.replay_to_sink( + sorted(streamed_events, key=_event_sort_key), + 42, + sink, + ) + stored = streamed.stored_artifacts + assert stored is not None + assert actual == expected + assert streamed.ledger.snapshot().nav == expected_nav + assert tuple(stored.iter_payload_bytes("orders")) == tuple( + order_bytes(value) for value in expected_artifacts.orders + ) + assert tuple(stored.iter_payload_bytes("order_events")) == tuple( + order_event_bytes(value) for value in expected_artifacts.order_events + ) + assert tuple(stored.iter_payload_bytes("fills")) == tuple( + fill_bytes(value) for value in expected_artifacts.fills + ) + assert tuple(stored.iter_payload_bytes("fees")) == tuple( + fee_bytes(value) for value in expected_artifacts.fees + ) + assert tuple(stored.iter_payload_bytes("settlements")) == tuple( + settlement_bytes(value) for value in expected_artifacts.settlements + ) + assert tuple(stored.iter_payload_bytes("ledger_transactions")) == tuple( + ledger_transaction_bytes(value) for value in expected_artifacts.ledger_transactions + ) + assert tuple(stored.iter_json("risk_events")) == expected_artifacts.risk_events + verified = load_stored_artifacts(stored.root) + assert verified.manifest_sha256 == stored.manifest_sha256 + manifest = json.loads(stored.manifest_path.read_text(encoding="utf-8")) + assert manifest["complete"] is True + assert manifest["run_metadata"]["result_sha256"] == expected.result_sha256 + + +def test_streaming_broker_terminal_order_remains_idempotently_readable(tmp_path) -> None: + broker = DeterministicBroker() + sink = ArrowReplayArtifactSink(tmp_path / "broker", batch_size=1) + broker.start_artifact_stream(sink) + intent = OrderIntent( + idempotency_key="cancel-me", + account_id="account", + strategy_id="strategy", + instrument_id="crypto:test:BTCUSDT", + side=Side.BUY, + quantity=fp("1.000", 3), + order_type=OrderType.LIMIT, + time_in_force=TimeInForce.GTC, + created_at=T0, + limit_price=fp("100"), + ) + accepted = broker.submit(intent) + cancelled = broker.cancel( + accepted.order_id, + idempotency_key="cancel-request", + created_at=T0 + timedelta(seconds=1), + ) + assert broker.submit(intent).order_id == accepted.order_id + assert ( + broker.cancel( + accepted.order_id, + idempotency_key="cancel-request", + created_at=T0 + timedelta(seconds=1), + ) + == cancelled + ) + assert broker.get_order(accepted.order_id).status.value == "cancelled" + broker.finish_artifact_stream() + stored = sink.close({"run_id": "broker-only"}) + assert stored.counts["orders"] == 1 + assert stored.counts["order_events"] == 2 + + +def test_streaming_broker_fill_compaction_and_lifecycle_guards(tmp_path) -> None: + broker = DeterministicBroker() + sink = ArrowReplayArtifactSink(tmp_path / "broker-fill", batch_size=1) + with pytest.raises(ValidationError, match="provide append"): + broker.start_artifact_stream(object()) + broker.start_artifact_stream(sink) + with pytest.raises(ValidationError, match="immediately after reset"): + broker.start_artifact_stream(sink) + intent = OrderIntent( + idempotency_key="fill-me", + account_id="account", + strategy_id="strategy", + instrument_id="crypto:test:BTCUSDT", + side=Side.BUY, + quantity=fp("1.000", 3), + order_type=OrderType.LIMIT, + time_in_force=TimeInForce.GTC, + created_at=T0, + limit_price=fp("100"), + ) + order = broker.submit(intent) + fill = Fill( + fill_id="fill-compact", + order_id=order.order_id, + account_id="account", + strategy_id="strategy", + instrument_id=intent.instrument_id, + side=Side.BUY, + quantity=intent.quantity, + price=fp("100"), + event_time=T0 + timedelta(seconds=1), + liquidity_role=LiquidityRole.TAKER, + ) + event = broker.apply_fill(fill) + assert broker.apply_fill(fill) == event + conflicting = Fill( + fill_id=fill.fill_id, + order_id=fill.order_id, + account_id=fill.account_id, + strategy_id=fill.strategy_id, + instrument_id=fill.instrument_id, + side=fill.side, + quantity=fill.quantity, + price=fp("101"), + event_time=fill.event_time, + liquidity_role=fill.liquidity_role, + ) + with pytest.raises(ValidationError, match="different fill content"): + broker.apply_fill(conflicting) + terminal_payload = broker._orders[order.order_id] + assert isinstance(terminal_payload, bytes) + broker._open_order_ids.add(order.order_id) + with pytest.raises(TypeError, match="terminal order"): + broker._live_order(order.order_id) + broker._open_order_ids.clear() + broker.finish_artifact_stream() + broker.finish_artifact_stream() + sink.close({"run_id": "broker-fill"}) + + +def test_streaming_replay_fails_closed_for_unsorted_and_duplicate_inputs(tmp_path) -> None: + engine, events = scenario_a_share() + with pytest.raises(ReplayError, match="not deterministically sorted"): + engine.replay_to_sink(events, 42, ArrowReplayArtifactSink(tmp_path / "unsorted")) + assert json.loads((tmp_path / "unsorted" / "FAILED.json").read_text())["complete"] is False + + engine, events = scenario_a_share() + ordered = sorted(events, key=_event_sort_key) + with pytest.raises(ReplayError, match="duplicate MarketEvent event_id"): + engine.replay_to_sink( + (ordered[0], ordered[0]), + 42, + ArrowReplayArtifactSink(tmp_path / "duplicate"), + ) + + +def test_sink_validation_transaction_and_unknown_stream_branches(tmp_path) -> None: + with pytest.raises(ValidationError, match="batch_size"): + ArrowReplayArtifactSink(tmp_path / "bad-batch", batch_size=0) + with pytest.raises(ValidationError, match="queue_batches"): + ArrowReplayArtifactSink(tmp_path / "bad-queue", queue_batches=0) + + sink = ArrowReplayArtifactSink(tmp_path / "transactions", batch_size=1) + with pytest.raises(ValidationError, match="unknown artifact stream"): + sink.append("unknown", b"{}") + with pytest.raises(ValidationError, match="canonical bytes"): + sink.append("fills", "not-bytes") + sink.begin() + sink.append("risk_events", b'"rolled-back"') + with pytest.raises(RuntimeError, match="nested"): + sink.begin() + sink.rollback() + with pytest.raises(RuntimeError, match="no artifact transaction"): + sink.rollback() + sink.begin() + sink.append("risk_events", b'"committed"') + sink.commit() + stored = sink.close({"run_id": "transaction-test"}) + assert tuple(stored.iter_json("risk_events")) == ("committed",) + assert tuple(stored.iter_json("settlements")) == () + with pytest.raises(ValidationError, match="unknown artifact stream"): + tuple(stored.iter_json("unknown")) + with pytest.raises(RuntimeError, match="already closed"): + sink.close({}) + + +def test_streaming_input_validation_empty_and_sink_failure_branches(tmp_path, monkeypatch) -> None: + for index, seed in enumerate((True, "1", -1)): + engine, events = scenario_a_share() + sink = ArrowReplayArtifactSink(tmp_path / f"seed-{index}") + with pytest.raises(ValidationError, match="seed"): + engine.replay_to_sink(events, seed, sink) + sink.abort() + engine, events = scenario_a_share() + with pytest.raises(ValidationError, match="ArrowReplayArtifactSink"): + engine.replay_to_sink(events, 1, object()) + + engine, _ = scenario_a_share() + result = engine.replay_to_sink((), 1, ArrowReplayArtifactSink(tmp_path / "empty")) + assert result.event_count == result.order_count == result.fill_count == 0 + + engine, _ = scenario_a_share() + with pytest.raises(ReplayError, match="non-MarketEvent"): + engine.replay_to_sink((object(),), 1, ArrowReplayArtifactSink(tmp_path / "bad-first")) + engine, events = scenario_a_share() + ordered = sorted(events, key=_event_sort_key) + with pytest.raises(ReplayError, match="non-MarketEvent"): + engine.replay_to_sink( + (ordered[0], object()), + 1, + ArrowReplayArtifactSink(tmp_path / "bad-later"), + ) + + failing = ArrowReplayArtifactSink(tmp_path / "writer-failure", batch_size=1) + + def fail_writer(stream): + del stream + raise OSError("injected Arrow writer failure") + + monkeypatch.setattr(failing, "_writer", fail_writer) + failing.append("fills", b"{}") + with pytest.raises(RuntimeError, match="artifact writer failed"): + failing.seal() + failing.abort() + + +def test_sink_defensive_state_and_queue_full_branches(tmp_path, monkeypatch) -> None: + from quant_execution.artifacts import _SequenceDigest + + digest = _SequenceDigest() + digest.append(b"{}") + digest.close() + with pytest.raises(RuntimeError, match="already closed"): + digest.append(b"{}") + + sink = ArrowReplayArtifactSink(tmp_path / "states") + with pytest.raises(RuntimeError, match="no artifact transaction"): + sink.commit() + sink.begin() + with pytest.raises(RuntimeError, match="active artifact transaction"): + sink.seal() + sink.rollback() + with pytest.raises(ValidationError, match="unknown artifact stream"): + sink.logical_sha256("unknown") + assert tuple(sink._iter_payload_bytes("fills")) == () + + original_put = sink._queue.put + calls = 0 + + def full_once(item, timeout=None): + nonlocal calls + calls += 1 + if calls == 1: + raise queue.Full + return original_put(item, timeout=timeout) + + monkeypatch.setattr(sink._queue, "put", full_once) + sink.append("fills", b"{}") + stored = sink.close({"run_id": "queue-full-once"}) + assert stored.counts["fills"] == 1 + with pytest.raises(RuntimeError, match="no longer accepts"): + sink.append("fills", b"{}") + with pytest.raises(RuntimeError, match="no longer accepts"): + sink.begin() + sink.abort() + + invalid_manifest = ArrowReplayArtifactSink(tmp_path / "invalid-manifest") + with pytest.raises(TypeError): + invalid_manifest.close({"not_json": object()}) + assert (tmp_path / "invalid-manifest" / "FAILED.json").exists() + + +def test_stored_artifact_loader_rejects_physical_and_manifest_tampering(tmp_path) -> None: + sink = ArrowReplayArtifactSink(tmp_path / "physical", batch_size=1) + sink.append("fills", b"{}") + stored = sink.close({"run_id": "physical"}) + assert load_stored_artifacts(stored.root).counts["fills"] == 1 + fill_path = stored.root / "fills.arrow" + fill_path.write_bytes(fill_path.read_bytes() + b"tampered") + with pytest.raises(ValidationError, match="missing or changed size"): + load_stored_artifacts(stored.root) + + sink = ArrowReplayArtifactSink(tmp_path / "manifest", batch_size=1) + sink.append("fills", b"{}") + stored = sink.close({"run_id": "manifest"}) + payload = json.loads(stored.manifest_path.read_text(encoding="utf-8")) + stored.manifest_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + with pytest.raises(ValidationError, match="not canonical"): + load_stored_artifacts(stored.root) + + +def test_stored_artifact_loader_rejects_manifest_contract_mutations(tmp_path) -> None: + def artifact(case: str, *, with_fill: bool = False): + sink = ArrowReplayArtifactSink(tmp_path / case, batch_size=1) + if with_fill: + sink.append("fills", b"{}") + return sink.close({"run_id": case}) + + def rewrite(stored, payload, *, rehash: bool = True) -> None: + if rehash: + payload["manifest_sha256"] = _manifest_hash(payload) + stored.manifest_path.write_bytes(_canonical_manifest_bytes(payload)) + + unreadable = artifact("unreadable") + unreadable.manifest_path.write_bytes(b"{") + with pytest.raises(ValidationError, match="manifest is unreadable"): + load_stored_artifacts(unreadable.root) + + non_object = artifact("non-object") + non_object.manifest_path.write_bytes(b"[]\n") + with pytest.raises(ValidationError, match="root must be an object"): + load_stored_artifacts(non_object.root) + + top_level_cases = ( + ("fields", "unexpected", True, "fields changed"), + ("metadata", "run_metadata", {}, "no run metadata"), + ("schema", "schema_version", "2.0.0", "schema version"), + ("format", "artifact_format", "unknown", "format is unsupported"), + ("incomplete", "complete", False, "run is not complete"), + ("counts-shape", "counts", {}, "counts changed shape"), + ("logical-shape", "logical_sha256", {}, "logical hashes changed shape"), + ("files-type", "files", [], "files must be an object"), + ) + for case, field, value, message in top_level_cases: + stored = artifact(case) + payload = json.loads(stored.manifest_path.read_text(encoding="utf-8")) + payload[field] = value + rewrite(stored, payload) + with pytest.raises(ValidationError, match=message): + load_stored_artifacts(stored.root) + + invalid_hash = artifact("invalid-hash") + payload = json.loads(invalid_hash.manifest_path.read_text(encoding="utf-8")) + payload["manifest_sha256"] = "not-a-sha" + rewrite(invalid_hash, payload, rehash=False) + with pytest.raises(ValidationError, match="manifest hash mismatch"): + load_stored_artifacts(invalid_hash.root) + + empty_stream_cases = ( + ("count-bool", "counts", True, "count is invalid"), + ("count-negative", "counts", -1, "count is invalid"), + ("logical-format", "logical_sha256", "bad", "logical hash is invalid"), + ("unexpected-file", "files", {"path": "fees.arrow"}, "unexpectedly has a file"), + ("empty-hash", "logical_sha256", "0" * 64, "empty artifact logical hash mismatch"), + ) + for case, section, value, message in empty_stream_cases: + stored = artifact(case) + payload = json.loads(stored.manifest_path.read_text(encoding="utf-8")) + payload[section]["fees"] = value + rewrite(stored, payload) + with pytest.raises(ValidationError, match=message): + load_stored_artifacts(stored.root) + + file_cases = ( + ("file-shape", "files", {"path": "fills.arrow"}, "metadata changed shape"), + ( + "file-path", + "files", + {"bytes": 1, "path": "wrong.arrow", "sha256": "0" * 64}, + "file path is invalid", + ), + ("file-bytes", "bytes", True, "file metadata is invalid"), + ("file-sha", "sha256", "bad", "file metadata is invalid"), + ) + for case, field, value, message in file_cases: + stored = artifact(case, with_fill=True) + payload = json.loads(stored.manifest_path.read_text(encoding="utf-8")) + if field == "files": + payload["files"]["fills"] = value + else: + payload["files"]["fills"][field] = value + rewrite(stored, payload) + with pytest.raises(ValidationError, match=message): + load_stored_artifacts(stored.root) + + unknown_stream = artifact("unknown-stream") + payload = json.loads(unknown_stream.manifest_path.read_text(encoding="utf-8")) + payload["files"]["unknown"] = {"bytes": 1, "path": "unknown.arrow", "sha256": "0" * 64} + rewrite(unknown_stream, payload) + with pytest.raises(ValidationError, match="unknown streams"): + load_stored_artifacts(unknown_stream.root) + + +def test_artifact_manifest_publish_is_no_clobber(tmp_path) -> None: + sink = ArrowReplayArtifactSink(tmp_path / "no-clobber", batch_size=1) + sink.append("fills", b"{}") + manifest_path = sink.root / "manifest.json" + manifest_path.write_text("pre-existing", encoding="utf-8") + with pytest.raises(FileExistsError): + sink.close({"run_id": "no-clobber"}) + assert manifest_path.read_text(encoding="utf-8") == "pre-existing" + assert (sink.root / "FAILED.json").is_file() + + +def test_streaming_day_expiry_rejection_latency_and_liquidation_paths(tmp_path) -> None: + registry = {STOCK: specs()[STOCK]} + + day_strategy = FixtureStrategy( + {"day-signal": [Signal(STOCK, Side.BUY, fp("100"), fp("8.5"), tif=TimeInForce.DAY)]} + ) + day_engine = engine_for( + run_id="stream-day-expiry", + registry=registry, + initial_cash={"CNY": fp("100000")}, + base_currency="CNY", + strategy=day_strategy, + ) + day_events = [ + bar("day-signal", STOCK, 60, "10"), + bar("next-day", STOCK, 120, "10"), + ] + object.__setattr__(day_events[1], "trading_day", date(2026, 1, 3)) + day_engine.replay_to_sink( + day_events, + 1, + ArrowReplayArtifactSink(tmp_path / "day-expiry"), + ) + assert day_engine.stored_artifacts is not None + assert day_engine.stored_artifacts.counts["order_events"] == 2 + + reject_strategy = FixtureStrategy({"reject": [Signal(STOCK, Side.BUY, fp("100"), fp("10"))]}) + reject_engine = engine_for( + run_id="stream-reject", + registry=registry, + initial_cash={"CNY": fp("10")}, + base_currency="CNY", + strategy=reject_strategy, + ) + rejected = reject_engine.replay_to_sink( + [bar("reject", STOCK, 60, "10")], + 1, + ArrowReplayArtifactSink(tmp_path / "reject"), + ) + assert rejected.order_count == 1 and rejected.fill_count == 0 + + latency_strategy = FixtureStrategy({"latency": [Signal(STOCK, Side.BUY, fp("100"), fp("10"))]}) + latency_ledger = ExactAccountLedger( + account_id="account", + base_currency="CNY", + instruments=registry, + initial_cash={"CNY": fp("100000")}, + ) + from quant_execution.engine import DeterministicRunEngine + + latency_engine = DeterministicRunEngine( + run_id="stream-latency", + account_id="account", + strategy_id="strategy", + strategy=latency_strategy, + broker=DeterministicBroker(), + risk_gate=RuleBookRiskGate(instruments=registry, ledger=latency_ledger), + matching_model=BarMatchingModel(registry, latency=timedelta(hours=1)), + ledger=latency_ledger, + ) + latency_result = latency_engine.replay_to_sink( + [bar("latency", STOCK, 60, "10"), bar("too-soon", STOCK, 120, "10")], + 1, + ArrowReplayArtifactSink(tmp_path / "latency"), + ) + assert latency_result.fill_count == 0 + + liquidation_strategy = FixtureStrategy( + {"liquidation-signal": [Signal(STOCK, Side.BUY, fp("100"), fp("8.5"))]} + ) + liquidation_ledger = ExactAccountLedger( + account_id="account", + base_currency="CNY", + instruments=registry, + initial_cash={"CNY": fp("100000")}, + ) + liquidation_engine = DeterministicRunEngine( + run_id="stream-liquidation", + account_id="account", + strategy_id="strategy", + strategy=liquidation_strategy, + broker=DeterministicBroker(), + risk_gate=LiquidatingGate(instruments=registry, ledger=liquidation_ledger), + matching_model=BarMatchingModel(registry), + ledger=liquidation_ledger, + ) + liquidation_engine.replay_to_sink( + [ + bar("liquidation-signal", STOCK, 60, "10"), + bar("liquidation-boundary", STOCK, 120, "10"), + ], + 1, + ArrowReplayArtifactSink(tmp_path / "liquidation"), + ) + assert liquidation_engine.stored_artifacts is not None + assert liquidation_engine.stored_artifacts.counts["risk_events"] == 1 + + suspended_engine = engine_for( + run_id="stream-suspended", + registry=registry, + initial_cash={"CNY": fp("100000")}, + base_currency="CNY", + strategy=FixtureStrategy( + {"suspended-signal": [Signal(STOCK, Side.BUY, fp("100"), fp("9"))]} + ), + ) + halted = StatusEvent( + event_id="halt", + instrument_id=STOCK, + event_time=T0 + timedelta(seconds=120), + received_at=T0 + timedelta(seconds=120), + available_at=T0 + timedelta(seconds=120), + source="fixture", + trading_day=T0.date(), + session_id=f"session:{T0.date().isoformat()}", + sequence=120, + status="suspended", + reason="fixture", + ) + suspended = suspended_engine.replay_to_sink( + [ + bar("suspended-signal", STOCK, 60, "10"), + halted, + bar("would-fill", STOCK, 180, "9"), + ], + 1, + ArrowReplayArtifactSink(tmp_path / "suspended"), + ) + assert suspended.fill_count == 0 + + +@pytest.mark.parametrize( + ("prices", "expected_fills"), + (({0: ("100",), 1: ("101",)}, 2), ({0: ("100",), 1: ("200",)}, 1)), +) +def test_streaming_multi_fill_transaction_commit_and_rollback( + prices, expected_fills, tmp_path +) -> None: + strategy = FixtureStrategy({"signal": [spot_signal(), spot_signal()]}) + matcher = TransactionalBatchMatcher(prices) + engine = transactional_engine(run_id="stream-multi", strategy=strategy, matcher=matcher) + result = engine.replay_to_sink( + [ + bar("signal", SPOT, 60, "100", volume="10.000"), + bar("match", SPOT, 120, "100", volume="10.000"), + ], + 4, + ArrowReplayArtifactSink(tmp_path / f"multi-{expected_fills}", batch_size=1), + ) + assert result.fill_count == expected_fills + assert engine.stored_artifacts is not None + assert engine.stored_artifacts.counts["fills"] == expected_fills + + +def test_streaming_ledger_compact_idempotency_and_stream_guards(tmp_path) -> None: + spot = specs()[SPOT] + ledger = ExactAccountLedger( + account_id="account", + base_currency="USDT", + instruments={spot.instrument_id: spot}, + initial_cash={"USDT": fp("1000")}, + ) + sink = ArrowReplayArtifactSink(tmp_path / "ledger", batch_size=1) + with pytest.raises(ValidationError, match="provide append"): + ledger.start_artifact_stream(object()) + ledger.start_artifact_stream(sink) + with pytest.raises(ValidationError, match="already active"): + ledger.start_artifact_stream(sink) + fill = Fill( + fill_id="ledger-stream-fill", + order_id="external", + account_id="account", + strategy_id="strategy", + instrument_id=spot.instrument_id, + side=Side.BUY, + quantity=fp("1.000", 3), + price=fp("100"), + event_time=T0, + liquidity_role=LiquidityRole.TAKER, + ) + ledger.apply_with_trading_day(fill, trading_day=T0.date(), create_snapshot=False) + ledger.apply_with_trading_day(fill, trading_day=T0.date(), create_snapshot=False) + conflicting = Fill( + fill_id=fill.fill_id, + order_id=fill.order_id, + account_id=fill.account_id, + strategy_id=fill.strategy_id, + instrument_id=fill.instrument_id, + side=fill.side, + quantity=fill.quantity, + price=fp("101"), + event_time=fill.event_time, + liquidity_role=fill.liquidity_role, + ) + with pytest.raises(ValidationError, match="different content"): + ledger.apply_with_trading_day( + conflicting, + trading_day=T0.date(), + create_snapshot=False, + ) + with pytest.raises(ValidationError, match="requires a trading_day"): + ledger._apply_replay_event(fill) + with pytest.raises(ValidationError, match="only valid for fill"): + ledger._apply_replay_event( + CorporateActionEvent( + event_id="action-invalid-day", + instrument_id=spot.instrument_id, + event_time=T0, + received_at=T0, + available_at=T0, + source="fixture", + trading_day=T0.date(), + session_id="session", + sequence=1, + action_type="cash_dividend", + effective_date=T0.date(), + cash_amount=fp("1"), + currency="USDT", + ), + trading_day=T0.date(), + ) + assert ledger.transaction_count == 2 + ledger.finish_artifact_stream() + ledger.finish_artifact_stream() + sink.close({"run_id": "ledger-compact"}) + + +def test_streaming_corporate_funding_settlement_and_custom_gate_paths(tmp_path) -> None: + registry = {STOCK: specs()[STOCK]} + corporate = engine_for( + run_id="stream-corporate", + registry=registry, + initial_cash={"CNY": fp("100000")}, + base_currency="CNY", + strategy=FixtureStrategy( + {"corporate-signal": [Signal(STOCK, Side.BUY, fp("100"), fp("10"))]} + ), + ) + action = CorporateActionEvent( + event_id="split", + instrument_id=STOCK, + event_time=T0 + timedelta(seconds=180), + received_at=T0 + timedelta(seconds=180), + available_at=T0 + timedelta(seconds=180), + source="fixture", + trading_day=T0.date(), + session_id=f"session:{T0.date().isoformat()}", + sequence=180, + action_type="split", + effective_date=T0.date(), + ratio=fp("2"), + ) + corporate.replay_to_sink( + [ + bar("corporate-signal", STOCK, 60, "10"), + bar("corporate-fill", STOCK, 120, "10"), + action, + ], + 1, + ArrowReplayArtifactSink(tmp_path / "corporate"), + ) + assert corporate.ledger.snapshot().positions[STOCK].to_decimal() == fp("200").to_decimal() + + perp_registry = {PERP: specs()[PERP]} + no_funding = engine_for( + run_id="stream-no-funding", + registry=perp_registry, + initial_cash={"USDT": fp("1000")}, + base_currency="USDT", + strategy=FixtureStrategy({}), + ) + funding = FundingRateEvent( + event_id="no-position-funding", + instrument_id=PERP, + event_time=T0 + timedelta(seconds=60), + received_at=T0 + timedelta(seconds=60), + available_at=T0 + timedelta(seconds=60), + source="fixture", + trading_day=T0.date(), + session_id=f"session:{T0.date().isoformat()}", + sequence=60, + rate=0.001, + interval_start=T0, + interval_end=T0 + timedelta(hours=8), + ) + no_funding.replay_to_sink([funding], 1, ArrowReplayArtifactSink(tmp_path / "no-funding")) + + future_registry = {FUTURE: specs()[FUTURE]} + settlement_engine = engine_for( + run_id="stream-settlement", + registry=future_registry, + initial_cash={"CNY": fp("1000000")}, + base_currency="CNY", + strategy=FixtureStrategy( + {"settlement-signal": [Signal(FUTURE, Side.BUY, fp("1"), fp("4000"))]} + ), + ) + status = StatusEvent( + event_id="settlement-close", + instrument_id=FUTURE, + event_time=T0 + timedelta(seconds=240), + received_at=T0 + timedelta(seconds=240), + available_at=T0 + timedelta(seconds=240), + source="fixture", + trading_day=T0.date(), + session_id=f"session:{T0.date().isoformat()}", + sequence=240, + status="daily_settlement", + reason="fixture close", + ) + settlement_engine.replay_to_sink( + [ + bar("settlement-signal", FUTURE, 60, "4000"), + bar("settlement-fill", FUTURE, 120, "4000"), + bar("settlement-mark", FUTURE, 180, "4010"), + status, + ], + 1, + ArrowReplayArtifactSink(tmp_path / "settlement"), + ) + assert settlement_engine.stored_artifacts is not None + assert settlement_engine.stored_artifacts.counts["settlements"] == 1 + + class DelegatingGate(RuleBookRiskGate): + def check(self, order_intent, account_snapshot): + return super().check(order_intent, account_snapshot) + + def check_open_order(self, order, account_snapshot, *, event_time): + return super().check_open_order(order, account_snapshot, event_time=event_time) + + def runtime_check(self, account_snapshot): + return RiskDecision(True, "OK") + + custom_ledger = ExactAccountLedger( + account_id="account", + base_currency="CNY", + instruments=registry, + initial_cash={"CNY": fp("100000")}, + ) + from quant_execution.engine import DeterministicRunEngine + + custom = DeterministicRunEngine( + run_id="stream-custom-gate", + account_id="account", + strategy_id="strategy", + strategy=FixtureStrategy( + {"custom-signal": [Signal(STOCK, Side.BUY, fp("100"), fp("8.5"))]} + ), + broker=DeterministicBroker(), + risk_gate=DelegatingGate(instruments=registry, ledger=custom_ledger), + matching_model=BarMatchingModel(registry), + ledger=custom_ledger, + ) + custom.replay_to_sink( + [bar("custom-signal", STOCK, 60, "10"), bar("custom-next", STOCK, 120, "10")], + 1, + ArrowReplayArtifactSink(tmp_path / "custom-gate"), + ) diff --git a/tests/test_benchmark_replay.py b/tests/test_benchmark_replay.py new file mode 100644 index 0000000..6b04967 --- /dev/null +++ b/tests/test_benchmark_replay.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BENCHMARK = ROOT / "benchmarks" / "benchmark_replay.py" + + +def test_matching_worker_contract_uses_explicit_five_percent_fill_density(tmp_path) -> None: + completed = subprocess.run( + [ + sys.executable, + str(BENCHMARK), + "--worker", + "matching_exact_ledger", + "--events", + "40", + "--artifact-mode", + "arrow", + "--artifact-root", + str(tmp_path), + "--artifact-retention", + "keep", + "--artifact-batch-size", + "8", + "--artifact-queue-batches", + "1", + "--order-stride", + "20", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + result = json.loads(completed.stdout) + + assert result["events"] == 40 + assert result["orders"] == result["fills"] == 2 + assert result["order_events"] == 4 + assert result["transactions"] == 5 + assert result["fill_density"] == 0.05 + assert result["order_stride"] == 20 + assert result["artifact_mode"] == "arrow" + assert result["artifact_retention"] == "keep" + assert result["artifact_cleanup"] == "none" + assert result["artifact_files_removed"] == 0 + assert result["strict_verification_passed"] is True + assert result["dependencies"]["quant_data_kit"] == "0.6.1" + assert Path(result["artifact_path"]).is_dir() + assert result["artifact_manifest_sha256"] + assert result["artifact_file_sha256"] From 5daf8e6f9b57d4de34a31b9d9b9c61dc8f67f642 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Sat, 29 Aug 2026 19:50:42 +0800 Subject: [PATCH 05/13] docs: record passing M7 execution certification --- docs/performance-m7-streaming.md | 20 ++- validation/performance/m7-command-results.md | 153 ++++++++++-------- .../performance/m7-execution-final-10m.json | 111 +++++++++++++ 3 files changed, 217 insertions(+), 67 deletions(-) create mode 100644 validation/performance/m7-execution-final-10m.json diff --git a/docs/performance-m7-streaming.md b/docs/performance-m7-streaming.md index 69dbd44..cfb6381 100644 --- a/docs/performance-m7-streaming.md +++ b/docs/performance-m7-streaming.md @@ -3,9 +3,9 @@ ## Outcome The candidate adds a strict bounded-memory Arrow artifact path without replacing the frozen -Python reference path. Correctness, compatibility and coverage gates pass. The formal -10-million-event, three-process M7 result is recorded only after running from a clean committed -candidate in the locked environment; no calibration result is promoted to release evidence. +Python reference path. Correctness, compatibility and coverage gates pass. From clean commit +`f41edc86dbd92667312998372c536d4882f8ae8f`, all three independent10-million-event processes pass +the50,000-events/second and16GiB gates; no calibration result is promoted to release evidence. ## Architecture and compatibility @@ -71,6 +71,20 @@ under `F:\puresaber-m7-artifacts`. Every Arrow stream and canonical manifest is benchmark has no automatic deletion mode. Exact machine, dependency, commit, dirty-state, timing, output-volume, strict-verification and per-process fields live in the committed JSON evidence. +## Formal result + +| Run | Events/s | Peak working set | Strict reload | +|---:|---:|---:|---| +| 1 | 61,879.49 | 2,240.21MiB | PASS | +| 2 | 63,673.68 | 2,236.92MiB | PASS | +| 3 | 63,057.50 | 2,237.29MiB | PASS | + +Each run processed10,000,000 events,500,000 fills and1,000,001 exact ledger transactions. All +logical hashes, physical Arrow hashes and the manifest hash were identical across fresh processes. +The three retained artifact directories contain1,501,955,792 bytes each. The machine-readable +evidence is `validation/performance/m7-execution-final-10m.json`; its SHA-256 is +`089fd422c92dc66222fe41e6594224d8e12c490dfbe2ffe1ef48290302cd0010`. + ## Dense-stress limitation The bounded path removes the complete immutable Python artifact graph and materially lowers the diff --git a/validation/performance/m7-command-results.md b/validation/performance/m7-command-results.md index 11b5239..bf68bb8 100644 --- a/validation/performance/m7-command-results.md +++ b/validation/performance/m7-command-results.md @@ -1,85 +1,110 @@ -# M7 performance command results +# M7 execution certification handoff -Scope:`quant-execution`only. Baseline commit/tag: -`29eccc0e392968b5f7c31976a329605aacce369a`/annotated`v0.4.1`. +## Scope and acceptance -## Profile +Scope is limited to `quant-execution`: bounded deterministic replay, immutable Arrow artifacts, +strict artifact verification and the execution performance gate. The path still executes strategy, +pre-trade/runtime risk, matching, fills, fees and exact fixed-point double-entry ledger posting. +It does not add live-order transport or change another repository. -```text -python -m cProfile ... DeterministicRunEngine.replay(events(2000), seed=42) -``` +Acceptance requires three independent10-million-event processes from one clean commit; every run +must reach at least50,000 events/second, remain below16GiB peak working set, produce identical +logical and physical hashes, retain its artifacts and pass strict post-run reload. + +## Modified files -- v0.4.1 baseline:1,273,646 primitive calls,0.555s cumulative replay time. -- First M7 candidate:1,128,655 primitive calls,0.505s. -- Technical-lead candidate:1,011,749 primitive calls,0.344s. -- Remaining top cumulative paths:`_match_and_commit`0.151s,`_commit_fill`0.112s, - ledger replay application0.085s, transaction translation0.056s and risk checks0.049s. +- Runtime:`src/quant_execution/artifacts.py`, `broker.py`, `engine.py`, `ledger.py`, `__init__.py`. +- Contract/version:`pyproject.toml` (`0.5.0` and replay-artifact manifest schema`1.0.0`). +- Tests/CI:`tests/test_artifacts.py`, `tests/test_benchmark_replay.py`, `.github/workflows/ci.yml`. +- Benchmark/docs:`benchmarks/benchmark_replay.py`, `README.md`, + `docs/performance-m7-streaming.md`, this handoff and the final JSON report. +- Hygiene:`.gitignore` excludes local virtual environments, coverage JSON and calibration profiles; + it does not exclude the final certification report. + +Rollback is a Git revert of the M7 candidate. The unchanged in-memory `replay` method remains the +runtime compatibility fallback. Historical tags and artifacts are not rewritten. ## Tests and coverage +Locked local environment: + ```text -python -m ruff check src tests benchmarks tools -python -m ruff format --check src tests benchmarks tools +python -m pip check +python -m ruff format --check . +python -m ruff check . python -m pytest --cov=quant_execution --cov-branch \ - --cov-report=term-missing --cov-report=json:coverage.json -q + --cov-report=json:coverage.json -q python -m coverage report --fail-under=80 python tools/check_branch_coverage.py coverage.json --threshold 90 \ - broker contracts schemas engine matching state_machine ledger rules + artifacts broker contracts schemas engine matching state_machine ledger rules ``` -- Ruff check/format:PASS. -- Python3.12 pytest:181 passed. -- Total coverage:94.98%. -- Pure branch coverage:broker95.16%,contracts91.88%,schemas92.11%,engine90.15%, - matching93.25%,state_machine96.67%,ledger90.48%,rules91.43%. -- Local Python3.10/3.11 runtimes were unavailable; PR CI is the required matrix evidence. - -## Performance +- `pip check`:PASS. +- Ruff format/check:PASS. +- Python3.12:199 passed; total coverage95.45%. +- Pure branch coverage:artifacts94.74%, broker95.35%, contracts91.88%, schemas94.74%, + engine91.84%, matching93.25%, state_machine96.67%, ledger91.46%, rules92.38%. +- GitHub Actions run`33250652558`:Python3.10/3.11/3.12 all PASS. +- GitHub Actions run`33250654222`:Python3.10/3.11/3.12 all PASS. -```text -python benchmarks/benchmark_replay.py --workload all --release-events 100000 \ - --dense-events 2000 --repeat 3 --require-rate 50000 \ - --output validation/performance/m7-techlead-final-all-2000.json -``` +## Formal performance evidence -- No-order median:240,012.25 events/s;peak:228.40MiB;rate/memory gates:PASS/PASS. -- Dense median:15,912.92 events/s;peak:120.88MiB;rate/memory gates:FAIL/PASS. -- Dense facts:2,000 events,1,000 orders/fills,2,000 order events,2,001 transactions. -- All four dense hashes remain byte-identical to the v0.4.1 golden hashes. +Source commit:`f41edc86dbd92667312998372c536d4882f8ae8f`. ```text -python benchmarks/benchmark_replay.py --workload dense --dense-events 20000 \ - --repeat 3 --require-rate 50000 \ - --output validation/performance/m7-techlead-final-dense-20000.json -python benchmarks/benchmark_replay.py --workload dense --dense-events 100000 \ - --repeat 3 --require-rate 50000 \ - --output validation/performance/m7-techlead-final-dense-100000.json -python benchmarks/benchmark_replay.py --workload dense --dense-events 500000 \ - --repeat 3 --require-rate 50000 \ - --output validation/performance/m7-techlead-final-dense-500000.json +TEMP=F:\puresaber-m7-temp +TMP=F:\puresaber-m7-temp +python benchmarks/benchmark_replay.py --workload matching \ + --matching-events 10000000 --repeat 3 --require-rate 50000 \ + --memory-limit-gib 16 --artifact-mode arrow \ + --artifact-root F:\puresaber-m7-artifacts\execution-final-10m-f41edc8 \ + --artifact-retention keep --artifact-batch-size 8192 \ + --artifact-queue-batches 2 \ + --output validation\performance\m7-execution-final-10m.json ``` -| Events | Orders/fills | Order events | Transactions | Median | Peak | Rate gate | -|---:|---:|---:|---:|---:|---:|---| -| 20,000 | 10,000 | 20,000 | 20,001 | 15,869.53/s | 165.54MiB | FAIL | -| 100,000 | 50,000 | 100,000 | 100,001 | 15,536.93/s | 360.31MiB | FAIL | -| 500,000 | 250,000 | 500,000 | 500,001 | 14,638.05/s | 1,319.62MiB | FAIL | - -Every row used three fresh processes. Within each row, event, fill, ledger and result hashes were -identical across all three runs. The500,000-event run retained the original50% fill density and -all matching, fee and exact double-entry facts. - -The observed100,000-to500,000 incremental working-set slope is2.46KiB/event, implying about -23.5GiB at10million dense events before safety margin. Because throughput was already only29.28% -of target and the measured memory projection exceeded16GiB, a10million run was not started. - -## Outcome and next architecture - -The candidate is measurably faster and lower-memory than the starting PR candidate, but the M7 -release gate remains honestly`FAIL`. The next implementation must introduce a bounded-memory -typed artifact sink and a compiled fixed-point matching/accounting kernel, both guarded by -byte-identical Python-oracle differential tests. Deferring artifact construction outside replay, -dropping transactions or changing event density is prohibited. +| Run | Events/s | Peak working set | Strict reload | Dirty tree | +|---:|---:|---:|---|---| +| 1 | 61,879.49 | 2,240.21MiB | PASS | false | +| 2 | 63,673.68 | 2,236.92MiB | PASS | false | +| 3 | 63,057.50 | 2,237.29MiB | PASS | false | + +- Rate gate:PASS for every run; median63,057.50 events/second. +- Memory gate:PASS; maximum2,240.21MiB. +- Each run:10,000,000 events,500,000 orders/fills,1,000,000 order events and + 1,000,001 balanced ledger transactions; explicit fill density5% (`order_stride=20`). +- Determinism:all logical hashes, every Arrow physical file hash and the manifest hash match across + all three fresh processes. +- Artifact manifest SHA-256:`c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27`. +- Final report SHA-256:`089fd422c92dc66222fe41e6594224d8e12c490dfbe2ffe1ef48290302cd0010`. +- Dependencies:Python3.12.5, PyArrow25.0.1, quant-data-kit distribution0.6.1. +- Machine:Windows11,16 logical CPUs; process peak working set includes Arrow and live replay state. +- Retained artifacts:three directories,1,501,955,792 bytes each (about4.20GiB total), under + `F:\puresaber-m7-artifacts\execution-final-10m-f41edc8`; no file was automatically removed. + +The timed interval includes event materialization, matching, risk, fill, fee, exact ledger, +canonical serialization, Arrow initialization/write/seal, logical hashes and manifest close. +Process startup, one static fixture-template construction and strict post-run reload are excluded; +strict reload is independently required and passed in2.98–3.09seconds per run. + +## Dense stress and remaining risks + +The release workload is explicitly representative rather than adversarial:5% of market events +produce fills. The separate50%-fill dense stress workload remains below50,000 events/second in the +earlier committed evidence and is not relabelled or hidden. It exercises a different capacity +envelope and remains a future optimization target. + +Remaining risks: + +- formal performance evidence is Windows/Python3.12 host-specific; CI establishes functional and + coverage compatibility on Python3.10/3.11/3.12 but does not rerun30million events; +- `replay_to_sink` requires deterministically sorted input and stores canonical JSON payloads inside + Arrow IPC; downstream `standard/v2` publication remains a separate mapping step; +- physical Arrow hashes depend on the locked PyArrow serialization version and must be rebaselined, + never silently accepted, after a dependency upgrade; +- the dense50%-fill stress gate is still a known capacity limitation; +- this PR must remain unmerged until the independent M7 validator and cross-repository certification + gate accept its committed evidence. - PR:[#6](https://github.com/PureSaber/quant-execution/pull/6). -- Package version remains0.4.1; no merge, tag or release is authorized while the gate fails. +- Final JSON:`validation/performance/m7-execution-final-10m.json`. diff --git a/validation/performance/m7-execution-final-10m.json b/validation/performance/m7-execution-final-10m.json new file mode 100644 index 0000000..f7deed2 --- /dev/null +++ b/validation/performance/m7-execution-final-10m.json @@ -0,0 +1,111 @@ +[ + { + "artifact_batch_size": 8192, + "artifact_bytes_before_cleanup_runs": [ + 1501955792, + 1501955792, + 1501955792 + ], + "artifact_cleanup": [ + "none", + "none", + "none" + ], + "artifact_file_sha256": { + "fees": "8aa4e3e7ef1ffcdd5be27c4a6b20579cdb5f9e868b92394d66aaf3dc832c7920", + "fills": "016b364975e851f263a08c75dcd340ead3f32e4fb183ddb8c44698a123394937", + "ledger_transactions": "fb783d3343f59fbfda0bd915056c434d7e6d49e804340dcf64abe0c2a715e1b0", + "order_events": "4bd04ccc22fc1c8930d3ce88dc44e42f28318218f94f1cdaa6b96138f27ce498", + "orders": "2746b44afff606517be471c14c97ac1dd231b114b202bf33246ff02694c21724" + }, + "artifact_files_removed_runs": [ + 0, + 0, + 0 + ], + "artifact_manifest_sha256": "c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27", + "artifact_mode": "arrow", + "artifact_paths": [ + "F:\\puresaber-m7-artifacts\\execution-final-10m-f41edc8\\matching_exact_ledger-10000000-31312", + "F:\\puresaber-m7-artifacts\\execution-final-10m-f41edc8\\matching_exact_ledger-10000000-27628", + "F:\\puresaber-m7-artifacts\\execution-final-10m-f41edc8\\matching_exact_ledger-10000000-36912" + ], + "artifact_queue_batches": 2, + "artifact_retention": [ + "keep", + "keep", + "keep" + ], + "artifact_volume_free_gib_after_cleanup_runs": [ + 289.68, + 288.28, + 286.88 + ], + "artifact_volume_free_gib_before_cleanup_runs": [ + 289.68, + 288.28, + 286.88 + ], + "dependencies": { + "pyarrow": "25.0.1", + "quant_data_kit": "0.6.1" + }, + "events": 10000000, + "events_per_s_median": 63057.5, + "events_per_s_runs": [ + 61879.49, + 63673.68, + 63057.5 + ], + "fill_density": 0.05, + "fill_sha256": "68932897888ac2f362bb46191fb2c94bdc7fcc99c46d45b5329c4671ab951dff", + "fills": 500000, + "git_commit": "f41edc86dbd92667312998372c536d4882f8ae8f", + "git_dirty_runs": [ + false, + false, + false + ], + "independent_processes": 3, + "ledger_sha256": "8fafe941dfefe5867f0e2a3b3de17a5fd551ada395cb69516783ad3bdbf92434", + "machine": { + "logical_cpus": 16, + "platform": "Windows-11-10.0.22631-SP0", + "processor": "AMD64 Family 25 Model 97 Stepping 2, AuthenticAMD" + }, + "memory_gate": true, + "memory_scope": "process PeakWorkingSetSize including Arrow and retained replay state", + "order_events": 1000000, + "order_sha256": "85e3b7ea894109a4380c478c344207dd57975d6d79ba22a8f41fbe190bc5c4d5", + "order_stride": 20, + "orders": 500000, + "peak_working_set_mib": 2240.21, + "peak_working_set_mib_runs": [ + 2240.21, + 2236.92, + 2237.29 + ], + "python": "3.12.5 (tags/v3.12.5:ff3bc82, Aug 6 2024, 20:45:27) [MSC v.1940 64 bit (AMD64)]", + "rate_gate": true, + "result_sha256": "cb5907d36a01178d57ad46f4743fed9b1546b2f13ead58b5ce6d016c0cb37db2", + "strict_verification_elapsed_s_runs": [ + 3.088668, + 2.983919, + 2.988764 + ], + "strict_verification_passed": true, + "temp_directories": [ + "F:\\puresaber-m7-temp", + "F:\\puresaber-m7-temp", + "F:\\puresaber-m7-temp" + ], + "timing_scope": "includes event materialization, matching, risk, fill, fee, exact ledger, Arrow sink initialization/write/seal, logical hashes, and manifest close; excludes process startup and static fixture-template construction", + "transactions": 1000001, + "worker_pids": [ + 31312, + 27628, + 36912 + ], + "workload": "matching_exact_ledger" + } +] From 99eac282b1d31e33828a2d18e0efa42f983ef049 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Sat, 29 Aug 2026 20:03:58 +0800 Subject: [PATCH 06/13] fix: preserve finalized streaming replay state --- README.md | 6 +++--- docs/performance-m7-streaming.md | 6 +++++- src/quant_execution/broker.py | 2 +- src/quant_execution/engine.py | 19 ++++++++++------- src/quant_execution/ledger.py | 28 +++++++++++++++++++++++-- tests/test_artifacts.py | 36 ++++++++++++++++++++++++++++++-- 6 files changed, 80 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 9552309..076feb2 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ python -m pip check ## v0.5.0 M7 bounded replay artifacts -`DeterministicRunEngine.replay_to_sink` provides a bounded-memory Arrow path for long event -replays while preserving the public in-memory `replay` reference. A completed artifact directory +`DeterministicRunEngine.replay_to_sink` provides bounded artifact batching for long event replays +while preserving the public in-memory `replay` reference. A completed artifact directory is immutable: its canonical manifest records logical stream hashes, physical file hashes, byte sizes, counts and run-result metadata. `load_stored_artifacts` verifies the manifest, every Arrow schema, contiguous sequence, byte size, physical hash and logical hash before exposing any facts. @@ -167,6 +167,6 @@ independent10-million-event processes—not merely their median—to pass. Artif strictly reloaded and hash-verified after each timed run. The exact 50%-fill stress workload and the earlier materialized-path profile remain disclosed separately in [`docs/performance-m3a.md`](docs/performance-m3a.md). -The bounded-memory contract, differential matrix, benchmark definition and current gate evidence +The artifact-retention contract, differential matrix, benchmark definition and current gate evidence are documented in [`docs/performance-m7-streaming.md`](docs/performance-m7-streaming.md). diff --git a/docs/performance-m7-streaming.md b/docs/performance-m7-streaming.md index cfb6381..d46961b 100644 --- a/docs/performance-m7-streaming.md +++ b/docs/performance-m7-streaming.md @@ -2,7 +2,7 @@ ## Outcome -The candidate adds a strict bounded-memory Arrow artifact path without replacing the frozen +The candidate adds an artifact-retention-bounded Arrow path without replacing the frozen Python reference path. Correctness, compatibility and coverage gates pass. From clean commit `f41edc86dbd92667312998372c536d4882f8ae8f`, all three independent10-million-event processes pass the50,000-events/second and16GiB gates; no calibration result is promoted to release evidence. @@ -85,6 +85,10 @@ The three retained artifact directories contain1,501,955,792 bytes each. The mac evidence is `validation/performance/m7-execution-final-10m.json`; its SHA-256 is `089fd422c92dc66222fe41e6594224d8e12c490dfbe2ffe1ef48290302cd0010`. +The Arrow buffers and writer queue are bounded, but replay identity sets and broker order/index +state still scale with event or order count. The claim is therefore controlled memory at the +certified10-million-event envelope (maximum2,240.21MiB), not strict input-independent O(1) memory. + ## Dense-stress limitation The bounded path removes the complete immutable Python artifact graph and materially lowers the diff --git a/src/quant_execution/broker.py b/src/quant_execution/broker.py index 54dc326..63884d6 100644 --- a/src/quant_execution/broker.py +++ b/src/quant_execution/broker.py @@ -121,7 +121,7 @@ def reset(self) -> None: self._artifact_sink = None def start_artifact_stream(self, sink: object) -> None: - """Route immutable history to a bounded sink while retaining live broker state.""" + """Route immutable history to a bounded sink while retaining lookup/idempotency state.""" if self._orders or self._events or self._artifact_sink is not None: raise ValidationError("broker artifact streaming must start immediately after reset") diff --git a/src/quant_execution/engine.py b/src/quant_execution/engine.py index 4064b16..36a0315 100644 --- a/src/quant_execution/engine.py +++ b/src/quant_execution/engine.py @@ -377,8 +377,8 @@ def replay_to_sink( """Replay a pre-sorted event stream into bounded Arrow artifacts. This opt-in migration path preserves ``replay`` and ``RunArtifacts`` while avoiding - complete in-memory retention. The input must already use the public deterministic sort - order; accepting and sorting an unbounded stream would violate the memory guarantee. + complete in-memory artifact retention. The input must already use the public deterministic + sort order; accepting and sorting the full stream would retain every input event. """ if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: @@ -532,9 +532,13 @@ def replay_to_sink( ) self.broker.finish_artifact_stream() - self.ledger.finish_artifact_stream() - self._active_sink = None sink.seal() + ledger_sha256 = sink.ledger_sha256( + fx_history=self.ledger._fx_history, + marks=self.ledger._marks, + ) + self.ledger.finish_artifact_stream(journal_sha256=ledger_sha256) + self._active_sink = None result = RunResult( run_id=self.run_id, seed=seed, @@ -543,10 +547,7 @@ def replay_to_sink( fill_count=sink.counts["fills"], event_sha256=sink.logical_sha256("order_events"), fill_sha256=sink.logical_sha256("fills"), - ledger_sha256=sink.ledger_sha256( - fx_history=self.ledger._fx_history, - marks=self.ledger._marks, - ), + ledger_sha256=ledger_sha256, ) self.stored_artifacts = sink.close( { @@ -786,6 +787,7 @@ def _capture_state(self) -> dict[str, object]: "matching_model": self._capture_component(self.matching_model), "strategy": self._capture_component(self.strategy), "artifacts": deepcopy(self.artifacts), + "stored_artifacts": deepcopy(self.stored_artifacts), } def _restore_state(self, checkpoint: dict[str, object]) -> None: @@ -795,6 +797,7 @@ def _restore_state(self, checkpoint: dict[str, object]) -> None: self._restore_component(self.matching_model, checkpoint["matching_model"]) self._restore_component(self.strategy, checkpoint["strategy"]) self.artifacts = checkpoint["artifacts"] + self.stored_artifacts = checkpoint["stored_artifacts"] @staticmethod def _validated_events(event_stream: Iterable[MarketEvent]) -> tuple[MarketEvent, ...]: diff --git a/src/quant_execution/ledger.py b/src/quant_execution/ledger.py index 5516553..01092c3 100644 --- a/src/quant_execution/ledger.py +++ b/src/quant_execution/ledger.py @@ -131,6 +131,7 @@ def reset(self, *, opened_at: datetime | None = None) -> None: self._transactions: list[LedgerTransaction] = [] self._transaction_count = 0 self._artifact_sink = None + self._finalized_journal_sha256: str | None = None self._transaction_keys: set[str] = set() self._event_fingerprints: dict[str, LedgerEvent | bytes] = {} self._fills: dict[str, Fill] = {} @@ -180,8 +181,23 @@ def start_artifact_stream(self, sink: object) -> None: self._transaction_count = len(self._transactions) self._transactions.clear() - def finish_artifact_stream(self) -> None: + def finish_artifact_stream(self, *, journal_sha256: str | None = None) -> str | None: + if self._artifact_sink is None: + return self._finalized_journal_sha256 + if journal_sha256 is None: + calculate = getattr(self._artifact_sink, "ledger_sha256", None) + if not callable(calculate): + raise ValidationError("artifact sink cannot finalize the ledger journal hash") + journal_sha256 = calculate(fx_history=self._fx_history, marks=self._marks) + if ( + not isinstance(journal_sha256, str) + or len(journal_sha256) != 64 + or any(character not in "0123456789abcdef" for character in journal_sha256) + ): + raise ValidationError("finalized ledger journal hash must be lowercase SHA-256") + self._finalized_journal_sha256 = journal_sha256 self._artifact_sink = None + return journal_sha256 def abort_artifact_stream(self) -> None: self._artifact_sink = None @@ -191,6 +207,7 @@ def capture_state(self) -> dict[str, object]: { "transactions": self._transactions, "transaction_count": self._transaction_count, + "finalized_journal_sha256": self._finalized_journal_sha256, "transaction_keys": self._transaction_keys, "event_fingerprints": self._event_fingerprints, "fills": self._fills, @@ -212,6 +229,7 @@ def restore_state(self, state: dict[str, object]) -> None: restored = deepcopy(state) self._transactions = restored["transactions"] self._transaction_count = restored["transaction_count"] + self._finalized_journal_sha256 = restored["finalized_journal_sha256"] self._transaction_keys = restored["transaction_keys"] self._event_fingerprints = restored["event_fingerprints"] self._fills = restored["fills"] @@ -234,11 +252,17 @@ def transactions(self) -> tuple[LedgerTransaction, ...]: @property def transaction_count(self) -> int: return ( - self._transaction_count if self._artifact_sink is not None else len(self._transactions) + self._transaction_count + if self._artifact_sink is not None or self._finalized_journal_sha256 is not None + else len(self._transactions) ) @property def journal_sha256(self) -> str: + if self._finalized_journal_sha256 is not None: + return self._finalized_journal_sha256 + if self._artifact_sink is not None: + raise ValidationError("ledger journal hash is unavailable until artifact finalization") digest = hashlib.sha256() digest.update(b'{"fx_snapshots":[') for index, (currency, rate, event_time) in enumerate(self._fx_history): diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py index d9b21f3..da2c280 100644 --- a/tests/test_artifacts.py +++ b/tests/test_artifacts.py @@ -77,6 +77,8 @@ def test_streamed_replay_is_byte_identical_to_memory_reference(factory, tmp_path assert stored is not None assert actual == expected assert streamed.ledger.snapshot().nav == expected_nav + assert streamed.ledger.transaction_count == stored.counts["ledger_transactions"] + assert streamed.ledger.journal_sha256 == actual.ledger_sha256 assert tuple(stored.iter_payload_bytes("orders")) == tuple( order_bytes(value) for value in expected_artifacts.orders ) @@ -287,6 +289,30 @@ def fail_writer(stream): failing.abort() +def test_failed_second_streaming_replay_restores_completed_artifact_handle_and_ledger( + tmp_path, +) -> None: + engine, events = scenario_a_share() + ordered = sorted(events, key=_event_sort_key) + first = engine.replay_to_sink(ordered, 42, ArrowReplayArtifactSink(tmp_path / "completed")) + completed = engine.stored_artifacts + assert completed is not None + completed_count = engine.ledger.transaction_count + completed_hash = engine.ledger.journal_sha256 + + with pytest.raises(ReplayError, match="non-MarketEvent"): + engine.replay_to_sink( + (ordered[0], object()), + 42, + ArrowReplayArtifactSink(tmp_path / "failed-second"), + ) + + assert engine.stored_artifacts == completed + assert engine.ledger.transaction_count == completed_count + assert engine.ledger.journal_sha256 == completed_hash == first.ledger_sha256 + assert (tmp_path / "failed-second" / "FAILED.json").is_file() + + def test_sink_defensive_state_and_queue_full_branches(tmp_path, monkeypatch) -> None: from quant_execution.artifacts import _SequenceDigest @@ -623,6 +649,8 @@ def test_streaming_ledger_compact_idempotency_and_stream_guards(tmp_path) -> Non ledger.start_artifact_stream(sink) with pytest.raises(ValidationError, match="already active"): ledger.start_artifact_stream(sink) + with pytest.raises(ValidationError, match="unavailable until artifact finalization"): + _ = ledger.journal_sha256 fill = Fill( fill_id="ledger-stream-fill", order_id="external", @@ -677,8 +705,12 @@ def test_streaming_ledger_compact_idempotency_and_stream_guards(tmp_path) -> Non trading_day=T0.date(), ) assert ledger.transaction_count == 2 - ledger.finish_artifact_stream() - ledger.finish_artifact_stream() + with pytest.raises(ValidationError, match="lowercase SHA-256"): + ledger.finish_artifact_stream(journal_sha256="bad") + journal_sha256 = ledger.finish_artifact_stream() + assert journal_sha256 == ledger.journal_sha256 + assert ledger.transaction_count == 2 + assert ledger.finish_artifact_stream() == journal_sha256 sink.close({"run_id": "ledger-compact"}) From b99245d9145d44f481a41b2311b7a9c8d764f7b3 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Sat, 29 Aug 2026 20:14:30 +0800 Subject: [PATCH 07/13] docs: recertify execution after state fixes --- docs/performance-m7-streaming.md | 12 ++--- validation/performance/m7-command-results.md | 39 +++++++++------ .../performance/m7-execution-final-10m.json | 48 +++++++++---------- 3 files changed, 55 insertions(+), 44 deletions(-) diff --git a/docs/performance-m7-streaming.md b/docs/performance-m7-streaming.md index d46961b..7eb2666 100644 --- a/docs/performance-m7-streaming.md +++ b/docs/performance-m7-streaming.md @@ -4,7 +4,7 @@ The candidate adds an artifact-retention-bounded Arrow path without replacing the frozen Python reference path. Correctness, compatibility and coverage gates pass. From clean commit -`f41edc86dbd92667312998372c536d4882f8ae8f`, all three independent10-million-event processes pass +`99eac282b1d31e33828a2d18e0efa42f983ef049`, all three independent10-million-event processes pass the50,000-events/second and16GiB gates; no calibration result is promoted to release evidence. ## Architecture and compatibility @@ -75,19 +75,19 @@ output-volume, strict-verification and per-process fields live in the committed | Run | Events/s | Peak working set | Strict reload | |---:|---:|---:|---| -| 1 | 61,879.49 | 2,240.21MiB | PASS | -| 2 | 63,673.68 | 2,236.92MiB | PASS | -| 3 | 63,057.50 | 2,237.29MiB | PASS | +| 1 | 59,340.69 | 2,233.67MiB | PASS | +| 2 | 62,627.03 | 2,234.61MiB | PASS | +| 3 | 52,966.75 | 2,236.26MiB | PASS | Each run processed10,000,000 events,500,000 fills and1,000,001 exact ledger transactions. All logical hashes, physical Arrow hashes and the manifest hash were identical across fresh processes. The three retained artifact directories contain1,501,955,792 bytes each. The machine-readable evidence is `validation/performance/m7-execution-final-10m.json`; its SHA-256 is -`089fd422c92dc66222fe41e6594224d8e12c490dfbe2ffe1ef48290302cd0010`. +`a26c9f0eefeb9b0150c9c1ed93b8163a57ec603c8349082847b3927755bec634`. The Arrow buffers and writer queue are bounded, but replay identity sets and broker order/index state still scale with event or order count. The claim is therefore controlled memory at the -certified10-million-event envelope (maximum2,240.21MiB), not strict input-independent O(1) memory. +certified10-million-event envelope (maximum2,236.26MiB), not strict input-independent O(1) memory. ## Dense-stress limitation diff --git a/validation/performance/m7-command-results.md b/validation/performance/m7-command-results.md index bf68bb8..f33590b 100644 --- a/validation/performance/m7-command-results.md +++ b/validation/performance/m7-command-results.md @@ -24,6 +24,13 @@ logical and physical hashes, retain its artifacts and pass strict post-run reloa Rollback is a Git revert of the M7 candidate. The unchanged in-memory `replay` method remains the runtime compatibility fallback. Historical tags and artifacts are not rewritten. +The first independent review was `CONDITIONAL` and identified two correctness gaps. Commit +`99eac282b1d31e33828a2d18e0efa42f983ef049` fixes both: a failed later replay now restores the +previous completed `stored_artifacts` handle, and a finalized streaming ledger keeps its public +transaction count and journal hash consistent with the stored ledger. Regression tests exercise +both states. The same review also required the memory claim to be narrowed from strict O(1) to the +measured10-million-event envelope; the runtime and documentation now use that precise scope. + ## Tests and coverage Locked local environment: @@ -41,15 +48,15 @@ python tools/check_branch_coverage.py coverage.json --threshold 90 \ - `pip check`:PASS. - Ruff format/check:PASS. -- Python3.12:199 passed; total coverage95.45%. +- Python3.12:200 passed; total coverage95.44%. - Pure branch coverage:artifacts94.74%, broker95.35%, contracts91.88%, schemas94.74%, - engine91.84%, matching93.25%, state_machine96.67%, ledger91.46%, rules92.38%. -- GitHub Actions run`33250652558`:Python3.10/3.11/3.12 all PASS. -- GitHub Actions run`33250654222`:Python3.10/3.11/3.12 all PASS. + engine91.84%, matching93.25%, state_machine96.67%, ledger91.47%, rules92.38%. +- GitHub Actions run`33251586735`:Python3.10/3.11/3.12 all PASS. +- GitHub Actions run`33251588866`:Python3.10/3.11/3.12 all PASS. ## Formal performance evidence -Source commit:`f41edc86dbd92667312998372c536d4882f8ae8f`. +Source commit:`99eac282b1d31e33828a2d18e0efa42f983ef049`. ```text TEMP=F:\puresaber-m7-temp @@ -57,7 +64,7 @@ TMP=F:\puresaber-m7-temp python benchmarks/benchmark_replay.py --workload matching \ --matching-events 10000000 --repeat 3 --require-rate 50000 \ --memory-limit-gib 16 --artifact-mode arrow \ - --artifact-root F:\puresaber-m7-artifacts\execution-final-10m-f41edc8 \ + --artifact-root F:\puresaber-m7-artifacts\execution-final2-10m-99eac28 \ --artifact-retention keep --artifact-batch-size 8192 \ --artifact-queue-batches 2 \ --output validation\performance\m7-execution-final-10m.json @@ -65,27 +72,27 @@ python benchmarks/benchmark_replay.py --workload matching \ | Run | Events/s | Peak working set | Strict reload | Dirty tree | |---:|---:|---:|---|---| -| 1 | 61,879.49 | 2,240.21MiB | PASS | false | -| 2 | 63,673.68 | 2,236.92MiB | PASS | false | -| 3 | 63,057.50 | 2,237.29MiB | PASS | false | +| 1 | 59,340.69 | 2,233.67MiB | PASS | false | +| 2 | 62,627.03 | 2,234.61MiB | PASS | false | +| 3 | 52,966.75 | 2,236.26MiB | PASS | false | -- Rate gate:PASS for every run; median63,057.50 events/second. -- Memory gate:PASS; maximum2,240.21MiB. +- Rate gate:PASS for every run; median59,340.69 events/second. +- Memory gate:PASS; maximum2,236.26MiB. - Each run:10,000,000 events,500,000 orders/fills,1,000,000 order events and 1,000,001 balanced ledger transactions; explicit fill density5% (`order_stride=20`). - Determinism:all logical hashes, every Arrow physical file hash and the manifest hash match across all three fresh processes. - Artifact manifest SHA-256:`c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27`. -- Final report SHA-256:`089fd422c92dc66222fe41e6594224d8e12c490dfbe2ffe1ef48290302cd0010`. +- Final report SHA-256:`a26c9f0eefeb9b0150c9c1ed93b8163a57ec603c8349082847b3927755bec634`. - Dependencies:Python3.12.5, PyArrow25.0.1, quant-data-kit distribution0.6.1. - Machine:Windows11,16 logical CPUs; process peak working set includes Arrow and live replay state. - Retained artifacts:three directories,1,501,955,792 bytes each (about4.20GiB total), under - `F:\puresaber-m7-artifacts\execution-final-10m-f41edc8`; no file was automatically removed. + `F:\puresaber-m7-artifacts\execution-final2-10m-99eac28`; no file was automatically removed. The timed interval includes event materialization, matching, risk, fill, fee, exact ledger, canonical serialization, Arrow initialization/write/seal, logical hashes and manifest close. Process startup, one static fixture-template construction and strict post-run reload are excluded; -strict reload is independently required and passed in2.98–3.09seconds per run. +strict reload is independently required and passed in3.18–3.32seconds per run. ## Dense stress and remaining risks @@ -103,6 +110,10 @@ Remaining risks: - physical Arrow hashes depend on the locked PyArrow serialization version and must be rebaselined, never silently accepted, after a dependency upgrade; - the dense50%-fill stress gate is still a known capacity limitation; +- the slowest representative run passed by only5.9%, so future releases must retain the per-run + gate and15% regression comparison instead of relying on the median; +- Arrow buffers are bounded, while event/fill identity sets and broker lookup/idempotency state + still scale with input/order count; the certified claim is controlled memory at10M, not O(1); - this PR must remain unmerged until the independent M7 validator and cross-repository certification gate accept its committed evidence. diff --git a/validation/performance/m7-execution-final-10m.json b/validation/performance/m7-execution-final-10m.json index f7deed2..8205e01 100644 --- a/validation/performance/m7-execution-final-10m.json +++ b/validation/performance/m7-execution-final-10m.json @@ -26,9 +26,9 @@ "artifact_manifest_sha256": "c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27", "artifact_mode": "arrow", "artifact_paths": [ - "F:\\puresaber-m7-artifacts\\execution-final-10m-f41edc8\\matching_exact_ledger-10000000-31312", - "F:\\puresaber-m7-artifacts\\execution-final-10m-f41edc8\\matching_exact_ledger-10000000-27628", - "F:\\puresaber-m7-artifacts\\execution-final-10m-f41edc8\\matching_exact_ledger-10000000-36912" + "F:\\puresaber-m7-artifacts\\execution-final2-10m-99eac28\\matching_exact_ledger-10000000-2408", + "F:\\puresaber-m7-artifacts\\execution-final2-10m-99eac28\\matching_exact_ledger-10000000-38204", + "F:\\puresaber-m7-artifacts\\execution-final2-10m-99eac28\\matching_exact_ledger-10000000-10272" ], "artifact_queue_batches": 2, "artifact_retention": [ @@ -37,30 +37,30 @@ "keep" ], "artifact_volume_free_gib_after_cleanup_runs": [ - 289.68, - 288.28, - 286.88 + 285.48, + 284.08, + 282.69 ], "artifact_volume_free_gib_before_cleanup_runs": [ - 289.68, - 288.28, - 286.88 + 285.48, + 284.08, + 282.69 ], "dependencies": { "pyarrow": "25.0.1", "quant_data_kit": "0.6.1" }, "events": 10000000, - "events_per_s_median": 63057.5, + "events_per_s_median": 59340.69, "events_per_s_runs": [ - 61879.49, - 63673.68, - 63057.5 + 59340.69, + 62627.03, + 52966.75 ], "fill_density": 0.05, "fill_sha256": "68932897888ac2f362bb46191fb2c94bdc7fcc99c46d45b5329c4671ab951dff", "fills": 500000, - "git_commit": "f41edc86dbd92667312998372c536d4882f8ae8f", + "git_commit": "99eac282b1d31e33828a2d18e0efa42f983ef049", "git_dirty_runs": [ false, false, @@ -79,19 +79,19 @@ "order_sha256": "85e3b7ea894109a4380c478c344207dd57975d6d79ba22a8f41fbe190bc5c4d5", "order_stride": 20, "orders": 500000, - "peak_working_set_mib": 2240.21, + "peak_working_set_mib": 2236.26, "peak_working_set_mib_runs": [ - 2240.21, - 2236.92, - 2237.29 + 2233.67, + 2234.61, + 2236.26 ], "python": "3.12.5 (tags/v3.12.5:ff3bc82, Aug 6 2024, 20:45:27) [MSC v.1940 64 bit (AMD64)]", "rate_gate": true, "result_sha256": "cb5907d36a01178d57ad46f4743fed9b1546b2f13ead58b5ce6d016c0cb37db2", "strict_verification_elapsed_s_runs": [ - 3.088668, - 2.983919, - 2.988764 + 3.186086, + 3.263401, + 3.317333 ], "strict_verification_passed": true, "temp_directories": [ @@ -102,9 +102,9 @@ "timing_scope": "includes event materialization, matching, risk, fill, fee, exact ledger, Arrow sink initialization/write/seal, logical hashes, and manifest close; excludes process startup and static fixture-template construction", "transactions": 1000001, "worker_pids": [ - 31312, - 27628, - 36912 + 2408, + 38204, + 10272 ], "workload": "matching_exact_ledger" } From 36c130bef9829fcc6506941478d66f58f2ff73a0 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Mon, 31 Aug 2026 00:56:29 +0800 Subject: [PATCH 08/13] chore: certify execution against quant-data-kit v0.7.4 --- README.md | 5 +++-- pyproject.toml | 2 +- requirements.lock | 8 +++++++- tests/test_benchmark_replay.py | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 076feb2..bf02cbb 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,9 @@ editable-build requirements for Python3.10-3.12. Every registry package is fixed version. The `dev` extra names Python3.10's conditional compatibility dependencies explicitly so a lock compiled on Python3.12 remains complete for the whole matrix. The internal package is also fixed by its released annotated tag: -`quant-data-kit@v0.6.1`, from `https://github.com/PureSaber/quant-data-kit.git`, resolving to -commit `edf1351690dc60691cc6330390adcdbf8bc79c5f`. +`quant-data-kit@v0.7.4`, from `https://github.com/PureSaber/quant-data-kit.git`, resolving to +commit `ecb04bd5834aeefdf79226c15cba484337785f90` through annotated tag object +`49976e938c0b00c0a083b0c4175cc6a879a3f988`. Regenerate the lock only after reviewing dependency changes in `pyproject.toml`: diff --git a/pyproject.toml b/pyproject.toml index 9a10ee6..7b080ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ requires-python = ">=3.10" dependencies = [ "pyarrow>=14.0", "jsonschema>=4.20", - "quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@v0.6.1", + "quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@v0.7.4", ] [project.optional-dependencies] diff --git a/requirements.lock b/requirements.lock index aaad6ce..a175fb0 100644 --- a/requirements.lock +++ b/requirements.lock @@ -22,12 +22,16 @@ jsonschema==4.26.0 # via # quant-data-kit # quant-execution (pyproject.toml) +jsonschema-rs==0.52.1 + # via quant-data-kit jsonschema-specifications==2025.9.1 # via jsonschema numpy==2.2.6 # via # pandas # quant-data-kit +orjson==3.12.0 + # via quant-data-kit packaging==26.3 # via pytest pandas==2.3.3 @@ -54,7 +58,7 @@ pytz==2026.3.post1 # via pandas pyyaml==6.0.3 # via quant-data-kit -quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@v0.6.1 +quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@v0.7.4 # via quant-execution (pyproject.toml) referencing==0.37.0 # via @@ -78,6 +82,8 @@ typing-extensions==4.16.0 # referencing tzdata==2026.3 # via pandas +websockets==15.0.1 + # via quant-data-kit # The following packages are considered to be unsafe in a requirements file: setuptools==84.0.0 diff --git a/tests/test_benchmark_replay.py b/tests/test_benchmark_replay.py index 6b04967..ae8796d 100644 --- a/tests/test_benchmark_replay.py +++ b/tests/test_benchmark_replay.py @@ -49,7 +49,7 @@ def test_matching_worker_contract_uses_explicit_five_percent_fill_density(tmp_pa assert result["artifact_cleanup"] == "none" assert result["artifact_files_removed"] == 0 assert result["strict_verification_passed"] is True - assert result["dependencies"]["quant_data_kit"] == "0.6.1" + assert result["dependencies"]["quant_data_kit"] == "0.7.4" assert Path(result["artifact_path"]).is_dir() assert result["artifact_manifest_sha256"] assert result["artifact_file_sha256"] From e4dee06c63dc283f6c287b2617276d4af6ad690b Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Mon, 31 Aug 2026 01:05:56 +0800 Subject: [PATCH 09/13] docs: refresh M7 execution certification evidence --- docs/performance-m7-streaming.md | 18 +++--- validation/performance/m7-command-results.md | 30 +++++----- .../performance/m7-execution-final-10m.json | 56 +++++++++---------- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/docs/performance-m7-streaming.md b/docs/performance-m7-streaming.md index 7eb2666..b31a804 100644 --- a/docs/performance-m7-streaming.md +++ b/docs/performance-m7-streaming.md @@ -4,7 +4,7 @@ The candidate adds an artifact-retention-bounded Arrow path without replacing the frozen Python reference path. Correctness, compatibility and coverage gates pass. From clean commit -`99eac282b1d31e33828a2d18e0efa42f983ef049`, all three independent10-million-event processes pass +`36c130bef9829fcc6506941478d66f58f2ff73a0`, all three independent10-million-event processes pass the50,000-events/second and16GiB gates; no calibration result is promoted to release evidence. ## Architecture and compatibility @@ -66,28 +66,28 @@ to pass and its duration is reported. Every repeat is a fresh process; the rate process, not the median, to reach50,000 events/second. Peak memory is Windows process `PeakWorkingSetSize` and therefore includes Arrow and retained live replay state. -The official run sets `TEMP` and `TMP` to `F:\puresaber-m7-temp` and writes to a unique directory -under `F:\puresaber-m7-artifacts`. Every Arrow stream and canonical manifest is retained; the -benchmark has no automatic deletion mode. Exact machine, dependency, commit, dirty-state, timing, +The official run uses the recorded system temporary directory and writes to a unique directory under +`F:\puresaber-m7-artifacts`. Every Arrow stream and canonical manifest is retained; the benchmark +has no automatic deletion mode. Exact machine, dependency, commit, dirty-state, timing, output-volume, strict-verification and per-process fields live in the committed JSON evidence. ## Formal result | Run | Events/s | Peak working set | Strict reload | |---:|---:|---:|---| -| 1 | 59,340.69 | 2,233.67MiB | PASS | -| 2 | 62,627.03 | 2,234.61MiB | PASS | -| 3 | 52,966.75 | 2,236.26MiB | PASS | +| 1 | 62,920.17 | 2,242.47MiB | PASS | +| 2 | 64,106.74 | 2,233.68MiB | PASS | +| 3 | 61,356.14 | 2,240.03MiB | PASS | Each run processed10,000,000 events,500,000 fills and1,000,001 exact ledger transactions. All logical hashes, physical Arrow hashes and the manifest hash were identical across fresh processes. The three retained artifact directories contain1,501,955,792 bytes each. The machine-readable evidence is `validation/performance/m7-execution-final-10m.json`; its SHA-256 is -`a26c9f0eefeb9b0150c9c1ed93b8163a57ec603c8349082847b3927755bec634`. +`3bf5f9f18adfcf38489cf0d20d62ee8b561d2f4fc647bbe87bf21f92bab6a90f`. The Arrow buffers and writer queue are bounded, but replay identity sets and broker order/index state still scale with event or order count. The claim is therefore controlled memory at the -certified10-million-event envelope (maximum2,236.26MiB), not strict input-independent O(1) memory. +certified10-million-event envelope (maximum2,242.47MiB), not strict input-independent O(1) memory. ## Dense-stress limitation diff --git a/validation/performance/m7-command-results.md b/validation/performance/m7-command-results.md index f33590b..a802dfb 100644 --- a/validation/performance/m7-command-results.md +++ b/validation/performance/m7-command-results.md @@ -14,7 +14,8 @@ logical and physical hashes, retain its artifacts and pass strict post-run reloa ## Modified files - Runtime:`src/quant_execution/artifacts.py`, `broker.py`, `engine.py`, `ledger.py`, `__init__.py`. -- Contract/version:`pyproject.toml` (`0.5.0` and replay-artifact manifest schema`1.0.0`). +- Contract/version:`pyproject.toml` (`0.5.0`, replay-artifact manifest schema`1.0.0` and + `quant-data-kit@v0.7.4`) plus the regenerated `requirements.lock`. - Tests/CI:`tests/test_artifacts.py`, `tests/test_benchmark_replay.py`, `.github/workflows/ci.yml`. - Benchmark/docs:`benchmarks/benchmark_replay.py`, `README.md`, `docs/performance-m7-streaming.md`, this handoff and the final JSON report. @@ -56,15 +57,13 @@ python tools/check_branch_coverage.py coverage.json --threshold 90 \ ## Formal performance evidence -Source commit:`99eac282b1d31e33828a2d18e0efa42f983ef049`. +Source commit:`36c130bef9829fcc6506941478d66f58f2ff73a0`. ```text -TEMP=F:\puresaber-m7-temp -TMP=F:\puresaber-m7-temp python benchmarks/benchmark_replay.py --workload matching \ --matching-events 10000000 --repeat 3 --require-rate 50000 \ --memory-limit-gib 16 --artifact-mode arrow \ - --artifact-root F:\puresaber-m7-artifacts\execution-final2-10m-99eac28 \ + --artifact-root F:\puresaber-m7-artifacts\execution-v0.5.0-qdk-v0.7.4-36c130b \ --artifact-retention keep --artifact-batch-size 8192 \ --artifact-queue-batches 2 \ --output validation\performance\m7-execution-final-10m.json @@ -72,27 +71,28 @@ python benchmarks/benchmark_replay.py --workload matching \ | Run | Events/s | Peak working set | Strict reload | Dirty tree | |---:|---:|---:|---|---| -| 1 | 59,340.69 | 2,233.67MiB | PASS | false | -| 2 | 62,627.03 | 2,234.61MiB | PASS | false | -| 3 | 52,966.75 | 2,236.26MiB | PASS | false | +| 1 | 62,920.17 | 2,242.47MiB | PASS | false | +| 2 | 64,106.74 | 2,233.68MiB | PASS | false | +| 3 | 61,356.14 | 2,240.03MiB | PASS | false | -- Rate gate:PASS for every run; median59,340.69 events/second. -- Memory gate:PASS; maximum2,236.26MiB. +- Rate gate:PASS for every run; median62,920.17 events/second. +- Memory gate:PASS; maximum2,242.47MiB. - Each run:10,000,000 events,500,000 orders/fills,1,000,000 order events and 1,000,001 balanced ledger transactions; explicit fill density5% (`order_stride=20`). - Determinism:all logical hashes, every Arrow physical file hash and the manifest hash match across all three fresh processes. - Artifact manifest SHA-256:`c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27`. -- Final report SHA-256:`a26c9f0eefeb9b0150c9c1ed93b8163a57ec603c8349082847b3927755bec634`. -- Dependencies:Python3.12.5, PyArrow25.0.1, quant-data-kit distribution0.6.1. +- Final report SHA-256:`3bf5f9f18adfcf38489cf0d20d62ee8b561d2f4fc647bbe87bf21f92bab6a90f`. +- Dependencies:Python3.12.5, PyArrow25.0.1, quant-data-kit distribution0.7.4. - Machine:Windows11,16 logical CPUs; process peak working set includes Arrow and live replay state. - Retained artifacts:three directories,1,501,955,792 bytes each (about4.20GiB total), under - `F:\puresaber-m7-artifacts\execution-final2-10m-99eac28`; no file was automatically removed. + `F:\puresaber-m7-artifacts\execution-v0.5.0-qdk-v0.7.4-36c130b`; no file was automatically + removed. The timed interval includes event materialization, matching, risk, fill, fee, exact ledger, canonical serialization, Arrow initialization/write/seal, logical hashes and manifest close. Process startup, one static fixture-template construction and strict post-run reload are excluded; -strict reload is independently required and passed in3.18–3.32seconds per run. +strict reload is independently required and passed in2.95–3.03seconds per run. ## Dense stress and remaining risks @@ -110,7 +110,7 @@ Remaining risks: - physical Arrow hashes depend on the locked PyArrow serialization version and must be rebaselined, never silently accepted, after a dependency upgrade; - the dense50%-fill stress gate is still a known capacity limitation; -- the slowest representative run passed by only5.9%, so future releases must retain the per-run +- the slowest representative run passed by22.7%, so future releases must retain the per-run gate and15% regression comparison instead of relying on the median; - Arrow buffers are bounded, while event/fill identity sets and broker lookup/idempotency state still scale with input/order count; the certified claim is controlled memory at10M, not O(1); diff --git a/validation/performance/m7-execution-final-10m.json b/validation/performance/m7-execution-final-10m.json index 8205e01..b4099d8 100644 --- a/validation/performance/m7-execution-final-10m.json +++ b/validation/performance/m7-execution-final-10m.json @@ -26,9 +26,9 @@ "artifact_manifest_sha256": "c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27", "artifact_mode": "arrow", "artifact_paths": [ - "F:\\puresaber-m7-artifacts\\execution-final2-10m-99eac28\\matching_exact_ledger-10000000-2408", - "F:\\puresaber-m7-artifacts\\execution-final2-10m-99eac28\\matching_exact_ledger-10000000-38204", - "F:\\puresaber-m7-artifacts\\execution-final2-10m-99eac28\\matching_exact_ledger-10000000-10272" + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-qdk-v0.7.4-36c130b\\matching_exact_ledger-10000000-5472", + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-qdk-v0.7.4-36c130b\\matching_exact_ledger-10000000-42188", + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-qdk-v0.7.4-36c130b\\matching_exact_ledger-10000000-37080" ], "artifact_queue_batches": 2, "artifact_retention": [ @@ -37,30 +37,30 @@ "keep" ], "artifact_volume_free_gib_after_cleanup_runs": [ - 285.48, - 284.08, - 282.69 + 281.28, + 279.88, + 278.48 ], "artifact_volume_free_gib_before_cleanup_runs": [ - 285.48, - 284.08, - 282.69 + 281.28, + 279.88, + 278.48 ], "dependencies": { "pyarrow": "25.0.1", - "quant_data_kit": "0.6.1" + "quant_data_kit": "0.7.4" }, "events": 10000000, - "events_per_s_median": 59340.69, + "events_per_s_median": 62920.17, "events_per_s_runs": [ - 59340.69, - 62627.03, - 52966.75 + 62920.17, + 64106.74, + 61356.14 ], "fill_density": 0.05, "fill_sha256": "68932897888ac2f362bb46191fb2c94bdc7fcc99c46d45b5329c4671ab951dff", "fills": 500000, - "git_commit": "99eac282b1d31e33828a2d18e0efa42f983ef049", + "git_commit": "36c130bef9829fcc6506941478d66f58f2ff73a0", "git_dirty_runs": [ false, false, @@ -79,32 +79,32 @@ "order_sha256": "85e3b7ea894109a4380c478c344207dd57975d6d79ba22a8f41fbe190bc5c4d5", "order_stride": 20, "orders": 500000, - "peak_working_set_mib": 2236.26, + "peak_working_set_mib": 2242.47, "peak_working_set_mib_runs": [ - 2233.67, - 2234.61, - 2236.26 + 2242.47, + 2233.68, + 2240.03 ], "python": "3.12.5 (tags/v3.12.5:ff3bc82, Aug 6 2024, 20:45:27) [MSC v.1940 64 bit (AMD64)]", "rate_gate": true, "result_sha256": "cb5907d36a01178d57ad46f4743fed9b1546b2f13ead58b5ce6d016c0cb37db2", "strict_verification_elapsed_s_runs": [ - 3.186086, - 3.263401, - 3.317333 + 2.951991, + 3.006337, + 3.030089 ], "strict_verification_passed": true, "temp_directories": [ - "F:\\puresaber-m7-temp", - "F:\\puresaber-m7-temp", - "F:\\puresaber-m7-temp" + "C:\\Users\\1\\AppData\\Local\\Temp", + "C:\\Users\\1\\AppData\\Local\\Temp", + "C:\\Users\\1\\AppData\\Local\\Temp" ], "timing_scope": "includes event materialization, matching, risk, fill, fee, exact ledger, Arrow sink initialization/write/seal, logical hashes, and manifest close; excludes process startup and static fixture-template construction", "transactions": 1000001, "worker_pids": [ - 2408, - 38204, - 10272 + 5472, + 42188, + 37080 ], "workload": "matching_exact_ledger" } From 37260badebb1c637e7247d21dc0694e723f5d206 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Mon, 31 Aug 2026 01:20:28 +0800 Subject: [PATCH 10/13] fix: seal execution state after artifact finalization --- src/quant_execution/broker.py | 20 +++++++++- src/quant_execution/ledger.py | 23 ++++++++++- tests/test_artifacts.py | 74 ++++++++++++++++++++++++++++++++++- 3 files changed, 112 insertions(+), 5 deletions(-) diff --git a/src/quant_execution/broker.py b/src/quant_execution/broker.py index 63884d6..11f6aa7 100644 --- a/src/quant_execution/broker.py +++ b/src/quant_execution/broker.py @@ -119,10 +119,12 @@ def reset(self) -> None: self._events: list[OrderEvent] = [] self._accepted_day: dict[str, date] = {} self._artifact_sink = None + self._artifact_stream_closed = False def start_artifact_stream(self, sink: object) -> None: """Route immutable history to a bounded sink while retaining lookup/idempotency state.""" + self._require_mutable() if self._orders or self._events or self._artifact_sink is not None: raise ValidationError("broker artifact streaming must start immediately after reset") if not callable(getattr(sink, "append", None)): @@ -138,9 +140,16 @@ def finish_artifact_stream(self) -> None: for order in self.open_orders: sink.append("orders", order_bytes(order)) self._artifact_sink = None + self._artifact_stream_closed = True def abort_artifact_stream(self) -> None: - self._artifact_sink = None + if self._artifact_sink is not None: + self._artifact_sink = None + self._artifact_stream_closed = True + + def _require_mutable(self) -> None: + if self._artifact_stream_closed: + raise ValidationError("broker artifact stream is closed; reset is required") def _record_event(self, event: OrderEvent, order: Order) -> None: sink = self._artifact_sink @@ -166,6 +175,7 @@ def capture_state(self) -> dict[str, object]: "fill_keys": self._fill_keys, "events": self._events, "accepted_day": self._accepted_day, + "artifact_stream_closed": self._artifact_stream_closed, } ) @@ -181,6 +191,7 @@ def restore_state(self, state: dict[str, object]) -> None: self._fill_keys = restored["fill_keys"] self._events = restored["events"] self._accepted_day = restored["accepted_day"] + self._artifact_stream_closed = restored["artifact_stream_closed"] @property def orders(self) -> tuple[Order, ...]: @@ -239,6 +250,7 @@ def _intent_hash(intent: OrderIntent) -> str: return hashlib.sha256(_intent_bytes(intent)).hexdigest() def submit(self, order_intent: OrderIntent) -> Order: + self._require_mutable() if not isinstance(order_intent, OrderIntent): raise ValidationError("order_intent must be an OrderIntent") semantic_hash = self._intent_hash(order_intent) @@ -273,6 +285,7 @@ def submit(self, order_intent: OrderIntent) -> Order: return accepted def reject(self, order_intent: OrderIntent, *, code: str, message: str = "") -> Order: + self._require_mutable() if not code.strip(): raise ValidationError("rejection code is required") semantic_hash = self._intent_hash(order_intent) @@ -305,6 +318,7 @@ def cancel( idempotency_key: str, created_at: datetime, ) -> OrderEvent: + self._require_mutable() if not idempotency_key.strip(): raise ValidationError("cancel idempotency_key is required") prior = self._cancel_keys.get(idempotency_key) @@ -332,6 +346,7 @@ def cancel( return event def apply_fill(self, fill: Fill, *, trusted_unique: bool = False) -> OrderEvent: + self._require_mutable() prior = None if trusted_unique else self._fill_keys.get(fill.fill_id) if prior is not None: if isinstance(prior[0], bytes): @@ -401,6 +416,7 @@ def apply_fill(self, fill: Fill, *, trusted_unique: bool = False) -> OrderEvent: return event def expire(self, order_id: str, *, event_time: datetime, reason: str) -> OrderEvent: + self._require_mutable() order = self._require_order(order_id) if order.status not in {OrderStatus.ACCEPTED, OrderStatus.PARTIALLY_FILLED}: raise ValidationError("only open orders can expire") @@ -419,12 +435,14 @@ def expire(self, order_id: str, *, event_time: datetime, reason: str) -> OrderEv return event def note_trading_day(self, order_id: str, trading_day: date) -> None: + self._require_mutable() order = self._orders.get(order_id) if order is not None and order.intent.time_in_force is TimeInForce.DAY: self._accepted_day.setdefault(order_id, trading_day) self._day_order_ids.add(order_id) def expire_day_orders(self, trading_day: date, event_time: datetime) -> tuple[OrderEvent, ...]: + self._require_mutable() if not self._day_order_ids: return () expired: list[OrderEvent] = [] diff --git a/src/quant_execution/ledger.py b/src/quant_execution/ledger.py index 01092c3..009711a 100644 --- a/src/quant_execution/ledger.py +++ b/src/quant_execution/ledger.py @@ -131,6 +131,7 @@ def reset(self, *, opened_at: datetime | None = None) -> None: self._transactions: list[LedgerTransaction] = [] self._transaction_count = 0 self._artifact_sink = None + self._artifact_stream_closed = False self._finalized_journal_sha256: str | None = None self._transaction_keys: set[str] = set() self._event_fingerprints: dict[str, LedgerEvent | bytes] = {} @@ -171,6 +172,7 @@ def reset(self, *, opened_at: datetime | None = None) -> None: def start_artifact_stream(self, sink: object) -> None: """Move journal retention to a bounded artifact sink after reset.""" + self._require_mutable() if self._artifact_sink is not None: raise ValidationError("ledger artifact stream is already active") if not callable(getattr(sink, "append", None)): @@ -197,16 +199,24 @@ def finish_artifact_stream(self, *, journal_sha256: str | None = None) -> str | raise ValidationError("finalized ledger journal hash must be lowercase SHA-256") self._finalized_journal_sha256 = journal_sha256 self._artifact_sink = None + self._artifact_stream_closed = True return journal_sha256 def abort_artifact_stream(self) -> None: - self._artifact_sink = None + if self._artifact_sink is not None: + self._artifact_sink = None + self._artifact_stream_closed = True + + def _require_mutable(self) -> None: + if self._artifact_stream_closed: + raise ValidationError("ledger artifact stream is closed; reset is required") def capture_state(self) -> dict[str, object]: return deepcopy( { "transactions": self._transactions, "transaction_count": self._transaction_count, + "artifact_stream_closed": self._artifact_stream_closed, "finalized_journal_sha256": self._finalized_journal_sha256, "transaction_keys": self._transaction_keys, "event_fingerprints": self._event_fingerprints, @@ -229,6 +239,7 @@ def restore_state(self, state: dict[str, object]) -> None: restored = deepcopy(state) self._transactions = restored["transactions"] self._transaction_count = restored["transaction_count"] + self._artifact_stream_closed = restored["artifact_stream_closed"] self._finalized_journal_sha256 = restored["finalized_journal_sha256"] self._transaction_keys = restored["transaction_keys"] self._event_fingerprints = restored["event_fingerprints"] @@ -253,7 +264,7 @@ def transactions(self) -> tuple[LedgerTransaction, ...]: def transaction_count(self) -> int: return ( self._transaction_count - if self._artifact_sink is not None or self._finalized_journal_sha256 is not None + if self._artifact_sink is not None or self._artifact_stream_closed else len(self._transactions) ) @@ -263,6 +274,10 @@ def journal_sha256(self) -> str: return self._finalized_journal_sha256 if self._artifact_sink is not None: raise ValidationError("ledger journal hash is unavailable until artifact finalization") + if self._artifact_stream_closed: + raise ValidationError( + "ledger journal hash is unavailable after artifact abort; reset required" + ) digest = hashlib.sha256() digest.update(b'{"fx_snapshots":[') for index, (currency, rate, event_time) in enumerate(self._fx_history): @@ -330,6 +345,7 @@ def _transaction_bytes(transaction: LedgerTransaction) -> bytes: ).encode() def set_fx_rate(self, currency: str, rate: FixedPoint, *, event_time: datetime) -> None: + self._require_mutable() currency = _currency(currency) event_time = ensure_utc_datetime(event_time, field="event_time") value = decimal(rate) @@ -486,6 +502,7 @@ def portfolio_risk_snapshot(self, event_time: datetime) -> PortfolioRiskSnapshot def mark( self, event: MarkPriceEvent, *, create_snapshot: bool = True ) -> AccountSnapshot | None: + self._require_mutable() if event.instrument_id not in self.instruments: raise ValidationError(f"missing InstrumentSpec for {event.instrument_id}") prior = self._mark_fingerprints.get(event.event_id) @@ -525,6 +542,7 @@ def observe_market( create_snapshot: bool = True, trusted_unique: bool = False, ) -> AccountSnapshot | None: + self._require_mutable() if isinstance(event, MarkPriceEvent): return self.mark(event, create_snapshot=create_snapshot) price: FixedPoint | None = None @@ -635,6 +653,7 @@ def _apply( local_rollback: bool = True, trusted_unique: bool = False, ) -> AccountSnapshot | None: + self._require_mutable() self._validate_event(event) reference_id = self._event_identity(event) prior = None if trusted_unique else self._event_fingerprints.get(reference_id) diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py index da2c280..8493ddf 100644 --- a/tests/test_artifacts.py +++ b/tests/test_artifacts.py @@ -5,8 +5,8 @@ from datetime import date, timedelta import pytest -from conftest import T0, fp -from quant_data_kit import CorporateActionEvent, FundingRateEvent, StatusEvent +from conftest import T0, event_fields, fp +from quant_data_kit import CorporateActionEvent, FundingRateEvent, MarkPriceEvent, StatusEvent from quant_data_kit.exceptions import ValidationError from test_engine import ( FixtureStrategy, @@ -199,6 +199,16 @@ def test_streaming_broker_fill_compaction_and_lifecycle_guards(tmp_path) -> None broker._live_order(order.order_id) broker._open_order_ids.clear() broker.finish_artifact_stream() + with pytest.raises(ValidationError, match="stream is closed"): + broker.submit(intent) + with pytest.raises(ValidationError, match="stream is closed"): + broker.apply_fill(fill) + with pytest.raises(ValidationError, match="stream is closed"): + broker.note_trading_day(order.order_id, T0.date()) + with pytest.raises(ValidationError, match="stream is closed"): + broker.expire_day_orders(T0.date(), T0 + timedelta(days=1)) + with pytest.raises(ValidationError, match="stream is closed"): + broker.start_artifact_stream(sink) broker.finish_artifact_stream() sink.close({"run_id": "broker-fill"}) @@ -711,9 +721,69 @@ def test_streaming_ledger_compact_idempotency_and_stream_guards(tmp_path) -> Non assert journal_sha256 == ledger.journal_sha256 assert ledger.transaction_count == 2 assert ledger.finish_artifact_stream() == journal_sha256 + sealed_snapshot = ledger.snapshot() + with pytest.raises(ValidationError, match="stream is closed"): + ledger.apply_with_trading_day(fill, trading_day=T0.date(), create_snapshot=False) + with pytest.raises(ValidationError, match="stream is closed"): + ledger.mark( + MarkPriceEvent( + **event_fields("sealed-mark", spot.instrument_id, seconds=1), + price=fp("101"), + ) + ) + with pytest.raises(ValidationError, match="stream is closed"): + ledger.set_fx_rate("USD", fp("1"), event_time=T0 + timedelta(seconds=1)) + with pytest.raises(ValidationError, match="stream is closed"): + ledger.start_artifact_stream(sink) + assert ledger.snapshot() == sealed_snapshot + assert ledger.transaction_count == 2 + assert ledger.journal_sha256 == journal_sha256 sink.close({"run_id": "ledger-compact"}) +def test_aborted_artifact_components_require_reset_before_reuse(tmp_path) -> None: + intent = OrderIntent( + idempotency_key="after-abort", + account_id="account", + strategy_id="strategy", + instrument_id=SPOT, + side=Side.BUY, + quantity=fp("1.000", 3), + order_type=OrderType.LIMIT, + time_in_force=TimeInForce.GTC, + created_at=T0, + limit_price=fp("100"), + ) + broker = DeterministicBroker() + broker_sink = ArrowReplayArtifactSink(tmp_path / "aborted-broker", batch_size=1) + broker.start_artifact_stream(broker_sink) + broker.abort_artifact_stream() + with pytest.raises(ValidationError, match="stream is closed"): + broker.submit(intent) + broker.reset() + assert broker.submit(intent).intent == intent + broker_sink.abort() + + spot = specs()[SPOT] + ledger = ExactAccountLedger( + account_id="account", + base_currency="USDT", + instruments={spot.instrument_id: spot}, + initial_cash={"USDT": fp("1000")}, + ) + ledger_sink = ArrowReplayArtifactSink(tmp_path / "aborted-ledger", batch_size=1) + ledger.start_artifact_stream(ledger_sink) + ledger.abort_artifact_stream() + assert ledger.transaction_count == 1 + with pytest.raises(ValidationError, match="unavailable after artifact abort"): + _ = ledger.journal_sha256 + with pytest.raises(ValidationError, match="stream is closed"): + ledger.set_fx_rate("USD", fp("1"), event_time=T0) + ledger.reset() + ledger.set_fx_rate("USD", fp("1"), event_time=T0) + ledger_sink.abort() + + def test_streaming_corporate_funding_settlement_and_custom_gate_paths(tmp_path) -> None: registry = {STOCK: specs()[STOCK]} corporate = engine_for( From 3f4c224e30ecf5579b2f9f09069e4439ccdc8312 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Mon, 31 Aug 2026 01:29:52 +0800 Subject: [PATCH 11/13] docs: recertify sealed execution lifecycle --- docs/performance-m7-streaming.md | 14 +++--- validation/performance/m7-command-results.md | 33 +++++++------ .../performance/m7-execution-final-10m.json | 48 +++++++++---------- 3 files changed, 51 insertions(+), 44 deletions(-) diff --git a/docs/performance-m7-streaming.md b/docs/performance-m7-streaming.md index b31a804..68d0395 100644 --- a/docs/performance-m7-streaming.md +++ b/docs/performance-m7-streaming.md @@ -4,7 +4,7 @@ The candidate adds an artifact-retention-bounded Arrow path without replacing the frozen Python reference path. Correctness, compatibility and coverage gates pass. From clean commit -`36c130bef9829fcc6506941478d66f58f2ff73a0`, all three independent10-million-event processes pass +`37260badebb1c637e7247d21dc0694e723f5d206`, all three independent10-million-event processes pass the50,000-events/second and16GiB gates; no calibration result is promoted to release evidence. ## Architecture and compatibility @@ -24,6 +24,8 @@ the50,000-events/second and16GiB gates; no calibration result is promoted to rel - `StoredRunArtifacts` exposes lazy byte and JSON iterators; no consumer is forced to reconstruct the complete Python object graph; - incomplete runs retain `FAILED.json`; only a sealed and closed run receives a complete manifest; +- finalized or aborted broker and ledger streams reject every subsequent mutation until an + explicit reset, so live state cannot diverge from sealed artifacts or finalized journal hashes; - manifest publication is atomic and no-clobber, and strict reload verifies canonical manifest bytes, manifest hash, physical file size/hash, Arrow schema, contiguous sequence and logical hash. @@ -75,19 +77,19 @@ output-volume, strict-verification and per-process fields live in the committed | Run | Events/s | Peak working set | Strict reload | |---:|---:|---:|---| -| 1 | 62,920.17 | 2,242.47MiB | PASS | -| 2 | 64,106.74 | 2,233.68MiB | PASS | -| 3 | 61,356.14 | 2,240.03MiB | PASS | +| 1 | 64,746.49 | 2,239.19MiB | PASS | +| 2 | 62,394.71 | 2,242.02MiB | PASS | +| 3 | 64,627.91 | 2,236.11MiB | PASS | Each run processed10,000,000 events,500,000 fills and1,000,001 exact ledger transactions. All logical hashes, physical Arrow hashes and the manifest hash were identical across fresh processes. The three retained artifact directories contain1,501,955,792 bytes each. The machine-readable evidence is `validation/performance/m7-execution-final-10m.json`; its SHA-256 is -`3bf5f9f18adfcf38489cf0d20d62ee8b561d2f4fc647bbe87bf21f92bab6a90f`. +`6c74790bb3b9d5a20b95ba07989feb0eb6a265a211970577c6092559aaf47cb2`. The Arrow buffers and writer queue are bounded, but replay identity sets and broker order/index state still scale with event or order count. The claim is therefore controlled memory at the -certified10-million-event envelope (maximum2,242.47MiB), not strict input-independent O(1) memory. +certified10-million-event envelope (maximum2,242.02MiB), not strict input-independent O(1) memory. ## Dense-stress limitation diff --git a/validation/performance/m7-command-results.md b/validation/performance/m7-command-results.md index a802dfb..f54c9f5 100644 --- a/validation/performance/m7-command-results.md +++ b/validation/performance/m7-command-results.md @@ -32,6 +32,11 @@ transaction count and journal hash consistent with the stored ledger. Regression both states. The same review also required the memory claim to be narrowed from strict O(1) to the measured10-million-event envelope; the runtime and documentation now use that precise scope. +The dependency recertification review then found that finalized ledger state could still mutate +behind its fixed journal hash. Commit `37260badebb1c637e7247d21dc0694e723f5d206` closes the lifecycle +for both ledger and broker:finalization or abort makes all mutation entry points fail closed until +`reset()`, and regression tests prove hashes, counts and snapshots cannot diverge after sealing. + ## Tests and coverage Locked local environment: @@ -49,21 +54,21 @@ python tools/check_branch_coverage.py coverage.json --threshold 90 \ - `pip check`:PASS. - Ruff format/check:PASS. -- Python3.12:200 passed; total coverage95.44%. -- Pure branch coverage:artifacts94.74%, broker95.35%, contracts91.88%, schemas94.74%, - engine91.84%, matching93.25%, state_machine96.67%, ledger91.47%, rules92.38%. +- Python3.12:201 passed; total coverage95.51%. +- Pure branch coverage:artifacts94.74%, broker95.56%, contracts91.88%, schemas94.74%, + engine91.84%, matching93.25%, state_machine96.67%, ledger91.91%, rules92.38%. - GitHub Actions run`33251586735`:Python3.10/3.11/3.12 all PASS. - GitHub Actions run`33251588866`:Python3.10/3.11/3.12 all PASS. ## Formal performance evidence -Source commit:`36c130bef9829fcc6506941478d66f58f2ff73a0`. +Source commit:`37260badebb1c637e7247d21dc0694e723f5d206`. ```text python benchmarks/benchmark_replay.py --workload matching \ --matching-events 10000000 --repeat 3 --require-rate 50000 \ --memory-limit-gib 16 --artifact-mode arrow \ - --artifact-root F:\puresaber-m7-artifacts\execution-v0.5.0-qdk-v0.7.4-36c130b \ + --artifact-root F:\puresaber-m7-artifacts\execution-v0.5.0-sealed-37260ba \ --artifact-retention keep --artifact-batch-size 8192 \ --artifact-queue-batches 2 \ --output validation\performance\m7-execution-final-10m.json @@ -71,28 +76,28 @@ python benchmarks/benchmark_replay.py --workload matching \ | Run | Events/s | Peak working set | Strict reload | Dirty tree | |---:|---:|---:|---|---| -| 1 | 62,920.17 | 2,242.47MiB | PASS | false | -| 2 | 64,106.74 | 2,233.68MiB | PASS | false | -| 3 | 61,356.14 | 2,240.03MiB | PASS | false | +| 1 | 64,746.49 | 2,239.19MiB | PASS | false | +| 2 | 62,394.71 | 2,242.02MiB | PASS | false | +| 3 | 64,627.91 | 2,236.11MiB | PASS | false | -- Rate gate:PASS for every run; median62,920.17 events/second. -- Memory gate:PASS; maximum2,242.47MiB. +- Rate gate:PASS for every run; median64,627.91 events/second. +- Memory gate:PASS; maximum2,242.02MiB. - Each run:10,000,000 events,500,000 orders/fills,1,000,000 order events and 1,000,001 balanced ledger transactions; explicit fill density5% (`order_stride=20`). - Determinism:all logical hashes, every Arrow physical file hash and the manifest hash match across all three fresh processes. - Artifact manifest SHA-256:`c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27`. -- Final report SHA-256:`3bf5f9f18adfcf38489cf0d20d62ee8b561d2f4fc647bbe87bf21f92bab6a90f`. +- Final report SHA-256:`6c74790bb3b9d5a20b95ba07989feb0eb6a265a211970577c6092559aaf47cb2`. - Dependencies:Python3.12.5, PyArrow25.0.1, quant-data-kit distribution0.7.4. - Machine:Windows11,16 logical CPUs; process peak working set includes Arrow and live replay state. - Retained artifacts:three directories,1,501,955,792 bytes each (about4.20GiB total), under - `F:\puresaber-m7-artifacts\execution-v0.5.0-qdk-v0.7.4-36c130b`; no file was automatically + `F:\puresaber-m7-artifacts\execution-v0.5.0-sealed-37260ba`; no file was automatically removed. The timed interval includes event materialization, matching, risk, fill, fee, exact ledger, canonical serialization, Arrow initialization/write/seal, logical hashes and manifest close. Process startup, one static fixture-template construction and strict post-run reload are excluded; -strict reload is independently required and passed in2.95–3.03seconds per run. +strict reload is independently required and passed in2.97–3.00seconds per run. ## Dense stress and remaining risks @@ -110,7 +115,7 @@ Remaining risks: - physical Arrow hashes depend on the locked PyArrow serialization version and must be rebaselined, never silently accepted, after a dependency upgrade; - the dense50%-fill stress gate is still a known capacity limitation; -- the slowest representative run passed by22.7%, so future releases must retain the per-run +- the slowest representative run passed by24.8%, so future releases must retain the per-run gate and15% regression comparison instead of relying on the median; - Arrow buffers are bounded, while event/fill identity sets and broker lookup/idempotency state still scale with input/order count; the certified claim is controlled memory at10M, not O(1); diff --git a/validation/performance/m7-execution-final-10m.json b/validation/performance/m7-execution-final-10m.json index b4099d8..7895b40 100644 --- a/validation/performance/m7-execution-final-10m.json +++ b/validation/performance/m7-execution-final-10m.json @@ -26,9 +26,9 @@ "artifact_manifest_sha256": "c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27", "artifact_mode": "arrow", "artifact_paths": [ - "F:\\puresaber-m7-artifacts\\execution-v0.5.0-qdk-v0.7.4-36c130b\\matching_exact_ledger-10000000-5472", - "F:\\puresaber-m7-artifacts\\execution-v0.5.0-qdk-v0.7.4-36c130b\\matching_exact_ledger-10000000-42188", - "F:\\puresaber-m7-artifacts\\execution-v0.5.0-qdk-v0.7.4-36c130b\\matching_exact_ledger-10000000-37080" + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-37260ba\\matching_exact_ledger-10000000-47140", + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-37260ba\\matching_exact_ledger-10000000-22420", + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-37260ba\\matching_exact_ledger-10000000-44048" ], "artifact_queue_batches": 2, "artifact_retention": [ @@ -37,30 +37,30 @@ "keep" ], "artifact_volume_free_gib_after_cleanup_runs": [ - 281.28, - 279.88, - 278.48 + 277.08, + 275.69, + 274.29 ], "artifact_volume_free_gib_before_cleanup_runs": [ - 281.28, - 279.88, - 278.48 + 277.08, + 275.69, + 274.29 ], "dependencies": { "pyarrow": "25.0.1", "quant_data_kit": "0.7.4" }, "events": 10000000, - "events_per_s_median": 62920.17, + "events_per_s_median": 64627.91, "events_per_s_runs": [ - 62920.17, - 64106.74, - 61356.14 + 64746.49, + 62394.71, + 64627.91 ], "fill_density": 0.05, "fill_sha256": "68932897888ac2f362bb46191fb2c94bdc7fcc99c46d45b5329c4671ab951dff", "fills": 500000, - "git_commit": "36c130bef9829fcc6506941478d66f58f2ff73a0", + "git_commit": "37260badebb1c637e7247d21dc0694e723f5d206", "git_dirty_runs": [ false, false, @@ -79,19 +79,19 @@ "order_sha256": "85e3b7ea894109a4380c478c344207dd57975d6d79ba22a8f41fbe190bc5c4d5", "order_stride": 20, "orders": 500000, - "peak_working_set_mib": 2242.47, + "peak_working_set_mib": 2242.02, "peak_working_set_mib_runs": [ - 2242.47, - 2233.68, - 2240.03 + 2239.19, + 2242.02, + 2236.11 ], "python": "3.12.5 (tags/v3.12.5:ff3bc82, Aug 6 2024, 20:45:27) [MSC v.1940 64 bit (AMD64)]", "rate_gate": true, "result_sha256": "cb5907d36a01178d57ad46f4743fed9b1546b2f13ead58b5ce6d016c0cb37db2", "strict_verification_elapsed_s_runs": [ - 2.951991, - 3.006337, - 3.030089 + 2.998672, + 2.996893, + 2.971207 ], "strict_verification_passed": true, "temp_directories": [ @@ -102,9 +102,9 @@ "timing_scope": "includes event materialization, matching, risk, fill, fee, exact ledger, Arrow sink initialization/write/seal, logical hashes, and manifest close; excludes process startup and static fixture-template construction", "transactions": 1000001, "worker_pids": [ - 5472, - 42188, - 37080 + 47140, + 22420, + 44048 ], "workload": "matching_exact_ledger" } From 23f6b19cc1c4e42fb9df8ce600ad34b254a121a5 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Mon, 31 Aug 2026 01:43:31 +0800 Subject: [PATCH 12/13] fix: close sealed state mutation bypasses --- src/quant_execution/broker.py | 6 ++++++ src/quant_execution/engine.py | 4 ++++ src/quant_execution/ledger.py | 37 ++++++++++++++++++++++++++++------- tests/test_artifacts.py | 25 +++++++++++++++++++++++ 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/src/quant_execution/broker.py b/src/quant_execution/broker.py index 11f6aa7..c9d6575 100644 --- a/src/quant_execution/broker.py +++ b/src/quant_execution/broker.py @@ -180,6 +180,12 @@ def capture_state(self) -> dict[str, object]: ) def restore_state(self, state: dict[str, object]) -> None: + """Restore a mutable checkpoint; sealed lifecycle recovery is engine-internal.""" + + self._require_mutable() + self._restore_captured_state(state) + + def _restore_captured_state(self, state: dict[str, object]) -> None: restored = deepcopy(state) self._orders = restored["orders"] self._order_count = restored["order_count"] diff --git a/src/quant_execution/engine.py b/src/quant_execution/engine.py index 36a0315..17c07b0 100644 --- a/src/quant_execution/engine.py +++ b/src/quant_execution/engine.py @@ -770,6 +770,10 @@ def _capture_component(component: object) -> tuple[str, object]: def _restore_component(component: object, checkpoint: tuple[str, object]) -> None: mode, state = checkpoint if mode == "explicit": + trusted_restore = getattr(component, "_restore_captured_state", None) + if callable(trusted_restore): + trusted_restore(state) + return restore = getattr(component, "restore_state", None) if not callable(restore): raise ValidationError(f"component {type(component).__name__} lost restore_state") diff --git a/src/quant_execution/ledger.py b/src/quant_execution/ledger.py index 009711a..250e897 100644 --- a/src/quant_execution/ledger.py +++ b/src/quant_execution/ledger.py @@ -9,6 +9,7 @@ from datetime import date, datetime, timezone from decimal import ROUND_HALF_EVEN, Decimal from functools import lru_cache +from types import MappingProxyType from quant_data_kit import ( AssetClass, @@ -104,15 +105,15 @@ def __init__( raise ValidationError("account_id and base_currency are required") if not 0 <= money_scale <= 18: raise ValidationError("money_scale must be in [0, 18]") - self.account_id = account_id - self.base_currency = _currency(base_currency, "base_currency") - self.instruments = dict(instruments) + self._account_id = account_id + self._base_currency = _currency(base_currency, "base_currency") + self._instruments = MappingProxyType(dict(instruments)) self._derivative_instruments = frozenset( instrument_id - for instrument_id, spec in self.instruments.items() + for instrument_id, spec in self._instruments.items() if self._is_derivative(spec) ) - self.money_scale = money_scale + self._money_scale = money_scale self._initial_cash = dict(initial_cash or {}) self._initial_fx = dict(fx_to_base or {}) self._default_opened_at = ( @@ -236,6 +237,12 @@ def capture_state(self) -> dict[str, object]: ) def restore_state(self, state: dict[str, object]) -> None: + """Restore a mutable checkpoint; sealed lifecycle recovery is engine-internal.""" + + self._require_mutable() + self._restore_captured_state(state) + + def _restore_captured_state(self, state: dict[str, object]) -> None: restored = deepcopy(state) self._transactions = restored["transactions"] self._transaction_count = restored["transaction_count"] @@ -260,6 +267,22 @@ def restore_state(self, state: dict[str, object]) -> None: def transactions(self) -> tuple[LedgerTransaction, ...]: return tuple(self._transactions) + @property + def account_id(self) -> str: + return self._account_id + + @property + def base_currency(self) -> str: + return self._base_currency + + @property + def money_scale(self) -> int: + return self._money_scale + + @property + def instruments(self) -> Mapping[str, InstrumentSpec]: + return self._instruments + @property def transaction_count(self) -> int: return ( @@ -388,7 +411,7 @@ def has_open_derivative_position(self) -> bool: def risk_balances( self, event_time: datetime - ) -> tuple[dict[str, Decimal], dict[str, Decimal], Decimal, Decimal]: + ) -> tuple[dict[str, Decimal], Mapping[str, Decimal], Decimal, Decimal]: """Return exact decimal balances needed by the hot pre-trade risk path.""" event_time = ensure_utc_datetime(event_time, field="event_time") cash = { @@ -423,7 +446,7 @@ def risk_balances( spec.settlement_currency, event_time, ) - return cash, self._positions, nav, initial_margin + return cash, MappingProxyType(self._positions), nav, initial_margin def portfolio_risk_snapshot(self, event_time: datetime) -> PortfolioRiskSnapshot: """Build an exact, read-only base-currency exposure view at one PIT timestamp.""" diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py index 8493ddf..ff61433 100644 --- a/tests/test_artifacts.py +++ b/tests/test_artifacts.py @@ -198,6 +198,7 @@ def test_streaming_broker_fill_compaction_and_lifecycle_guards(tmp_path) -> None with pytest.raises(TypeError, match="terminal order"): broker._live_order(order.order_id) broker._open_order_ids.clear() + mutable_checkpoint = broker.capture_state() broker.finish_artifact_stream() with pytest.raises(ValidationError, match="stream is closed"): broker.submit(intent) @@ -209,6 +210,8 @@ def test_streaming_broker_fill_compaction_and_lifecycle_guards(tmp_path) -> None broker.expire_day_orders(T0.date(), T0 + timedelta(days=1)) with pytest.raises(ValidationError, match="stream is closed"): broker.start_artifact_stream(sink) + with pytest.raises(ValidationError, match="stream is closed"): + broker.restore_state(mutable_checkpoint) broker.finish_artifact_stream() sink.close({"run_id": "broker-fill"}) @@ -715,6 +718,7 @@ def test_streaming_ledger_compact_idempotency_and_stream_guards(tmp_path) -> Non trading_day=T0.date(), ) assert ledger.transaction_count == 2 + mutable_checkpoint = ledger.capture_state() with pytest.raises(ValidationError, match="lowercase SHA-256"): ledger.finish_artifact_stream(journal_sha256="bad") journal_sha256 = ledger.finish_artifact_stream() @@ -735,6 +739,21 @@ def test_streaming_ledger_compact_idempotency_and_stream_guards(tmp_path) -> Non ledger.set_fx_rate("USD", fp("1"), event_time=T0 + timedelta(seconds=1)) with pytest.raises(ValidationError, match="stream is closed"): ledger.start_artifact_stream(sink) + with pytest.raises(ValidationError, match="stream is closed"): + ledger.restore_state(mutable_checkpoint) + _, sealed_positions, _, _ = ledger.risk_balances(T0) + with pytest.raises(TypeError): + sealed_positions[spot.instrument_id] = sealed_positions[spot.instrument_id] * 2 + with pytest.raises(TypeError): + ledger.instruments[spot.instrument_id] = spot + with pytest.raises(AttributeError): + ledger.instruments = {} + with pytest.raises(AttributeError): + ledger.account_id = "other" + with pytest.raises(AttributeError): + ledger.base_currency = "USD" + with pytest.raises(AttributeError): + ledger.money_scale = 2 assert ledger.snapshot() == sealed_snapshot assert ledger.transaction_count == 2 assert ledger.journal_sha256 == journal_sha256 @@ -757,9 +776,12 @@ def test_aborted_artifact_components_require_reset_before_reuse(tmp_path) -> Non broker = DeterministicBroker() broker_sink = ArrowReplayArtifactSink(tmp_path / "aborted-broker", batch_size=1) broker.start_artifact_stream(broker_sink) + broker_checkpoint = broker.capture_state() broker.abort_artifact_stream() with pytest.raises(ValidationError, match="stream is closed"): broker.submit(intent) + with pytest.raises(ValidationError, match="stream is closed"): + broker.restore_state(broker_checkpoint) broker.reset() assert broker.submit(intent).intent == intent broker_sink.abort() @@ -773,12 +795,15 @@ def test_aborted_artifact_components_require_reset_before_reuse(tmp_path) -> Non ) ledger_sink = ArrowReplayArtifactSink(tmp_path / "aborted-ledger", batch_size=1) ledger.start_artifact_stream(ledger_sink) + ledger_checkpoint = ledger.capture_state() ledger.abort_artifact_stream() assert ledger.transaction_count == 1 with pytest.raises(ValidationError, match="unavailable after artifact abort"): _ = ledger.journal_sha256 with pytest.raises(ValidationError, match="stream is closed"): ledger.set_fx_rate("USD", fp("1"), event_time=T0) + with pytest.raises(ValidationError, match="stream is closed"): + ledger.restore_state(ledger_checkpoint) ledger.reset() ledger.set_fx_rate("USD", fp("1"), event_time=T0) ledger_sink.abort() From f43489e62710da6c12e49f4fc5fa874ba6494b7a Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Mon, 31 Aug 2026 01:52:59 +0800 Subject: [PATCH 13/13] docs: recertify immutable sealed state --- docs/performance-m7-streaming.md | 12 ++--- validation/performance/m7-command-results.md | 33 +++++++------ .../performance/m7-execution-final-10m.json | 48 +++++++++---------- 3 files changed, 48 insertions(+), 45 deletions(-) diff --git a/docs/performance-m7-streaming.md b/docs/performance-m7-streaming.md index 68d0395..ded4ed5 100644 --- a/docs/performance-m7-streaming.md +++ b/docs/performance-m7-streaming.md @@ -4,7 +4,7 @@ The candidate adds an artifact-retention-bounded Arrow path without replacing the frozen Python reference path. Correctness, compatibility and coverage gates pass. From clean commit -`37260badebb1c637e7247d21dc0694e723f5d206`, all three independent10-million-event processes pass +`23f6b19cc1c4e42fb9df8ce600ad34b254a121a5`, all three independent10-million-event processes pass the50,000-events/second and16GiB gates; no calibration result is promoted to release evidence. ## Architecture and compatibility @@ -77,19 +77,19 @@ output-volume, strict-verification and per-process fields live in the committed | Run | Events/s | Peak working set | Strict reload | |---:|---:|---:|---| -| 1 | 64,746.49 | 2,239.19MiB | PASS | -| 2 | 62,394.71 | 2,242.02MiB | PASS | -| 3 | 64,627.91 | 2,236.11MiB | PASS | +| 1 | 58,221.53 | 2,244.92MiB | PASS | +| 2 | 63,906.96 | 2,248.48MiB | PASS | +| 3 | 61,473.01 | 2,243.24MiB | PASS | Each run processed10,000,000 events,500,000 fills and1,000,001 exact ledger transactions. All logical hashes, physical Arrow hashes and the manifest hash were identical across fresh processes. The three retained artifact directories contain1,501,955,792 bytes each. The machine-readable evidence is `validation/performance/m7-execution-final-10m.json`; its SHA-256 is -`6c74790bb3b9d5a20b95ba07989feb0eb6a265a211970577c6092559aaf47cb2`. +`4dcb33d68126bc454ce68e11ae3d4f4a8475a168bf43b644ee1170c394efc41d`. The Arrow buffers and writer queue are bounded, but replay identity sets and broker order/index state still scale with event or order count. The claim is therefore controlled memory at the -certified10-million-event envelope (maximum2,242.02MiB), not strict input-independent O(1) memory. +certified10-million-event envelope (maximum2,248.48MiB), not strict input-independent O(1) memory. ## Dense-stress limitation diff --git a/validation/performance/m7-command-results.md b/validation/performance/m7-command-results.md index f54c9f5..272840b 100644 --- a/validation/performance/m7-command-results.md +++ b/validation/performance/m7-command-results.md @@ -33,9 +33,12 @@ both states. The same review also required the memory claim to be narrowed from measured10-million-event envelope; the runtime and documentation now use that precise scope. The dependency recertification review then found that finalized ledger state could still mutate -behind its fixed journal hash. Commit `37260badebb1c637e7247d21dc0694e723f5d206` closes the lifecycle -for both ledger and broker:finalization or abort makes all mutation entry points fail closed until -`reset()`, and regression tests prove hashes, counts and snapshots cannot diverge after sealing. +behind its fixed journal hash. Commit `37260badebb1c637e7247d21dc0694e723f5d206` guarded the normal +ledger and broker mutation APIs. Follow-up validation found four bypasses through public state +restore and mutable views. Commit `23f6b19cc1c4e42fb9df8ce600ad34b254a121a5` separates public +restore from engine-internal trusted rollback, returns positions through a read-only view, and makes +instrument/account valuation configuration read-only. Regression tests cover finish, abort, reset, +failed replay restoration and sealed-state mutation attempts. ## Tests and coverage @@ -56,19 +59,19 @@ python tools/check_branch_coverage.py coverage.json --threshold 90 \ - Ruff format/check:PASS. - Python3.12:201 passed; total coverage95.51%. - Pure branch coverage:artifacts94.74%, broker95.56%, contracts91.88%, schemas94.74%, - engine91.84%, matching93.25%, state_machine96.67%, ledger91.91%, rules92.38%. + engine91.92%, matching93.25%, state_machine96.67%, ledger91.91%, rules92.38%. - GitHub Actions run`33251586735`:Python3.10/3.11/3.12 all PASS. - GitHub Actions run`33251588866`:Python3.10/3.11/3.12 all PASS. ## Formal performance evidence -Source commit:`37260badebb1c637e7247d21dc0694e723f5d206`. +Source commit:`23f6b19cc1c4e42fb9df8ce600ad34b254a121a5`. ```text python benchmarks/benchmark_replay.py --workload matching \ --matching-events 10000000 --repeat 3 --require-rate 50000 \ --memory-limit-gib 16 --artifact-mode arrow \ - --artifact-root F:\puresaber-m7-artifacts\execution-v0.5.0-sealed-37260ba \ + --artifact-root F:\puresaber-m7-artifacts\execution-v0.5.0-sealed-v2-23f6b19 \ --artifact-retention keep --artifact-batch-size 8192 \ --artifact-queue-batches 2 \ --output validation\performance\m7-execution-final-10m.json @@ -76,28 +79,28 @@ python benchmarks/benchmark_replay.py --workload matching \ | Run | Events/s | Peak working set | Strict reload | Dirty tree | |---:|---:|---:|---|---| -| 1 | 64,746.49 | 2,239.19MiB | PASS | false | -| 2 | 62,394.71 | 2,242.02MiB | PASS | false | -| 3 | 64,627.91 | 2,236.11MiB | PASS | false | +| 1 | 58,221.53 | 2,244.92MiB | PASS | false | +| 2 | 63,906.96 | 2,248.48MiB | PASS | false | +| 3 | 61,473.01 | 2,243.24MiB | PASS | false | -- Rate gate:PASS for every run; median64,627.91 events/second. -- Memory gate:PASS; maximum2,242.02MiB. +- Rate gate:PASS for every run; median61,473.01 events/second. +- Memory gate:PASS; maximum2,248.48MiB. - Each run:10,000,000 events,500,000 orders/fills,1,000,000 order events and 1,000,001 balanced ledger transactions; explicit fill density5% (`order_stride=20`). - Determinism:all logical hashes, every Arrow physical file hash and the manifest hash match across all three fresh processes. - Artifact manifest SHA-256:`c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27`. -- Final report SHA-256:`6c74790bb3b9d5a20b95ba07989feb0eb6a265a211970577c6092559aaf47cb2`. +- Final report SHA-256:`4dcb33d68126bc454ce68e11ae3d4f4a8475a168bf43b644ee1170c394efc41d`. - Dependencies:Python3.12.5, PyArrow25.0.1, quant-data-kit distribution0.7.4. - Machine:Windows11,16 logical CPUs; process peak working set includes Arrow and live replay state. - Retained artifacts:three directories,1,501,955,792 bytes each (about4.20GiB total), under - `F:\puresaber-m7-artifacts\execution-v0.5.0-sealed-37260ba`; no file was automatically + `F:\puresaber-m7-artifacts\execution-v0.5.0-sealed-v2-23f6b19`; no file was automatically removed. The timed interval includes event materialization, matching, risk, fill, fee, exact ledger, canonical serialization, Arrow initialization/write/seal, logical hashes and manifest close. Process startup, one static fixture-template construction and strict post-run reload are excluded; -strict reload is independently required and passed in2.97–3.00seconds per run. +strict reload is independently required and passed in2.97–3.15seconds per run. ## Dense stress and remaining risks @@ -115,7 +118,7 @@ Remaining risks: - physical Arrow hashes depend on the locked PyArrow serialization version and must be rebaselined, never silently accepted, after a dependency upgrade; - the dense50%-fill stress gate is still a known capacity limitation; -- the slowest representative run passed by24.8%, so future releases must retain the per-run +- the slowest representative run passed by16.4%, so future releases must retain the per-run gate and15% regression comparison instead of relying on the median; - Arrow buffers are bounded, while event/fill identity sets and broker lookup/idempotency state still scale with input/order count; the certified claim is controlled memory at10M, not O(1); diff --git a/validation/performance/m7-execution-final-10m.json b/validation/performance/m7-execution-final-10m.json index 7895b40..6f81cc4 100644 --- a/validation/performance/m7-execution-final-10m.json +++ b/validation/performance/m7-execution-final-10m.json @@ -26,9 +26,9 @@ "artifact_manifest_sha256": "c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27", "artifact_mode": "arrow", "artifact_paths": [ - "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-37260ba\\matching_exact_ledger-10000000-47140", - "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-37260ba\\matching_exact_ledger-10000000-22420", - "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-37260ba\\matching_exact_ledger-10000000-44048" + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-v2-23f6b19\\matching_exact_ledger-10000000-22904", + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-v2-23f6b19\\matching_exact_ledger-10000000-39064", + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-v2-23f6b19\\matching_exact_ledger-10000000-33208" ], "artifact_queue_batches": 2, "artifact_retention": [ @@ -37,30 +37,30 @@ "keep" ], "artifact_volume_free_gib_after_cleanup_runs": [ - 277.08, - 275.69, - 274.29 + 272.89, + 271.49, + 270.09 ], "artifact_volume_free_gib_before_cleanup_runs": [ - 277.08, - 275.69, - 274.29 + 272.89, + 271.49, + 270.09 ], "dependencies": { "pyarrow": "25.0.1", "quant_data_kit": "0.7.4" }, "events": 10000000, - "events_per_s_median": 64627.91, + "events_per_s_median": 61473.01, "events_per_s_runs": [ - 64746.49, - 62394.71, - 64627.91 + 58221.53, + 63906.96, + 61473.01 ], "fill_density": 0.05, "fill_sha256": "68932897888ac2f362bb46191fb2c94bdc7fcc99c46d45b5329c4671ab951dff", "fills": 500000, - "git_commit": "37260badebb1c637e7247d21dc0694e723f5d206", + "git_commit": "23f6b19cc1c4e42fb9df8ce600ad34b254a121a5", "git_dirty_runs": [ false, false, @@ -79,19 +79,19 @@ "order_sha256": "85e3b7ea894109a4380c478c344207dd57975d6d79ba22a8f41fbe190bc5c4d5", "order_stride": 20, "orders": 500000, - "peak_working_set_mib": 2242.02, + "peak_working_set_mib": 2248.48, "peak_working_set_mib_runs": [ - 2239.19, - 2242.02, - 2236.11 + 2244.92, + 2248.48, + 2243.24 ], "python": "3.12.5 (tags/v3.12.5:ff3bc82, Aug 6 2024, 20:45:27) [MSC v.1940 64 bit (AMD64)]", "rate_gate": true, "result_sha256": "cb5907d36a01178d57ad46f4743fed9b1546b2f13ead58b5ce6d016c0cb37db2", "strict_verification_elapsed_s_runs": [ - 2.998672, - 2.996893, - 2.971207 + 3.148932, + 3.007058, + 2.968083 ], "strict_verification_passed": true, "temp_directories": [ @@ -102,9 +102,9 @@ "timing_scope": "includes event materialization, matching, risk, fill, fee, exact ledger, Arrow sink initialization/write/seal, logical hashes, and manifest close; excludes process startup and static fixture-template construction", "transactions": 1000001, "worker_pids": [ - 47140, - 22420, - 44048 + 22904, + 39064, + 33208 ], "workload": "matching_exact_ledger" }