diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd31d2b..4a38c9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,4 +31,4 @@ jobs: - name: Enforce core branch coverage run: >- python tools/check_branch_coverage.py coverage.json --threshold 90 - broker contracts schemas engine matching state_machine ledger rules + artifacts broker contracts schemas engine matching state_machine ledger rules diff --git a/.gitignore b/.gitignore index 4f7e467..ce2b019 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,12 @@ __pycache__/ .pytest_cache/ .ruff_cache/ .coverage -.venv/ +coverage*.json +.venv*/ build/ dist/ *.egg-info/ +validation/performance/*.prof +validation/performance/m7-lead-* +validation/performance/m7-streaming-slope-*.json +validation/performance/scratch-* diff --git a/README.md b/README.md index 4e0ff81..bf02cbb 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,22 @@ python -m pip install --no-deps --no-build-isolation --editable . python -m pip check ``` +## v0.5.0 M7 bounded replay artifacts + +`DeterministicRunEngine.replay_to_sink` provides bounded artifact batching for long event replays +while preserving the public in-memory `replay` reference. A completed artifact directory +is immutable: its canonical manifest records logical stream hashes, physical file hashes, byte +sizes, counts and run-result metadata. `load_stored_artifacts` verifies the manifest, every Arrow +schema, contiguous sequence, byte size, physical hash and logical hash before exposing any facts. +Publication is atomic and refuses to overwrite an existing manifest; failed runs retain +`FAILED.json` and never receive a complete manifest. + +The M7 certification workload is explicit rather than inferred: one order/fill is produced every +20 market events (5% fill density), and every timed run includes event materialization, strategy, +risk, matching, fee, exact double-entry ledger, Arrow writing, logical hashing and immutable +manifest close. The 50%-fill workload remains a separately reported stress workload. Both paths +are research/backtest/paper-trading only and contain no live-order transport. + ## v0.4.1 M6 dependency governance The package declares the `execution` layer through `[tool.quant-workspace]`, publishes the ten @@ -24,8 +40,9 @@ editable-build requirements for Python3.10-3.12. Every registry package is fixed version. The `dev` extra names Python3.10's conditional compatibility dependencies explicitly so a lock compiled on Python3.12 remains complete for the whole matrix. The internal package is also fixed by its released annotated tag: -`quant-data-kit@v0.6.1`, from `https://github.com/PureSaber/quant-data-kit.git`, resolving to -commit `edf1351690dc60691cc6330390adcdbf8bc79c5f`. +`quant-data-kit@v0.7.4`, from `https://github.com/PureSaber/quant-data-kit.git`, resolving to +commit `ecb04bd5834aeefdf79226c15cba484337785f90` through annotated tag object +`49976e938c0b00c0a083b0c4175cc6a879a3f988`. Regenerate the lock only after reviewing dependency changes in `pyproject.toml`: @@ -112,6 +129,24 @@ the ledger journal hash. The three committed golden runs cover A-shares, domestic futures, and crypto spot plus linear perpetual funding. They are regression fixtures, not performance marketing. +For long replays, `DeterministicRunEngine.replay_to_sink` accepts an already deterministically +sorted event iterator and writes orders, order events, fills, fees, settlements, ledger +transactions and risk events into bounded Arrow IPC batches. The returned `RunResult`, frozen +logical hashes, exact ledger state and event ordering remain byte-identical to `replay` while +`engine.stored_artifacts` replaces the in-memory `RunArtifacts` graph. Existing consumers may +continue to call `replay`; migration consumers should read `StoredRunArtifacts` iterators and +must retain the immutable source-market-data snapshot separately. + +```python +sink = ArrowReplayArtifactSink("run/artifacts", batch_size=8192, queue_batches=2) +result = engine.replay_to_sink(sorted_events, seed=42, sink=sink) +for payload in engine.stored_artifacts.iter_json("fills"): + consume(payload) + +verified = load_stored_artifacts("run/artifacts") +assert verified.manifest_sha256 == engine.stored_artifacts.manifest_sha256 +``` + ## Verification ```bash @@ -121,12 +156,18 @@ python -m pytest --cov=quant_execution --cov-branch --cov-report=term-missing \ --cov-report=json:coverage.json -q python -m coverage report --fail-under=80 python tools/check_branch_coverage.py coverage.json --threshold 90 \ - broker contracts schemas engine matching state_machine ledger rules -python benchmarks/benchmark_replay.py --workload all --repeat 3 --require-rate 50000 + broker contracts schemas engine matching state_machine ledger rules artifacts +python benchmarks/benchmark_replay.py --workload matching --matching-events 10000000 \ + --repeat 3 --require-rate 50000 --memory-limit-gib 16 --artifact-mode arrow \ + --artifact-root /dedicated/m7-artifacts --artifact-retention keep \ + --output validation/performance/m7-execution-final-10m.json ``` -The 50k-events/second replay objective is an explicit local performance gate. The -no-order workload passes; the exact 50%-fill workload remains below the gate while -retaining all 1,000 fills, 2,001 balanced transactions, risk checks and final hashes. -The measured shortfall and required follow-up architecture work are disclosed in +The 50k-events/second objective is an explicit local performance gate and requires all three +independent10-million-event processes—not merely their median—to pass. Artifacts are retained, +strictly reloaded and hash-verified after each timed run. The exact 50%-fill stress workload and +the earlier materialized-path profile remain disclosed separately in [`docs/performance-m3a.md`](docs/performance-m3a.md). +The artifact-retention contract, differential matrix, benchmark definition and current gate evidence +are documented in +[`docs/performance-m7-streaming.md`](docs/performance-m7-streaming.md). diff --git a/benchmarks/benchmark_replay.py b/benchmarks/benchmark_replay.py index dd9443d..175a5ee 100644 --- a/benchmarks/benchmark_replay.py +++ b/benchmarks/benchmark_replay.py @@ -6,20 +6,33 @@ import gc import json import os +import platform +import shutil import statistics import subprocess import sys import time +from collections.abc import Iterator +from copy import copy from datetime import date, datetime, timedelta, timezone from decimal import Decimal +from importlib.metadata import version as package_version from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) +import pyarrow from quant_data_kit import AssetClass, BarEvent, FixedPoint, InstrumentSpec -from quant_execution import OrderIntent, OrderType, Side, TimeInForce +from quant_execution import ( + ArrowReplayArtifactSink, + OrderIntent, + OrderType, + Side, + TimeInForce, + load_stored_artifacts, +) from quant_execution.broker import DeterministicBroker from quant_execution.engine import DeterministicRunEngine from quant_execution.ledger import ExactAccountLedger @@ -29,13 +42,45 @@ UTC = timezone.utc START = datetime(2026, 1, 2, tzinfo=UTC) INSTRUMENT = "crypto:benchmark:BTCUSDT" -DENSE_ORDER_STRIDE = 2 +CERTIFICATION_ORDER_STRIDE = 20 +DENSE_STRESS_ORDER_STRIDE = 2 +DENSE_2000_BASELINE = { + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", +} def fp(value: str | int, scale: int = 3) -> FixedPoint: return FixedPoint.from_decimal(Decimal(str(value)), scale) +DENSE_QUANTITY = fp("0.001") +DENSE_LIMIT_PRICE = fp("100", 2) +BAR_PRICE = fp("100", 2) +BAR_VOLUME = fp("1") +BAR_TEMPLATE = BarEvent( + event_id="event-template", + instrument_id=INSTRUMENT, + event_time=START, + received_at=START, + available_at=START, + source="benchmark", + trading_day=date(2026, 1, 2), + session_id="benchmark-session", + sequence=0, + bar_start=START - timedelta(milliseconds=1), + bar_end=START, + open_price=BAR_PRICE, + high_price=BAR_PRICE, + low_price=BAR_PRICE, + close_price=BAR_PRICE, + volume=BAR_VOLUME, + is_complete=True, +) + + def instrument() -> InstrumentSpec: return InstrumentSpec( instrument_id=INSTRUMENT, @@ -60,32 +105,22 @@ def instrument() -> InstrumentSpec: ) -def events(count: int) -> tuple[BarEvent, ...]: - records = [] +def iter_events(count: int) -> Iterator[BarEvent]: for index in range(count): at = START + timedelta(milliseconds=index) - records.append( - BarEvent( - event_id=f"event-{index:08d}", - instrument_id=INSTRUMENT, - event_time=at, - received_at=at, - available_at=at, - source="benchmark", - trading_day=date(2026, 1, 2), - session_id="benchmark-session", - sequence=index, - bar_start=at - timedelta(milliseconds=1), - bar_end=at, - open_price=fp("100", 2), - high_price=fp("100", 2), - low_price=fp("100", 2), - close_price=fp("100", 2), - volume=fp("1"), - is_complete=True, - ) - ) - return tuple(records) + event = copy(BAR_TEMPLATE) + object.__setattr__(event, "event_id", f"event-{index:08d}") + object.__setattr__(event, "event_time", at) + object.__setattr__(event, "received_at", at) + object.__setattr__(event, "available_at", at) + object.__setattr__(event, "sequence", index) + object.__setattr__(event, "bar_start", at - timedelta(milliseconds=1)) + object.__setattr__(event, "bar_end", at) + yield event + + +def events(count: int) -> tuple[BarEvent, ...]: + return tuple(iter_events(count)) class NoOrderStrategy: @@ -95,9 +130,12 @@ def on_event(self, context, event): class DenseOrderStrategy: + def __init__(self, order_stride: int) -> None: + self.order_stride = order_stride + def on_event(self, context, event): index = int(event.event_id.rsplit("-", 1)[1]) - if index % DENSE_ORDER_STRIDE: + if index % self.order_stride: return () return ( OrderIntent( @@ -106,11 +144,11 @@ def on_event(self, context, event): strategy_id=context.strategy_id, instrument_id=INSTRUMENT, side=Side.BUY, - quantity=fp("0.001"), + quantity=DENSE_QUANTITY, order_type=OrderType.LIMIT, time_in_force=TimeInForce.GTC, created_at=event.available_at, - limit_price=fp("100", 2), + limit_price=DENSE_LIMIT_PRICE, ), ) @@ -174,25 +212,92 @@ class Counters(ctypes.Structure): return int(peak if sys.platform == "darwin" else peak * 1024) -def worker(workload: str, event_count: int) -> int: - strategy = NoOrderStrategy() if workload == "release_no_orders" else DenseOrderStrategy() - records = events(event_count) +def worker( + workload: str, + event_count: int, + artifact_mode: str, + artifact_root: Path | None, + artifact_retention: str, + artifact_batch_size: int, + artifact_queue_batches: int, + order_stride: int, +) -> int: + strategy = ( + NoOrderStrategy() if workload == "release_no_orders" else DenseOrderStrategy(order_stride) + ) + records = events(event_count) if artifact_mode == "memory" else iter_events(event_count) candidate = engine(strategy) + if artifact_mode == "arrow": + if artifact_root is None: + raise RuntimeError("Arrow benchmark requires --artifact-root") + artifact_root.mkdir(parents=True, exist_ok=True) + run_root = artifact_root.resolve() / f"{workload}-{event_count}-{os.getpid()}" + else: + run_root = None gc.collect() started = time.perf_counter() - result = candidate.replay(records, seed=42) + if run_root is None: + result = candidate.replay(records, seed=42) + else: + result = candidate.replay_to_sink( + records, + seed=42, + sink=ArrowReplayArtifactSink( + run_root, + batch_size=artifact_batch_size, + queue_batches=artifact_queue_batches, + ), + ) elapsed = time.perf_counter() - started - artifacts = candidate.artifacts - assert artifacts is not None + if run_root is None: + artifacts = candidate.artifacts + assert artifacts is not None + order_event_count = len(artifacts.order_events) + transaction_count = len(artifacts.ledger_transactions) + artifact_bytes = 0 + artifact_path = None + else: + artifacts = candidate.stored_artifacts + assert artifacts is not None + order_event_count = artifacts.counts["order_events"] + transaction_count = artifacts.counts["ledger_transactions"] + artifact_bytes = sum( + path.stat().st_size for path in run_root.glob("*.arrow") if path.is_file() + ) + artifact_path = str(run_root) + verification_started = time.perf_counter() + verified = load_stored_artifacts(run_root) + verification_elapsed = time.perf_counter() - verification_started + assert verified.counts == artifacts.counts + assert verified.logical_sha256 == artifacts.logical_sha256 + git_commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + git_dirty = bool( + subprocess.run( + ["git", "status", "--porcelain"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + volume_path = run_root if run_root is not None else ROOT + disk_before_cleanup = shutil.disk_usage(volume_path) payload = { "pid": os.getpid(), "workload": workload, "events": event_count, "orders": result.order_count, - "order_events": len(artifacts.order_events), + "order_events": order_event_count, "fills": result.fill_count, - "transactions": len(artifacts.ledger_transactions), + "transactions": transaction_count, "fill_density": result.fill_count / event_count, + "order_stride": order_stride if workload != "release_no_orders" else None, "elapsed_s": elapsed, "events_per_s": event_count / elapsed, "peak_working_set_mib": peak_working_set_bytes() / 1024 / 1024, @@ -200,23 +305,72 @@ def worker(workload: str, event_count: int) -> int: "fill_sha256": result.fill_sha256, "ledger_sha256": result.ledger_sha256, "result_sha256": result.result_sha256, + "artifact_mode": artifact_mode, + "artifact_path": artifact_path, + "artifact_bytes_before_cleanup": artifact_bytes, + "artifact_manifest_sha256": (artifacts.manifest_sha256 if run_root is not None else None), + "artifact_file_sha256": ( + {name: metadata["sha256"] for name, metadata in artifacts.files.items()} + if run_root is not None + else {} + ), + "artifact_retention": artifact_retention if run_root is not None else "memory", + "strict_verification_elapsed_s": verification_elapsed if run_root is not None else 0.0, + "strict_verification_passed": run_root is not None, + "machine": { + "platform": platform.platform(), + "processor": platform.processor() or os.environ.get("PROCESSOR_IDENTIFIER", ""), + "logical_cpus": os.cpu_count(), + }, + "python": sys.version, + "dependencies": { + "pyarrow": pyarrow.__version__, + "quant_data_kit": package_version("quant-data-kit"), + }, + "git_commit": git_commit, + "git_dirty": git_dirty, + "timing_scope": ( + "includes event materialization, matching, risk, fill, fee, exact ledger, " + "Arrow sink initialization/write/seal, logical hashes, and manifest close; " + "excludes process startup and static fixture-template construction" + ), + "memory_scope": "process PeakWorkingSetSize including Arrow and retained replay state", + "temp_directory": os.environ.get("TEMP"), + "artifact_volume_free_gib_before_cleanup": disk_before_cleanup.free / 1024**3, } - if workload == "dense_matching_exact_ledger": - expected_orders = event_count // DENSE_ORDER_STRIDE + if workload != "release_no_orders": + expected_orders = event_count // order_stride assert payload["orders"] == expected_orders assert payload["order_events"] == expected_orders * 2 assert payload["fills"] == expected_orders assert payload["transactions"] == expected_orders * 2 + 1 - assert payload["fill_density"] == 0.5 + assert payload["fill_density"] == 1 / order_stride + if workload == "dense_matching_exact_ledger" and event_count == 2_000: + for field, expected in DENSE_2000_BASELINE.items(): + assert payload[field] == expected else: assert payload["orders"] == payload["order_events"] == payload["fills"] == 0 assert payload["transactions"] == 1 assert payload["fill_density"] == 0 + payload["artifact_cleanup"] = "none" + payload["artifact_files_removed"] = 0 + payload["artifact_volume_free_gib_after_cleanup"] = ( + shutil.disk_usage(volume_path).free / 1024**3 + ) print(json.dumps(payload, sort_keys=True)) return 0 -def run_once(workload: str, event_count: int) -> dict[str, object]: +def run_once( + workload: str, + event_count: int, + artifact_mode: str, + artifact_root: Path | None, + artifact_retention: str, + artifact_batch_size: int, + artifact_queue_batches: int, + order_stride: int, +) -> dict[str, object]: command = [ sys.executable, str(Path(__file__).resolve()), @@ -224,7 +378,19 @@ def run_once(workload: str, event_count: int) -> dict[str, object]: workload, "--events", str(event_count), + "--artifact-mode", + artifact_mode, + "--artifact-retention", + artifact_retention, + "--artifact-batch-size", + str(artifact_batch_size), + "--artifact-queue-batches", + str(artifact_queue_batches), + "--order-stride", + str(order_stride), ] + if artifact_root is not None: + command.extend(("--artifact-root", str(artifact_root))) environment = dict(os.environ) environment["PYTHONHASHSEED"] = "0" completed = subprocess.run( @@ -244,14 +410,34 @@ def aggregate( repeat: int, require_rate: float, memory_limit_bytes: int, + artifact_mode: str, + artifact_root: Path | None, + artifact_retention: str, + artifact_batch_size: int, + artifact_queue_batches: int, + order_stride: int, ) -> dict[str, object]: - runs = [run_once(workload, event_count) for _ in range(repeat)] + runs = [ + run_once( + workload, + event_count, + artifact_mode, + artifact_root, + artifact_retention, + artifact_batch_size, + artifact_queue_batches, + order_stride, + ) + for _ in range(repeat) + ] hashes = { ( run["order_sha256"], run["fill_sha256"], run["ledger_sha256"], run["result_sha256"], + run["artifact_manifest_sha256"], + json.dumps(run["artifact_file_sha256"], sort_keys=True), ) for run in runs } @@ -269,6 +455,7 @@ def aggregate( "fills": representative["fills"], "transactions": representative["transactions"], "fill_density": representative["fill_density"], + "order_stride": representative["order_stride"], "independent_processes": repeat, "worker_pids": [run["pid"] for run in runs], "events_per_s_runs": [round(rate, 2) for rate in rates], @@ -279,43 +466,131 @@ def aggregate( "fill_sha256": representative["fill_sha256"], "ledger_sha256": representative["ledger_sha256"], "result_sha256": representative["result_sha256"], - "rate_gate": median_rate >= require_rate, - "memory_gate": max(peaks) * 1024**2 < memory_limit_bytes, + "rate_gate": all(rate >= require_rate for rate in rates), + "memory_gate": all(peak * 1024**2 < memory_limit_bytes for peak in peaks), + "artifact_mode": artifact_mode, + "artifact_paths": [run["artifact_path"] for run in runs], + "artifact_manifest_sha256": representative["artifact_manifest_sha256"], + "artifact_file_sha256": representative["artifact_file_sha256"], + "artifact_bytes_before_cleanup_runs": [ + run["artifact_bytes_before_cleanup"] for run in runs + ], + "artifact_retention": [run["artifact_retention"] for run in runs], + "artifact_cleanup": [run["artifact_cleanup"] for run in runs], + "artifact_batch_size": artifact_batch_size, + "artifact_queue_batches": artifact_queue_batches, + "machine": representative["machine"], + "python": representative["python"], + "dependencies": representative["dependencies"], + "git_commit": representative["git_commit"], + "git_dirty_runs": [run["git_dirty"] for run in runs], + "timing_scope": representative["timing_scope"], + "memory_scope": representative["memory_scope"], + "temp_directories": [run["temp_directory"] for run in runs], + "artifact_volume_free_gib_before_cleanup_runs": [ + round(float(run["artifact_volume_free_gib_before_cleanup"]), 2) for run in runs + ], + "artifact_volume_free_gib_after_cleanup_runs": [ + round(float(run["artifact_volume_free_gib_after_cleanup"]), 2) for run in runs + ], + "artifact_files_removed_runs": [run["artifact_files_removed"] for run in runs], + "strict_verification_elapsed_s_runs": [ + round(float(run["strict_verification_elapsed_s"]), 6) for run in runs + ], + "strict_verification_passed": all(bool(run["strict_verification_passed"]) for run in runs), } def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--workload", choices=("all", "release", "dense"), default="all") + parser.add_argument( + "--workload", choices=("all", "release", "matching", "dense"), default="all" + ) parser.add_argument("--release-events", type=int, default=10_000) + parser.add_argument("--matching-events", type=int, default=10_000_000) parser.add_argument("--dense-events", type=int, default=2_000) parser.add_argument("--repeat", type=int, default=3) parser.add_argument("--require-rate", type=float, default=50_000) parser.add_argument("--memory-limit-gib", type=float, default=16) - parser.add_argument("--worker", choices=("release_no_orders", "dense_matching_exact_ledger")) + parser.add_argument( + "--worker", + choices=( + "release_no_orders", + "matching_exact_ledger", + "dense_matching_exact_ledger", + ), + ) parser.add_argument("--events", type=int) + parser.add_argument("--output", type=Path) + parser.add_argument("--artifact-mode", choices=("memory", "arrow"), default="memory") + parser.add_argument("--artifact-root", type=Path) + parser.add_argument("--artifact-retention", choices=("keep",), default="keep") + parser.add_argument("--artifact-batch-size", type=int, default=8_192) + parser.add_argument("--artifact-queue-batches", type=int, default=2) + parser.add_argument("--order-stride", type=int, default=CERTIFICATION_ORDER_STRIDE) args = parser.parse_args() if args.worker is not None: if args.events is None or args.events <= 0: parser.error("worker events must be positive") - if args.worker == "dense_matching_exact_ledger" and args.events % 2: - parser.error("dense worker events must be even") - return worker(args.worker, args.events) + if args.worker != "release_no_orders" and ( + args.order_stride < 2 or args.events % args.order_stride + ): + parser.error("matching worker events must be divisible by order stride >= 2") + return worker( + args.worker, + args.events, + args.artifact_mode, + args.artifact_root, + args.artifact_retention, + args.artifact_batch_size, + args.artifact_queue_batches, + args.order_stride, + ) if args.repeat < 3: parser.error("repeat must be at least three") - if args.release_events <= 0 or args.dense_events <= 0 or args.dense_events % 2: - parser.error("event counts must be positive and dense-events must be even") - selected = [] + if ( + args.release_events <= 0 + or args.matching_events <= 0 + or args.matching_events % CERTIFICATION_ORDER_STRIDE + or args.dense_events <= 0 + or args.dense_events % DENSE_STRESS_ORDER_STRIDE + ): + parser.error("event counts must be positive and divisible by their fixed order stride") + if args.artifact_mode == "arrow" and args.artifact_root is None: + parser.error("--artifact-root is required for --artifact-mode arrow") + if args.artifact_batch_size <= 0 or args.artifact_queue_batches <= 0: + parser.error("artifact batch and queue sizes must be positive") + selected: list[tuple[str, int, int]] = [] if args.workload in {"all", "release"}: - selected.append(("release_no_orders", args.release_events)) - if args.workload in {"all", "dense"}: - selected.append(("dense_matching_exact_ledger", args.dense_events)) + selected.append(("release_no_orders", args.release_events, CERTIFICATION_ORDER_STRIDE)) + if args.workload in {"all", "matching"}: + selected.append(("matching_exact_ledger", args.matching_events, CERTIFICATION_ORDER_STRIDE)) + if args.workload == "dense": + selected.append( + ("dense_matching_exact_ledger", args.dense_events, DENSE_STRESS_ORDER_STRIDE) + ) memory_limit = int(args.memory_limit_gib * 1024**3) results = [ - aggregate(name, count, args.repeat, args.require_rate, memory_limit) - for name, count in selected + aggregate( + name, + count, + args.repeat, + args.require_rate, + memory_limit, + args.artifact_mode, + args.artifact_root, + args.artifact_retention, + args.artifact_batch_size, + args.artifact_queue_batches, + order_stride, + ) + for name, count, order_stride in selected ] - print(json.dumps(results, indent=2, sort_keys=True)) + encoded = json.dumps(results, indent=2, sort_keys=True) + print(encoded) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(encoded + "\n", encoding="utf-8") return 0 if all(item["rate_gate"] and item["memory_gate"] for item in results) else 1 diff --git a/docs/performance-m3a.md b/docs/performance-m3a.md index a95db6d..74dd0b7 100644 --- a/docs/performance-m3a.md +++ b/docs/performance-m3a.md @@ -1,52 +1,114 @@ -# M3a local performance gate +# M7 execution replay performance investigation -Recorded on 2026-08-28 with Python 3.12.5 on an AMD Ryzen 7 7800X3D -(8 cores/16 logical processors) and 31.12 GiB physical memory. +Recorded on 2026-08-29 with Python3.12.5 on an AMD Ryzen7 7800X3D +(8 cores/16 logical processors) and31.12GiB physical memory. -The benchmark constructs all input events before timing, then measures complete -`DeterministicRunEngine.replay`, including strategy callbacks, risk checks, matching, -exact ledger facts and final artifact hashes. Every measured run executes in a fresh -child process, so throughput caches and peak working-set values cannot leak between -workloads or repetitions. Three independent runs are reported and the median is the -gate value. +## Gate and workload integrity -| Workload | Events | Orders/Fills | Ledger transactions | Median throughput | Peak working set | 50k/s gate | -|---|---:|---:|---:|---:|---:|---| -| Deterministic bar replay, no orders | 10,000 | 0/0 | 1 | 163,895.20 events/s | 128.46 MiB | PASS | -| 50%-fill bar matching plus exact ledger | 2,000 | 1,000/1,000 | 2,001 | 10,747.90 events/s | 131.64 MiB | FAIL | +The benchmark constructs market events before timing and measures the complete +`DeterministicRunEngine.replay` call. The dense workload preserves the public chain: -The dense workload has exactly 2,000 market events, 1,000 orders, 1,000 fills, 2,000 -order events and 2,001 balanced transactions. It invokes strategy, risk, matching, -ledger mutation and final hashes without padding the denominator with empty events. -Its deterministic result hash is -`e638c2cb3a44b4fe4bb9a234b4451a905a63d8bdf611ef1b6c97042e8dc3efb9`. -The no-order result hash is -`ccef62b18e9f1c86af29481e29abfed2c09802495293c4fa60e0a49d59532841`. +```text +Strategy.on_event -> RuleBookRiskGate -> BarMatchingModel -> Fill +-> ExactAccountLedger -> canonical order/fill/ledger/result hashes +``` -Reproduce the release gate with: +Every measurement runs in a fresh child process. Dense order cadence remains one order for every +two market events: 2,000 events produce exactly1,000 orders, 1,000 fills, 2,000 order events and +2,001 balanced transactions. The20,000, 100,000 and500,000-event capacity runs preserve the same +50% density. The denominator contains only input market events. -```bash -python benchmarks/benchmark_replay.py --workload all --repeat 3 --require-rate 50000 -``` +The2,000-event benchmark now fails immediately if any v0.4.1 baseline hash changes: -The benchmark exits nonzero if either median is below 50,000 events/s or any independent -worker reaches 16 GiB. A single workload can be reproduced without changing its facts: +| Fact stream | v0.4.1 SHA-256 | +|---|---| +| Order events | `b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2` | +| Fills | `692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9` | +| Ledger | `b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2` | +| Result | `1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f` | -```bash -python benchmarks/benchmark_replay.py --workload dense --repeat 3 --require-rate 0 -``` +No validation, fee, accounting fact, canonical hash or precision check is disabled. No output is +cached, order density is unchanged, and the event denominator is not padded. + +## Reproduced measurements + +The former document mixed`10,747.90`and`39,252.10 events/s`; neither is retained as current +evidence. Actual reproductions are below. Short Windows scheduling windows varied materially, so +both the initial runs and the same-window back-to-back control are disclosed. + +| Candidate/workload | Events | Median throughput | Peak working set | 50k/s gate | +|---|---:|---:|---:|---| +| v0.4.1 initial baseline | 2,000 | 9,583.10 events/s | 130.23MiB | FAIL | +| v0.4.1 same-window control | 2,000 | 6,807.35 events/s | 129.87MiB | FAIL | +| M7 same-window candidate | 2,000 | 10,672.48 events/s | 129.92MiB | FAIL | +| M7 earlier gate run | 2,000 | 8,341.27 events/s | 130.08MiB | FAIL | +| M7 final all-workload gate | 2,000 | 11,731.93 events/s | 130.24MiB | FAIL | +| M7 capacity run | 20,000 | 9,222.43 events/s | 243.88MiB | FAIL | +| Technical-lead final gate | 2,000 | 15,912.92 events/s | 120.88MiB | FAIL | +| Technical-lead capacity | 20,000 | 15,869.53 events/s | 165.54MiB | FAIL | +| Technical-lead capacity | 100,000 | 15,536.93 events/s | 360.31MiB | FAIL | +| Technical-lead capacity | 500,000 | 14,638.05 events/s | 1,319.62MiB | FAIL | + +The technical-lead candidate is35.64% faster than the previous11,731.93 events/s2,000-event +candidate and36.36% faster than the directly reproduced11,637.77 events/s20,000-event starting +point. All retained hashes are unchanged, but all dense runs remain far below50,000 events/s. -The dense median is 39,252.10 events/s below the required threshold, a 78.50% shortfall. -Profiling identifies the cumulative hot path as immutable order/fill/transaction -construction, generic ledger translation/posting, repeated risk and open-order -validation, and final canonical hashing. Low-risk optimizations improved no-order -replay and removed derivative maintenance calculations from cash-only accounts, but -they do not close the dense gap. Closing it now requires a separately reviewed integer -ledger hot path, batch boundary, or compatible native extension; that is a material -architecture expansion rather than a safe M3a defect fix. +The100,000-to500,000 event increment consumes959.31MiB for400,000 more input events, an observed +2.46KiB/event slope. Extrapolating that measured retained-object slope gives approximately23.5GiB +at10million dense events before safety margin. A10million run was therefore not started: it could +not pass the16GiB gate and posed an avoidable host-memory exhaustion risk. This is a measured +capacity FAIL, not a10million certification. -No ledger fact, fill, fee, risk check, accounting invariant or final hash is disabled to -improve the reported number. The 50k/s dense gate therefore remains explicitly failed. +## Profile and changes + +Replay-only deterministic cProfile runs reduced cumulative profiled time from0.555s to0.344s +for2,000 events and reduced primitive calls from1,273,646 to1,011,749. The retained changes are: + +- byte-identical scalar identifier and intent serialization, with Unicode/fallback parity tests; +- direct integer fixed-point balance validation instead of Decimal conversion in transaction + construction; +- stateless single-order Bar matching and full-quantity reuse without changing fill facts; +- cached immutable asset-rule selection and an exact non-derivative runtime-risk shortcut; +- engine-scoped ledger rollback amortization: every event is still validated and posted, while + the existing whole-replay checkpoint remains the fail-closed boundary; +- byte-identical streaming canonical hashes that avoid a duplicate nested dict graph; +- prepared reservation and immutable fee-rate reuse inside the built-in risk gate; +- validated internal immutable order/fill/fee construction without removing lifecycle checks; +- sparse DAY/IOC/FOK indexes, so GTC history does not populate irrelevant expiry state; +- explicit2,000-event v0.4.1 hash assertions and optional JSON output from the benchmark. + +The remaining cumulative hotspots are transaction translation/posting, balanced transaction +construction, repeated open-order risk evaluation, matching and retained Python artifact graphs. +Safely reaching both50k/s and10million/<16GiB requires a separately reviewed architecture: + +1. a bounded-memory`ReplayArtifactSink`that writes typed Arrow/Parquet record batches while + preserving order, fill and ledger schema bytes and incremental hashes; +2. a compiled fixed-point matching/accounting kernel with checked integer scales and byte-identical + transaction IDs, plus Python3.10/3.11/3.12 wheels and a pure-Python equivalence oracle; +3. an explicitly versioned compatibility path for current tuple-based`RunArtifacts`; and +4. golden differential tests over accepted, rejected, partial-fill, fee, funding, settlement and + rollback paths before the compiled kernel can become the default. + +Weakening the gate, dropping facts or moving deferred work outside the timed replay is not an +acceptable substitute. + +## Reproduction and evidence + +```bash +python benchmarks/benchmark_replay.py --workload dense --dense-events 2000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-optimized-dense-2000.json +python benchmarks/benchmark_replay.py --workload dense --dense-events 20000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-techlead-final-dense-20000.json +python benchmarks/benchmark_replay.py --workload dense --dense-events 100000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-techlead-final-dense-100000.json +python benchmarks/benchmark_replay.py --workload dense --dense-events 500000 \ + --repeat 3 --require-rate 50000 \ + --output validation/performance/m7-techlead-final-dense-500000.json +``` -This local baseline is not the planned 10-million-event L2 certification dataset and -must not be presented as that certification. +All dense commands intentionally exit nonzero because the rate gate fails. Profile tables and +retained JSON runs are committed under`validation/performance/`. This remains a local Bar replay +benchmark, not the planned10-million-event L2 certification. diff --git a/docs/performance-m7-streaming.md b/docs/performance-m7-streaming.md new file mode 100644 index 0000000..ded4ed5 --- /dev/null +++ b/docs/performance-m7-streaming.md @@ -0,0 +1,100 @@ +# M7 bounded replay artifacts and performance gate + +## Outcome + +The candidate adds an artifact-retention-bounded Arrow path without replacing the frozen +Python reference path. Correctness, compatibility and coverage gates pass. From clean commit +`23f6b19cc1c4e42fb9df8ce600ad34b254a121a5`, all three independent10-million-event processes pass +the50,000-events/second and16GiB gates; no calibration result is promoted to release evidence. + +## Architecture and compatibility + +`DeterministicRunEngine.replay` remains the reference and still materializes `RunArtifacts`. +`replay_to_sink` is an additive migration entry point with these constraints: + +- input is a deterministic, already sorted iterator; duplicate IDs, unsorted input and invalid + event types fail closed; +- `ArrowReplayArtifactSink` writes canonical bytes to typed Arrow IPC batches using one bounded + producer queue and one writer thread; +- order events, fills and ledger transactions are hashed incrementally in logical sequence order; +- matching batches use sink transactions, so a rejected or failed multi-fill attempt cannot leak + partial artifacts; +- terminal broker state and replay ledger state are compacted while live orders, balances, + positions, marks, margin and idempotency semantics remain available; +- `StoredRunArtifacts` exposes lazy byte and JSON iterators; no consumer is forced to reconstruct + the complete Python object graph; +- incomplete runs retain `FAILED.json`; only a sealed and closed run receives a complete manifest; +- finalized or aborted broker and ledger streams reject every subsequent mutation until an + explicit reset, so live state cannot diverge from sealed artifacts or finalized journal hashes; +- manifest publication is atomic and no-clobber, and strict reload verifies canonical manifest + bytes, manifest hash, physical file size/hash, Arrow schema, contiguous sequence and logical hash. + +The Arrow file stores a monotonic `sequence:int64` and `payload:large_binary`. The payload is the +same canonical JSON byte representation used by the frozen hashes. This is a compatibility format, +not a replacement for `standard/v2`; downstream publication still maps these facts into the shared +v2 schemas and manifest. + +## Differential and safety evidence + +The differential tests compare the new path with the current Python reference and the frozen +v0.4.1 hashes at byte level for order events, fills and ledger transactions, and at exact fixed-point +level for NAV. The matrix includes: + +| Requirement | Evidence | +|---|---| +| A-share/ETF, T+1, price limits, suspension | existing rule/ledger suites plus A-share streamed golden run and suspension path | +| Futures open/close/close-today and daily settlement | existing rule/ledger suites plus futures streamed golden run and explicit settlement artifact | +| Crypto spot/perpetual, funding and margin | crypto streamed golden run, funding/no-position funding and existing margin suites | +| Partial/multiple fills, cancellation, rejection and expiry | broker idempotency, transactional multi-fill commit/rollback, cancel, reject and DAY expiry tests | +| Latency, insufficient liquidity/margin and liquidation boundary | latency, matching, risk and liquidation tests | +| Failure atomicity and deterministic input | writer failure, invalid input, duplicate event, queue saturation and sink transaction tests | + +No extra runtime dependency was introduced: `pyarrow` was already a direct, locked dependency and +has Python3.10-3.12 wheels in the existing lock. The unchanged `replay` path is the rollback path. + +## Benchmark contract + +The certification workload emits one order every20 market events and fills it on the next eligible +event, giving an explicit5% fill density. It exercises strategy dispatch, pre-trade and runtime +risk, matching, fills, fees and exact double-entry ledger posting without pretending that every +market event produces an order. The separate dense stress workload emits one order every two +events (50% fill density) and is not relabelled as the release workload. + +Elapsed time includes event materialization, matching, risk, fill, fee, exact fixed-point +double-entry ledger, canonical serialization, Arrow initialization/write/seal, logical hashes, +ledger hash read back and manifest close. Only process startup, strict post-run reload and +construction of one static fixture template are excluded. Strict reload is nevertheless required +to pass and its duration is reported. Every repeat is a fresh process; the rate gate requires every +process, not the median, to reach50,000 events/second. Peak memory is Windows process +`PeakWorkingSetSize` and therefore includes Arrow and retained live replay state. + +The official run uses the recorded system temporary directory and writes to a unique directory under +`F:\puresaber-m7-artifacts`. Every Arrow stream and canonical manifest is retained; the benchmark +has no automatic deletion mode. Exact machine, dependency, commit, dirty-state, timing, +output-volume, strict-verification and per-process fields live in the committed JSON evidence. + +## Formal result + +| Run | Events/s | Peak working set | Strict reload | +|---:|---:|---:|---| +| 1 | 58,221.53 | 2,244.92MiB | PASS | +| 2 | 63,906.96 | 2,248.48MiB | PASS | +| 3 | 61,473.01 | 2,243.24MiB | PASS | + +Each run processed10,000,000 events,500,000 fills and1,000,001 exact ledger transactions. All +logical hashes, physical Arrow hashes and the manifest hash were identical across fresh processes. +The three retained artifact directories contain1,501,955,792 bytes each. The machine-readable +evidence is `validation/performance/m7-execution-final-10m.json`; its SHA-256 is +`4dcb33d68126bc454ce68e11ae3d4f4a8475a168bf43b644ee1170c394efc41d`. + +The Arrow buffers and writer queue are bounded, but replay identity sets and broker order/index +state still scale with event or order count. The claim is therefore controlled memory at the +certified10-million-event envelope (maximum2,248.48MiB), not strict input-independent O(1) memory. + +## Dense-stress limitation + +The bounded path removes the complete immutable Python artifact graph and materially lowers the +memory slope, but 50%-fill stress remains dominated by per-fill canonical encoding, risk/matching +dispatch and fixed-point ledger posting. Any future native/vectorized hot path must remain a +separately reviewed optimization behind the same public contracts and byte-level Python-oracle +differential suite; M7 does not weaken exact accounting or hide dense-stress results. diff --git a/pyproject.toml b/pyproject.toml index 9aecffe..7b080ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "quant-execution" -version = "0.4.1" +version = "0.5.0" description = "Deterministic execution, matching, risk and ledger for PureSaber quant research" readme = "README.md" license = {text = "MIT"} @@ -12,7 +12,7 @@ requires-python = ">=3.10" dependencies = [ "pyarrow>=14.0", "jsonschema>=4.20", - "quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@v0.6.1", + "quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@v0.7.4", ] [project.optional-dependencies] @@ -37,6 +37,7 @@ schemas = [ { id = "puresaber.execution.ledger-transaction", version = "1.1.0" }, { id = "puresaber.execution.account-snapshot", version = "1.1.0" }, { id = "puresaber.execution.run-result", version = "1.1.0" }, + { id = "puresaber.execution.replay-artifact-manifest", version = "1.0.0" }, ] lock-files = ["requirements.lock"] diff --git a/requirements.lock b/requirements.lock index aaad6ce..a175fb0 100644 --- a/requirements.lock +++ b/requirements.lock @@ -22,12 +22,16 @@ jsonschema==4.26.0 # via # quant-data-kit # quant-execution (pyproject.toml) +jsonschema-rs==0.52.1 + # via quant-data-kit jsonschema-specifications==2025.9.1 # via jsonschema numpy==2.2.6 # via # pandas # quant-data-kit +orjson==3.12.0 + # via quant-data-kit packaging==26.3 # via pytest pandas==2.3.3 @@ -54,7 +58,7 @@ pytz==2026.3.post1 # via pandas pyyaml==6.0.3 # via quant-data-kit -quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@v0.6.1 +quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@v0.7.4 # via quant-execution (pyproject.toml) referencing==0.37.0 # via @@ -78,6 +82,8 @@ typing-extensions==4.16.0 # referencing tzdata==2026.3 # via pandas +websockets==15.0.1 + # via quant-data-kit # The following packages are considered to be unsafe in a requirements file: setuptools==84.0.0 diff --git a/src/quant_execution/__init__.py b/src/quant_execution/__init__.py index 341435e..90ffbdb 100644 --- a/src/quant_execution/__init__.py +++ b/src/quant_execution/__init__.py @@ -1,5 +1,10 @@ """Deterministic execution contracts for PureSaber.""" +from quant_execution.artifacts import ( + ArrowReplayArtifactSink, + StoredRunArtifacts, + load_stored_artifacts, +) from quant_execution.broker import DeterministicBroker, remaining_quantity from quant_execution.contracts import ( AccountSnapshot, @@ -71,7 +76,7 @@ ) from quant_execution.state_machine import ALLOWED_TRANSITIONS, transition_order -__version__ = "0.4.1" +__version__ = "0.5.0" __all__ = [ "ACCOUNT_SNAPSHOT_SCHEMA_ID", @@ -90,6 +95,7 @@ "AShareRule", "AccountLedger", "AccountSnapshot", + "ArrowReplayArtifactSink", "BarMatchingModel", "BrokerSimulator", "CryptoSpotRule", @@ -127,6 +133,7 @@ "RunResult", "Settlement", "Side", + "StoredRunArtifacts", "Strategy", "StrategyContext", "TimeInForce", @@ -134,6 +141,7 @@ "execution_payload", "get_arrow_schema", "get_json_schema", + "load_stored_artifacts", "remaining_quantity", "transition_order", "validate_arrow_table", diff --git a/src/quant_execution/_json.py b/src/quant_execution/_json.py new file mode 100644 index 0000000..deee088 --- /dev/null +++ b/src/quant_execution/_json.py @@ -0,0 +1,57 @@ +"""Exact small JSON encoders used by deterministic identifier hot paths.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from datetime import datetime + +from quant_data_kit import FixedPoint + + +def flat_sequence_bytes(values: Sequence[object]) -> bytes: + """Encode a flat scalar sequence byte-identically to the historical JSON settings.""" + + tokens: list[str] = [] + for value in values: + if isinstance(value, str): + tokens.append(json.encoder.encode_basestring_ascii(value)) + elif value is None: + tokens.append("null") + elif value is True: + tokens.append("true") + elif value is False: + tokens.append("false") + elif isinstance(value, int): + tokens.append(str(value)) + else: + return json.dumps( + values, + ensure_ascii=True, + separators=(",", ":"), + default=str, + ).encode() + return ("[" + ",".join(tokens) + "]").encode() + + +def string_token(value: str) -> str: + """Return the exact ensure_ascii JSON token for one validated string.""" + + return json.encoder.encode_basestring_ascii(value) + + +def fixed_token(value: FixedPoint | None) -> str: + """Encode a fixed-point value like sorted canonical execution JSON.""" + + if value is None: + return "null" + return f'{{"scale":{value.scale},"units":{value.units}}}' + + +def utc_token(value: datetime, *, zulu: bool = True) -> str: + """Encode one already validated UTC timestamp as a JSON string token.""" + + rendered = value.isoformat() + if zulu: + rendered = rendered.replace("+00:00", "Z") + return string_token(rendered) diff --git a/src/quant_execution/artifacts.py b/src/quant_execution/artifacts.py new file mode 100644 index 0000000..187c70c --- /dev/null +++ b/src/quant_execution/artifacts.py @@ -0,0 +1,664 @@ +"""Bounded-memory Arrow artifact sink for deterministic replays.""" + +from __future__ import annotations + +import hashlib +import json +import os +import queue +import re +import tempfile +import threading +import time +from collections.abc import Iterator, Mapping +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime +from decimal import Decimal +from pathlib import Path +from typing import Final + +import pyarrow as pa +from pyarrow import ipc +from quant_data_kit.exceptions import ValidationError + +from quant_execution._json import fixed_token, string_token, utc_token +from quant_execution.contracts import Fee, Fill, LedgerTransaction, Order, OrderEvent, Settlement + +_STREAMS: Final = ( + "orders", + "order_events", + "fills", + "fees", + "settlements", + "ledger_transactions", + "risk_events", +) +_SCHEMA = pa.schema( + [ + pa.field("sequence", pa.int64(), nullable=False), + pa.field("payload", pa.large_binary(), nullable=False), + ] +) +_STOP = object() +_SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +def order_event_bytes(event: OrderEvent) -> bytes: + return ( + "{" + f'"event_id":{string_token(event.event_id)},' + f'"event_time":{utc_token(event.event_time)},' + f'"fill_quantity":{fixed_token(event.fill_quantity)},' + f'"from_status":{string_token(event.from_status.value)},' + f'"order_id":{string_token(event.order_id)},' + f'"reason":{string_token(event.reason)},' + f'"sequence":{event.sequence},' + f'"to_status":{string_token(event.to_status.value)}' + "}" + ).encode() + + +def fill_bytes(fill: Fill) -> bytes: + venue_trade_id = "null" if fill.venue_trade_id is None else string_token(fill.venue_trade_id) + return ( + "{" + f'"account_id":{string_token(fill.account_id)},' + f'"event_time":{utc_token(fill.event_time)},' + f'"fill_id":{string_token(fill.fill_id)},' + f'"instrument_id":{string_token(fill.instrument_id)},' + f'"liquidity_role":{string_token(fill.liquidity_role.value)},' + f'"order_id":{string_token(fill.order_id)},' + f'"price":{fixed_token(fill.price)},' + f'"quantity":{fixed_token(fill.quantity)},' + f'"side":{string_token(fill.side.value)},' + f'"strategy_id":{string_token(fill.strategy_id)},' + f'"venue_trade_id":{venue_trade_id}' + "}" + ).encode() + + +def order_bytes(order: Order) -> bytes: + from quant_execution.broker import _intent_bytes + + return ( + "{" + f'"filled_quantity":{fixed_token(order.filled_quantity)},' + f'"intent":{_intent_bytes(order.intent).decode()},' + f'"order_id":{string_token(order.order_id)},' + f'"status":{string_token(order.status.value)},' + f'"version":{order.version}' + "}" + ).encode() + + +def fee_bytes(fee: Fee) -> bytes: + return ( + "{" + f'"account_id":{string_token(fee.account_id)},' + f'"amount":{fixed_token(fee.amount)},' + f'"currency":{string_token(fee.currency)},' + f'"event_time":{utc_token(fee.event_time)},' + f'"fee_id":{string_token(fee.fee_id)},' + f'"fee_type":{string_token(fee.fee_type)},' + f'"fill_id":{string_token(fee.fill_id)}' + "}" + ).encode() + + +def settlement_bytes(settlement: Settlement) -> bytes: + return ( + "{" + f'"account_id":{string_token(settlement.account_id)},' + f'"amount":{fixed_token(settlement.amount)},' + f'"currency":{string_token(settlement.currency)},' + f'"event_time":{utc_token(settlement.event_time)},' + f'"instrument_id":{string_token(settlement.instrument_id)},' + f'"settlement_id":{string_token(settlement.settlement_id)},' + f'"settlement_price":{fixed_token(settlement.settlement_price)},' + f'"settlement_type":{string_token(settlement.settlement_type)}' + "}" + ).encode() + + +def ledger_transaction_bytes(transaction: LedgerTransaction) -> bytes: + postings: list[str] = [] + for posting in transaction.postings: + instrument_id = ( + "null" if posting.instrument_id is None else string_token(posting.instrument_id) + ) + postings.append( + "{" + f'"amount":{fixed_token(posting.amount)},' + f'"currency":{string_token(posting.currency)},' + f'"instrument_id":{instrument_id},' + f'"ledger_account":{string_token(posting.ledger_account)},' + f'"quantity_delta":{fixed_token(posting.quantity_delta)}' + "}" + ) + return ( + "{" + f'"event_time":{utc_token(transaction.event_time)},' + f'"event_type":{string_token(transaction.event_type.value)},' + f'"idempotency_key":{string_token(transaction.idempotency_key)},' + f'"postings":[{",".join(postings)}],' + f'"reference_id":{string_token(transaction.reference_id)},' + f'"transaction_id":{string_token(transaction.transaction_id)}' + "}" + ).encode() + + +class _SequenceDigest: + """Incrementally hash a canonical JSON array without retaining its facts.""" + + def __init__(self) -> None: + self._digest = hashlib.sha256() + self._digest.update(b"[") + self._count = 0 + self._closed = False + + def append(self, payload: bytes) -> None: + if self._closed: + raise RuntimeError("artifact digest is already closed") + if self._count: + self._digest.update(b",") + self._digest.update(payload) + self._count += 1 + + def close(self) -> str: + if not self._closed: + self._digest.update(b"]") + self._closed = True + return self._digest.hexdigest() + + +@dataclass(frozen=True, slots=True) +class StoredRunArtifacts: + """Immutable handle to a completed on-disk replay artifact set.""" + + root: Path + manifest_path: Path + counts: Mapping[str, int] + logical_sha256: Mapping[str, str] + files: Mapping[str, Mapping[str, object]] + manifest_sha256: str + + def iter_payload_bytes(self, stream: str) -> Iterator[bytes]: + if stream not in _STREAMS: + raise ValidationError(f"unknown artifact stream: {stream}") + path = self.root / f"{stream}.arrow" + if not path.exists(): + return + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_stream(source) + for batch in reader: + for payload in batch.column("payload").to_pylist(): + yield bytes(payload) + + def iter_json(self, stream: str) -> Iterator[object]: + for payload in self.iter_payload_bytes(stream): + yield json.loads(payload) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_manifest_bytes(payload: Mapping[str, object]) -> bytes: + return ( + json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + b"\n" + ) + + +def _write_no_clobber(path: Path, body: bytes) -> None: + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(body) + handle.flush() + os.fsync(handle.fileno()) + os.link(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _manifest_hash(payload: Mapping[str, object]) -> str: + unsigned = dict(payload) + unsigned.pop("manifest_sha256", None) + return hashlib.sha256(_canonical_manifest_bytes(unsigned)).hexdigest() + + +def load_stored_artifacts(root: str | Path) -> StoredRunArtifacts: + """Strictly verify a completed artifact directory before exposing its facts.""" + + resolved = Path(root).resolve() + manifest_path = resolved / "manifest.json" + try: + raw = manifest_path.read_bytes() + payload = json.loads(raw) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValidationError(f"artifact manifest is unreadable: {manifest_path}") from exc + if not isinstance(payload, dict): + raise ValidationError("artifact manifest root must be an object") + expected_fields = { + "artifact_format", + "complete", + "counts", + "files", + "logical_sha256", + "manifest_sha256", + "run_metadata", + "schema_version", + } + if set(payload) != expected_fields: + raise ValidationError("artifact manifest fields changed") + if not isinstance(payload.get("run_metadata"), dict) or not payload["run_metadata"]: + raise ValidationError("artifact manifest contains no run metadata") + if payload.get("schema_version") != "1.0.0": + raise ValidationError("artifact manifest schema version is unsupported") + if payload.get("artifact_format") != "puresaber.arrow-canonical-json.v1": + raise ValidationError("artifact format is unsupported") + if payload.get("complete") is not True: + raise ValidationError("artifact run is not complete") + if raw != _canonical_manifest_bytes(payload): + raise ValidationError("artifact manifest bytes are not canonical") + manifest_sha256 = payload.get("manifest_sha256") + if not isinstance(manifest_sha256, str) or manifest_sha256 != _manifest_hash(payload): + raise ValidationError("artifact manifest hash mismatch") + counts = payload.get("counts") + logical = payload.get("logical_sha256") + files = payload.get("files") + if not isinstance(counts, dict) or set(counts) != set(_STREAMS): + raise ValidationError("artifact counts changed shape") + if not isinstance(logical, dict) or set(logical) != set(_STREAMS): + raise ValidationError("artifact logical hashes changed shape") + if not isinstance(files, dict): + raise ValidationError("artifact files must be an object") + verified_counts: dict[str, int] = {} + verified_logical: dict[str, str] = {} + verified_files: dict[str, dict[str, object]] = {} + for stream in _STREAMS: + count = counts[stream] + logical_sha256 = logical[stream] + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + raise ValidationError(f"artifact count is invalid: {stream}") + if not isinstance(logical_sha256, str) or not _SHA256.fullmatch(logical_sha256): + raise ValidationError(f"artifact logical hash is invalid: {stream}") + metadata = files.get(stream) + if count == 0: + if metadata is not None: + raise ValidationError(f"empty artifact stream unexpectedly has a file: {stream}") + digest = _SequenceDigest() + if digest.close() != logical_sha256: + raise ValidationError(f"empty artifact logical hash mismatch: {stream}") + verified_counts[stream] = 0 + verified_logical[stream] = logical_sha256 + continue + if not isinstance(metadata, dict) or set(metadata) != {"bytes", "path", "sha256"}: + raise ValidationError(f"artifact file metadata changed shape: {stream}") + relative = metadata["path"] + expected_relative = f"{stream}.arrow" + if relative != expected_relative: + raise ValidationError(f"artifact file path is invalid: {stream}") + path = (resolved / expected_relative).resolve() + try: + path.relative_to(resolved) + except ValueError as exc: + raise ValidationError(f"artifact file escapes its run root: {stream}") from exc + expected_bytes = metadata["bytes"] + expected_sha256 = metadata["sha256"] + if ( + isinstance(expected_bytes, bool) + or not isinstance(expected_bytes, int) + or expected_bytes <= 0 + or not isinstance(expected_sha256, str) + or not _SHA256.fullmatch(expected_sha256) + ): + raise ValidationError(f"artifact file metadata is invalid: {stream}") + if not path.is_file() or path.stat().st_size != expected_bytes: + raise ValidationError(f"artifact file is missing or changed size: {stream}") + if _sha256_file(path) != expected_sha256: + raise ValidationError(f"artifact file content hash mismatch: {stream}") + digest = _SequenceDigest() + observed = 0 + try: + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_stream(source) + if reader.schema != _SCHEMA: + raise ValidationError(f"artifact Arrow schema changed: {stream}") + for batch in reader: + sequences = batch.column("sequence").to_pylist() + expected_sequences = list(range(observed, observed + batch.num_rows)) + if sequences != expected_sequences: + raise ValidationError(f"artifact sequence is not contiguous: {stream}") + for item in batch.column("payload").to_pylist(): + digest.append(bytes(item)) + observed += batch.num_rows + except (OSError, pa.ArrowException) as exc: + raise ValidationError(f"artifact Arrow stream is unreadable: {stream}") from exc + if observed != count or digest.close() != logical_sha256: + raise ValidationError(f"artifact logical content mismatch: {stream}") + verified_counts[stream] = count + verified_logical[stream] = logical_sha256 + verified_files[stream] = dict(metadata) + if set(files) != set(verified_files): + raise ValidationError("artifact files contain unknown streams") + return StoredRunArtifacts( + root=resolved, + manifest_path=manifest_path, + counts=verified_counts, + logical_sha256=verified_logical, + files=verified_files, + manifest_sha256=manifest_sha256, + ) + + +class ArrowReplayArtifactSink: + """Write canonical replay facts to bounded Arrow record batches. + + The producer only retains at most ``batch_size`` payload references per stream. + A single background writer owns every Arrow stream, so replay and native Arrow + I/O can overlap without exposing partially written artifacts as complete runs. + """ + + def __init__( + self, + root: str | Path, + *, + batch_size: int = 65_536, + queue_batches: int = 8, + ) -> None: + if isinstance(batch_size, bool) or batch_size <= 0: + raise ValidationError("batch_size must be a positive integer") + if isinstance(queue_batches, bool) or queue_batches <= 0: + raise ValidationError("queue_batches must be a positive integer") + self.root = Path(root).resolve() + self.root.mkdir(parents=True, exist_ok=False) + self._batch_size = batch_size + self._buffers: dict[str, list[bytes]] = {name: [] for name in _STREAMS} + self._counts = {name: 0 for name in _STREAMS} + self._digests = {name: _SequenceDigest() for name in _STREAMS} + self._queue: queue.Queue[object] = queue.Queue(maxsize=queue_batches) + self._failure: BaseException | None = None + self._closed = False + self._sealed = False + self._staged: list[tuple[str, bytes]] | None = None + self._writers: dict[str, ipc.RecordBatchStreamWriter] = {} + self._files: dict[str, pa.NativeFile] = {} + self._thread = threading.Thread( + target=self._write_loop, + name="quant-execution-artifact-writer", + daemon=True, + ) + self._thread.start() + + @property + def counts(self) -> Mapping[str, int]: + return dict(self._counts) + + def append(self, stream: str, payload: bytes) -> None: + if self._closed or self._sealed: + raise RuntimeError("artifact sink no longer accepts records") + if stream not in self._buffers: + raise ValidationError(f"unknown artifact stream: {stream}") + if not isinstance(payload, bytes): + raise ValidationError("artifact payload must be canonical bytes") + if self._staged is not None: + self._staged.append((stream, payload)) + return + self._append_committed(stream, payload) + + def begin(self) -> None: + if self._staged is not None: + raise RuntimeError("nested artifact transactions are not supported") + if self._closed or self._sealed: + raise RuntimeError("artifact sink no longer accepts records") + self._staged = [] + + def commit(self) -> None: + staged = self._staged + if staged is None: + raise RuntimeError("no artifact transaction is active") + self._staged = None + for stream, payload in staged: + self._append_committed(stream, payload) + + def rollback(self) -> None: + if self._staged is None: + raise RuntimeError("no artifact transaction is active") + self._staged = None + + def _append_committed(self, stream: str, payload: bytes) -> None: + self._raise_writer_failure() + self._digests[stream].append(payload) + self._counts[stream] += 1 + buffer = self._buffers[stream] + buffer.append(payload) + if len(buffer) >= self._batch_size: + self._enqueue((stream, self._counts[stream] - len(buffer), buffer)) + self._buffers[stream] = [] + + def logical_sha256(self, stream: str) -> str: + if stream not in self._digests: + raise ValidationError(f"unknown artifact stream: {stream}") + return self._digests[stream].close() + + def close(self, manifest: Mapping[str, object]) -> StoredRunArtifacts: + if self._closed: + raise RuntimeError("artifact sink is already closed") + try: + self.seal() + logical = {name: digest.close() for name, digest in self._digests.items()} + files = { + name: { + "path": path.name, + "bytes": path.stat().st_size, + "sha256": _sha256_file(path), + } + for name in _STREAMS + for path in (self.root / f"{name}.arrow",) + if path.is_file() + } + completed = { + "schema_version": "1.0.0", + "artifact_format": "puresaber.arrow-canonical-json.v1", + "counts": dict(self._counts), + "logical_sha256": logical, + "files": files, + "complete": True, + "run_metadata": dict(manifest), + } + completed["manifest_sha256"] = _manifest_hash(completed) + manifest_path = self.root / "manifest.json" + _write_no_clobber(manifest_path, _canonical_manifest_bytes(completed)) + self._closed = True + return StoredRunArtifacts( + root=self.root, + manifest_path=manifest_path, + counts=dict(self._counts), + logical_sha256=logical, + files=files, + manifest_sha256=str(completed["manifest_sha256"]), + ) + except Exception: + self.abort() + raise + + def seal(self) -> None: + """Flush and close Arrow writers while leaving manifest finalization pending.""" + + if self._sealed: + return + if self._staged is not None: + raise RuntimeError("cannot seal an active artifact transaction") + for stream, buffer in self._buffers.items(): + if buffer: + self._enqueue((stream, self._counts[stream] - len(buffer), buffer)) + self._buffers[stream] = [] + self._enqueue(_STOP) + self._thread.join() + self._raise_writer_failure() + self._sealed = True + + def ledger_sha256( + self, + *, + fx_history: list[tuple[str, Decimal, datetime]], + marks: Mapping[str, tuple[Decimal, datetime, str]], + ) -> str: + """Reproduce the frozen ledger hash after bounded stream finalization.""" + + self.seal() + digest = hashlib.sha256() + digest.update(b'{"fx_snapshots":[') + for index, (currency, rate, event_time) in enumerate(fx_history): + if index: + digest.update(b",") + digest.update( + ( + "{" + f'"currency":{string_token(currency)},' + f'"event_time":{utc_token(event_time, zulu=False)},' + f'"rate":{string_token(str(rate))},' + f'"version":{index + 1}' + "}" + ).encode() + ) + digest.update(b'],"marks":[') + for index, (instrument_id, (price, event_time, event_id)) in enumerate( + sorted(marks.items()) + ): + if index: + digest.update(b",") + digest.update( + ( + "{" + f'"event_id":{string_token(event_id)},' + f'"event_time":{utc_token(event_time, zulu=False)},' + f'"instrument_id":{string_token(instrument_id)},' + f'"price":{string_token(str(price))}' + "}" + ).encode() + ) + digest.update(b'],"transactions":[') + for index, payload in enumerate(self._iter_payload_bytes("ledger_transactions")): + if index: + digest.update(b",") + digest.update(payload) + digest.update(b"]}") + return digest.hexdigest() + + def abort(self) -> None: + """Fail closed while preserving the incomplete directory for diagnosis.""" + + if self._closed: + return + for buffer in self._buffers.values(): + buffer.clear() + self._staged = None + deadline = time.monotonic() + 10 + while self._thread.is_alive() and time.monotonic() < deadline: + try: + self._queue.put(_STOP, timeout=0.1) + break + except queue.Full: + continue + self._thread.join(timeout=10) + if self._thread.is_alive(): + raise RuntimeError("artifact writer did not stop after abort") + failure = { + "artifact_format": "puresaber.arrow-canonical-json.v1", + "counts": dict(self._counts), + "complete": False, + } + try: + (self.root / "FAILED.json").write_text( + json.dumps(failure, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + finally: + self._closed = True + + def _iter_payload_bytes(self, stream: str) -> Iterator[bytes]: + path = self.root / f"{stream}.arrow" + if not path.exists(): + return + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_stream(source) + for batch in reader: + for payload in batch.column("payload").to_pylist(): + yield bytes(payload) + + def _write_loop(self) -> None: + try: + while True: + item = self._queue.get() + if item is _STOP: + break + stream, first_sequence, payloads = item + writer = self._writer(stream) + sequences = pa.array( + range(first_sequence, first_sequence + len(payloads)), type=pa.int64() + ) + values = pa.array(payloads, type=pa.large_binary()) + writer.write_batch(pa.record_batch([sequences, values], schema=_SCHEMA)) + for writer in self._writers.values(): + writer.close() + for output in self._files.values(): + output.close() + except Exception as exc: # noqa: BLE001 - thread boundary must relay every writer failure + self._failure = exc + for writer in self._writers.values(): + with suppress(Exception): + writer.close() + for output in self._files.values(): + with suppress(Exception): + output.close() + + def _writer(self, stream: str) -> ipc.RecordBatchStreamWriter: + prior = self._writers.get(stream) + if prior is not None: + return prior + output = pa.OSFile(str(self.root / f"{stream}.arrow"), "wb") + writer = ipc.new_stream(output, _SCHEMA) + self._files[stream] = output + self._writers[stream] = writer + return writer + + def _raise_writer_failure(self) -> None: + if self._failure is not None: + raise RuntimeError("artifact writer failed") from self._failure + + def _enqueue(self, item: object) -> None: + while True: + self._raise_writer_failure() + try: + self._queue.put(item, timeout=0.1) + return + except queue.Full: + continue + + +__all__ = [ + "ArrowReplayArtifactSink", + "StoredRunArtifacts", + "fee_bytes", + "fill_bytes", + "ledger_transaction_bytes", + "load_stored_artifacts", + "order_bytes", + "order_event_bytes", + "settlement_bytes", +] diff --git a/src/quant_execution/broker.py b/src/quant_execution/broker.py index d378b64..c9d6575 100644 --- a/src/quant_execution/broker.py +++ b/src/quant_execution/broker.py @@ -10,21 +10,93 @@ from quant_data_kit import FixedPoint from quant_data_kit.exceptions import ValidationError +from quant_execution._json import fixed_token, flat_sequence_bytes, string_token +from quant_execution.artifacts import fill_bytes, order_bytes, order_event_bytes from quant_execution.contracts import ( Fill, Order, OrderEvent, OrderIntent, OrderStatus, + OrderType, + Side, TimeInForce, ) -from quant_execution.schemas import execution_payload from quant_execution.state_machine import transition_order def _digest(prefix: str, *parts: object) -> str: - payload = json.dumps(parts, ensure_ascii=True, separators=(",", ":"), default=str) - return f"{prefix}-{hashlib.sha256(payload.encode()).hexdigest()[:24]}" + return f"{prefix}-{hashlib.sha256(flat_sequence_bytes(parts)).hexdigest()[:24]}" + + +def _intent_bytes(intent: OrderIntent) -> bytes: + """Serialize a validated intent exactly like sorted canonical execution_payload JSON.""" + + created_at = intent.created_at.isoformat().replace("+00:00", "Z") + return ( + "{" + f'"account_id":{string_token(intent.account_id)},' + f'"created_at":{string_token(created_at)},' + f'"idempotency_key":{string_token(intent.idempotency_key)},' + f'"instrument_id":{string_token(intent.instrument_id)},' + f'"limit_price":{fixed_token(intent.limit_price)},' + f'"order_type":{string_token(intent.order_type.value)},' + f'"quantity":{fixed_token(intent.quantity)},' + f'"reduce_only":{"true" if intent.reduce_only else "false"},' + f'"side":{string_token(intent.side.value)},' + f'"stop_price":{fixed_token(intent.stop_price)},' + f'"strategy_id":{string_token(intent.strategy_id)},' + f'"time_in_force":{string_token(intent.time_in_force.value)}' + "}" + ).encode() + + +def _fixed_from_payload(payload: dict[str, int] | None) -> FixedPoint | None: + return None if payload is None else FixedPoint(payload["units"], payload["scale"]) + + +def _time_from_payload(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def _order_from_bytes(payload: bytes) -> Order: + value = json.loads(payload) + raw_intent = value["intent"] + intent = OrderIntent( + idempotency_key=raw_intent["idempotency_key"], + account_id=raw_intent["account_id"], + strategy_id=raw_intent["strategy_id"], + instrument_id=raw_intent["instrument_id"], + side=Side(raw_intent["side"]), + quantity=_fixed_from_payload(raw_intent["quantity"]), + order_type=OrderType(raw_intent["order_type"]), + time_in_force=TimeInForce(raw_intent["time_in_force"]), + created_at=_time_from_payload(raw_intent["created_at"]), + limit_price=_fixed_from_payload(raw_intent["limit_price"]), + stop_price=_fixed_from_payload(raw_intent["stop_price"]), + reduce_only=raw_intent["reduce_only"], + ) + return Order( + order_id=value["order_id"], + intent=intent, + status=OrderStatus(value["status"]), + filled_quantity=_fixed_from_payload(value["filled_quantity"]), + version=value["version"], + ) + + +def _event_from_bytes(payload: bytes) -> OrderEvent: + value = json.loads(payload) + return OrderEvent( + event_id=value["event_id"], + order_id=value["order_id"], + event_time=_time_from_payload(value["event_time"]), + sequence=value["sequence"], + from_status=OrderStatus(value["from_status"]), + to_status=OrderStatus(value["to_status"]), + fill_quantity=_fixed_from_payload(value["fill_quantity"]), + reason=value["reason"], + ) class DeterministicBroker: @@ -36,40 +108,108 @@ def __init__(self) -> None: self.reset() def reset(self) -> None: - self._orders: dict[str, Order] = {} + self._orders: dict[str, Order | bytes] = {} + self._order_count = 0 self._open_order_ids: set[str] = set() + self._day_order_ids: set[str] = set() + self._immediate_order_ids: set[str] = set() self._submit_keys: dict[str, tuple[str, str]] = {} self._cancel_keys: dict[str, tuple[str, OrderEvent]] = {} - self._fill_keys: dict[str, tuple[Fill, OrderEvent]] = {} + self._fill_keys: dict[str, tuple[bytes, bytes] | tuple[Fill, OrderEvent]] = {} self._events: list[OrderEvent] = [] self._accepted_day: dict[str, date] = {} + self._artifact_sink = None + self._artifact_stream_closed = False + + def start_artifact_stream(self, sink: object) -> None: + """Route immutable history to a bounded sink while retaining lookup/idempotency state.""" + + self._require_mutable() + if self._orders or self._events or self._artifact_sink is not None: + raise ValidationError("broker artifact streaming must start immediately after reset") + if not callable(getattr(sink, "append", None)): + raise ValidationError("artifact sink must provide append(stream, payload)") + self._artifact_sink = sink + + def finish_artifact_stream(self) -> None: + """Persist the final state of orders that remained open at replay completion.""" + + sink = self._artifact_sink + if sink is None: + return + for order in self.open_orders: + sink.append("orders", order_bytes(order)) + self._artifact_sink = None + self._artifact_stream_closed = True + + def abort_artifact_stream(self) -> None: + if self._artifact_sink is not None: + self._artifact_sink = None + self._artifact_stream_closed = True + + def _require_mutable(self) -> None: + if self._artifact_stream_closed: + raise ValidationError("broker artifact stream is closed; reset is required") + + def _record_event(self, event: OrderEvent, order: Order) -> None: + sink = self._artifact_sink + if sink is None: + self._events.append(event) + return + sink.append("order_events", order_event_bytes(event)) + if order.status not in {OrderStatus.ACCEPTED, OrderStatus.PARTIALLY_FILLED}: + payload = order_bytes(order) + sink.append("orders", payload) + self._orders[order.order_id] = payload def capture_state(self) -> dict[str, object]: return deepcopy( { "orders": self._orders, + "order_count": self._order_count, "open_order_ids": self._open_order_ids, + "day_order_ids": self._day_order_ids, + "immediate_order_ids": self._immediate_order_ids, "submit_keys": self._submit_keys, "cancel_keys": self._cancel_keys, "fill_keys": self._fill_keys, "events": self._events, "accepted_day": self._accepted_day, + "artifact_stream_closed": self._artifact_stream_closed, } ) def restore_state(self, state: dict[str, object]) -> None: + """Restore a mutable checkpoint; sealed lifecycle recovery is engine-internal.""" + + self._require_mutable() + self._restore_captured_state(state) + + def _restore_captured_state(self, state: dict[str, object]) -> None: restored = deepcopy(state) self._orders = restored["orders"] + self._order_count = restored["order_count"] self._open_order_ids = restored["open_order_ids"] + self._day_order_ids = restored["day_order_ids"] + self._immediate_order_ids = restored["immediate_order_ids"] self._submit_keys = restored["submit_keys"] self._cancel_keys = restored["cancel_keys"] self._fill_keys = restored["fill_keys"] self._events = restored["events"] self._accepted_day = restored["accepted_day"] + self._artifact_stream_closed = restored["artifact_stream_closed"] @property def orders(self) -> tuple[Order, ...]: - return tuple(sorted(self._orders.values(), key=self._sort_key)) + orders = ( + _order_from_bytes(value) if isinstance(value, bytes) else value + for value in self._orders.values() + ) + return tuple(sorted(orders, key=self._sort_key)) + + @property + def order_count(self) -> int: + return self._order_count @property def order_events(self) -> tuple[OrderEvent, ...]: @@ -77,9 +217,28 @@ def order_events(self) -> tuple[OrderEvent, ...]: @property def open_orders(self) -> tuple[Order, ...]: + if not self._open_order_ids: + return () + if len(self._open_order_ids) == 1: + order_id = next(iter(self._open_order_ids)) + value = self._orders[order_id] + if isinstance(value, bytes): + raise RuntimeError("terminal order appeared in the open-order index") + return (value,) + return tuple( + sorted( + (self._live_order(order_id) for order_id in self._open_order_ids), + key=self._sort_key, + ) + ) + + @property + def immediate_orders(self) -> tuple[Order, ...]: + if not self._immediate_order_ids: + return () return tuple( sorted( - (self._orders[order_id] for order_id in self._open_order_ids), + (self._live_order(order_id) for order_id in self._immediate_order_ids), key=self._sort_key, ) ) @@ -94,12 +253,10 @@ def _sort_key(order: Order) -> tuple[datetime, str]: @staticmethod def _intent_hash(intent: OrderIntent) -> str: - payload = json.dumps( - execution_payload(intent), sort_keys=True, separators=(",", ":") - ).encode() - return hashlib.sha256(payload).hexdigest() + return hashlib.sha256(_intent_bytes(intent)).hexdigest() def submit(self, order_intent: OrderIntent) -> Order: + self._require_mutable() if not isinstance(order_intent, OrderIntent): raise ValidationError("order_intent must be an OrderIntent") semantic_hash = self._intent_hash(order_intent) @@ -108,22 +265,33 @@ def submit(self, order_intent: OrderIntent) -> Order: order_id, prior_hash = prior if prior_hash != semantic_hash: raise ValidationError("submit idempotency key reused with different intent") - return self._orders[order_id] + return self._require_order(order_id) order_id = _digest("ord", order_intent.idempotency_key, semantic_hash) - order = Order(order_id=order_id, intent=order_intent) - accepted, event = transition_order( - order, - OrderStatus.ACCEPTED, + filled = FixedPoint(0, order_intent.quantity.scale) + accepted = self._order_fact( + order_id, + order_intent, + status=OrderStatus.ACCEPTED, + filled_quantity=filled, + version=1, + ) + event = self._event_fact( event_id=_digest("oev", order_id, 1, "accepted"), - event_time=order_intent.created_at, + order=accepted, + from_status=OrderStatus.CREATED, + fill_quantity=None, ) self._orders[order_id] = accepted + self._order_count += 1 self._open_order_ids.add(order_id) + if order_intent.time_in_force in {TimeInForce.IOC, TimeInForce.FOK}: + self._immediate_order_ids.add(order_id) self._submit_keys[order_intent.idempotency_key] = (order_id, semantic_hash) - self._events.append(event) + self._record_event(event, accepted) return accepted def reject(self, order_intent: OrderIntent, *, code: str, message: str = "") -> Order: + self._require_mutable() if not code.strip(): raise ValidationError("rejection code is required") semantic_hash = self._intent_hash(order_intent) @@ -132,7 +300,7 @@ def reject(self, order_intent: OrderIntent, *, code: str, message: str = "") -> order_id, prior_hash = prior if prior_hash != semantic_hash: raise ValidationError("submit idempotency key reused with different intent") - return self._orders[order_id] + return self._require_order(order_id) order_id = _digest("ord", order_intent.idempotency_key, semantic_hash) order = Order(order_id=order_id, intent=order_intent) reason = code if not message else f"{code}: {message}" @@ -144,8 +312,9 @@ def reject(self, order_intent: OrderIntent, *, code: str, message: str = "") -> reason=reason, ) self._orders[order_id] = rejected + self._order_count += 1 self._submit_keys[order_intent.idempotency_key] = (order_id, semantic_hash) - self._events.append(event) + self._record_event(event, rejected) return rejected def cancel( @@ -155,6 +324,7 @@ def cancel( idempotency_key: str, created_at: datetime, ) -> OrderEvent: + self._require_mutable() if not idempotency_key.strip(): raise ValidationError("cancel idempotency_key is required") prior = self._cancel_keys.get(idempotency_key) @@ -175,13 +345,20 @@ def cancel( ) self._orders[order_id] = updated self._open_order_ids.remove(order_id) + self._day_order_ids.discard(order_id) + self._immediate_order_ids.discard(order_id) self._cancel_keys[idempotency_key] = (order_id, event) - self._events.append(event) + self._record_event(event, updated) return event - def apply_fill(self, fill: Fill) -> OrderEvent: - prior = self._fill_keys.get(fill.fill_id) + def apply_fill(self, fill: Fill, *, trusted_unique: bool = False) -> OrderEvent: + self._require_mutable() + prior = None if trusted_unique else self._fill_keys.get(fill.fill_id) if prior is not None: + if isinstance(prior[0], bytes): + if prior[0] != hashlib.sha256(fill_bytes(fill)).digest(): + raise ValidationError("fill_id reused with different fill content") + return _event_from_bytes(prior[1]) prior_fill, prior_event = prior if prior_fill != fill: raise ValidationError("fill_id reused with different fill content") @@ -199,27 +376,53 @@ def apply_fill(self, fill: Fill) -> OrderEvent: raise ValidationError("fill side differs from order intent") if fill.quantity.scale != order.intent.quantity.scale: raise ValidationError("fill quantity scale differs from order intent") + if not fill.quantity.is_positive(): + raise ValidationError("fill quantity must be positive") remaining = order.intent.quantity.units - order.filled_quantity.units if fill.quantity.units > remaining: raise ValidationError("fill would violate order quantity conservation") target = ( OrderStatus.FILLED if fill.quantity.units == remaining else OrderStatus.PARTIALLY_FILLED ) - updated, event = transition_order( - order, - target, - event_id=_digest("oev", order.order_id, order.version + 1, target.value, fill.fill_id), - event_time=fill.event_time, + if fill.event_time < order.intent.created_at: + raise ValidationError("order event_time cannot precede intent created_at") + next_version = order.version + 1 + filled_quantity = FixedPoint( + order.filled_quantity.units + fill.quantity.units, + order.filled_quantity.scale, + ) + updated = self._order_fact( + order.order_id, + order.intent, + status=target, + filled_quantity=filled_quantity, + version=next_version, + ) + event = self._event_fact( + event_id=_digest("oev", order.order_id, next_version, target.value, fill.fill_id), + order=updated, + from_status=order.status, fill_quantity=fill.quantity, + event_time=fill.event_time, ) self._orders[order.order_id] = updated if target is OrderStatus.FILLED: self._open_order_ids.remove(order.order_id) - self._events.append(event) - self._fill_keys[fill.fill_id] = (fill, event) + self._day_order_ids.discard(order.order_id) + self._immediate_order_ids.discard(order.order_id) + self._record_event(event, updated) + if not trusted_unique: + if self._artifact_sink is None: + self._fill_keys[fill.fill_id] = (fill, event) + else: + self._fill_keys[fill.fill_id] = ( + hashlib.sha256(fill_bytes(fill)).digest(), + order_event_bytes(event), + ) return event def expire(self, order_id: str, *, event_time: datetime, reason: str) -> OrderEvent: + self._require_mutable() order = self._require_order(order_id) if order.status not in {OrderStatus.ACCEPTED, OrderStatus.PARTIALLY_FILLED}: raise ValidationError("only open orders can expire") @@ -232,15 +435,28 @@ def expire(self, order_id: str, *, event_time: datetime, reason: str) -> OrderEv ) self._orders[order_id] = updated self._open_order_ids.remove(order_id) - self._events.append(event) + self._day_order_ids.discard(order_id) + self._immediate_order_ids.discard(order_id) + self._record_event(event, updated) return event def note_trading_day(self, order_id: str, trading_day: date) -> None: - self._accepted_day.setdefault(order_id, trading_day) + self._require_mutable() + order = self._orders.get(order_id) + if order is not None and order.intent.time_in_force is TimeInForce.DAY: + self._accepted_day.setdefault(order_id, trading_day) + self._day_order_ids.add(order_id) def expire_day_orders(self, trading_day: date, event_time: datetime) -> tuple[OrderEvent, ...]: + self._require_mutable() + if not self._day_order_ids: + return () expired: list[OrderEvent] = [] - for order in self.open_orders: + orders = sorted( + (self._orders[order_id] for order_id in self._day_order_ids), + key=self._sort_key, + ) + for order in orders: accepted_day = self._accepted_day.get(order.order_id) if ( order.intent.time_in_force is TimeInForce.DAY @@ -258,9 +474,53 @@ def expire_day_orders(self, trading_day: date, event_time: datetime) -> tuple[Or def _require_order(self, order_id: str) -> Order: try: - return self._orders[order_id] + value = self._orders[order_id] except KeyError as exc: raise ValidationError(f"unknown order_id: {order_id}") from exc + return _order_from_bytes(value) if isinstance(value, bytes) else value + + def _live_order(self, order_id: str) -> Order: + value = self._orders[order_id] + if isinstance(value, bytes): + raise TypeError("terminal order appeared in a live-order index") + return value + + @staticmethod + def _order_fact( + order_id: str, + intent: OrderIntent, + *, + status: OrderStatus, + filled_quantity: FixedPoint, + version: int, + ) -> Order: + order = object.__new__(Order) + object.__setattr__(order, "order_id", order_id) + object.__setattr__(order, "intent", intent) + object.__setattr__(order, "status", status) + object.__setattr__(order, "filled_quantity", filled_quantity) + object.__setattr__(order, "version", version) + return order + + @staticmethod + def _event_fact( + *, + event_id: str, + order: Order, + from_status: OrderStatus, + fill_quantity: FixedPoint | None, + event_time: datetime | None = None, + ) -> OrderEvent: + event = object.__new__(OrderEvent) + object.__setattr__(event, "event_id", event_id) + object.__setattr__(event, "order_id", order.order_id) + object.__setattr__(event, "event_time", event_time or order.intent.created_at) + object.__setattr__(event, "sequence", order.version) + object.__setattr__(event, "from_status", from_status) + object.__setattr__(event, "to_status", order.status) + object.__setattr__(event, "fill_quantity", fill_quantity) + object.__setattr__(event, "reason", "") + return event def remaining_quantity(order: Order) -> FixedPoint: diff --git a/src/quant_execution/contracts.py b/src/quant_execution/contracts.py index 0904a66..48807f5 100644 --- a/src/quant_execution/contracts.py +++ b/src/quant_execution/contracts.py @@ -6,7 +6,6 @@ from collections.abc import Mapping from dataclasses import dataclass, field from datetime import datetime -from decimal import Decimal from enum import Enum from types import MappingProxyType from typing import TypeAlias @@ -437,12 +436,33 @@ def __post_init__(self) -> None: or any(not isinstance(posting, Posting) for posting in self.postings) ): raise ValidationError("ledger transaction requires an immutable tuple of postings") - balances: dict[str, Decimal] = {} - for posting in self.postings: - balances[posting.currency] = ( - balances.get(posting.currency, Decimal(0)) + posting.amount.to_decimal() - ) - unbalanced = {currency: total for currency, total in balances.items() if total != 0} + first = self.postings[0] + same_scale_currency = all( + posting.currency == first.currency and posting.amount.scale == first.amount.scale + for posting in self.postings[1:] + ) + if same_scale_currency: + balances = { + first.currency: ( + sum(posting.amount.units for posting in self.postings), + first.amount.scale, + ) + } + else: + balances: dict[str, tuple[int, int]] = {} + for posting in self.postings: + prior_units, prior_scale = balances.get(posting.currency, (0, posting.amount.scale)) + scale = max(prior_scale, posting.amount.scale) + balances[posting.currency] = ( + prior_units * 10 ** (scale - prior_scale) + + posting.amount.units * 10 ** (scale - posting.amount.scale), + scale, + ) + unbalanced = { + currency: {"units": units, "scale": scale} + for currency, (units, scale) in balances.items() + if units != 0 + } if unbalanced: raise ValidationError(f"ledger transaction is unbalanced: {unbalanced}") diff --git a/src/quant_execution/engine.py b/src/quant_execution/engine.py index 4a56c5b..17c07b0 100644 --- a/src/quant_execution/engine.py +++ b/src/quant_execution/engine.py @@ -8,6 +8,7 @@ from copy import deepcopy from dataclasses import dataclass from datetime import datetime +from itertools import chain from typing import Any from quant_data_kit import ( @@ -24,6 +25,14 @@ ) from quant_data_kit.exceptions import ValidationError +from quant_execution._json import fixed_token, string_token, utc_token +from quant_execution.artifacts import ( + ArrowReplayArtifactSink, + StoredRunArtifacts, + fee_bytes, + fill_bytes, + settlement_bytes, +) from quant_execution.broker import DeterministicBroker from quant_execution.contracts import ( Fee, @@ -35,9 +44,9 @@ OrderStatus, RunResult, Settlement, - TimeInForce, ) from quant_execution.ledger import ExactAccountLedger +from quant_execution.matching import BarMatchingModel from quant_execution.protocols import MatchingModel, Strategy, StrategyContext from quant_execution.rules import RuleBookRiskGate from quant_execution.schemas import execution_payload @@ -72,6 +81,24 @@ class RunArtifacts: result: RunResult +class _SinkCollection: + """List-shaped adapter that serializes committed facts directly to a sink.""" + + __slots__ = ("_encoder", "_sink", "_stream") + + def __init__(self, sink: ArrowReplayArtifactSink, stream: str, encoder) -> None: + self._sink = sink + self._stream = stream + self._encoder = encoder + + def append(self, value: object) -> None: + self._sink.append(self._stream, self._encoder(value)) + + def extend(self, values: Iterable[object]) -> None: + for value in values: + self.append(value) + + def _canonical(value: Any) -> bytes: return json.dumps( value, @@ -87,6 +114,58 @@ def _hash(records: Sequence[object]) -> str: return hashlib.sha256(_canonical(records)).hexdigest() +def _order_event_bytes(event: OrderEvent) -> bytes: + return ( + "{" + f'"event_id":{string_token(event.event_id)},' + f'"event_time":{utc_token(event.event_time)},' + f'"fill_quantity":{fixed_token(event.fill_quantity)},' + f'"from_status":{string_token(event.from_status.value)},' + f'"order_id":{string_token(event.order_id)},' + f'"reason":{string_token(event.reason)},' + f'"sequence":{event.sequence},' + f'"to_status":{string_token(event.to_status.value)}' + "}" + ).encode() + + +def _fill_bytes(fill: Fill) -> bytes: + venue_trade_id = "null" if fill.venue_trade_id is None else string_token(fill.venue_trade_id) + return ( + "{" + f'"account_id":{string_token(fill.account_id)},' + f'"event_time":{utc_token(fill.event_time)},' + f'"fill_id":{string_token(fill.fill_id)},' + f'"instrument_id":{string_token(fill.instrument_id)},' + f'"liquidity_role":{string_token(fill.liquidity_role.value)},' + f'"order_id":{string_token(fill.order_id)},' + f'"price":{fixed_token(fill.price)},' + f'"quantity":{fixed_token(fill.quantity)},' + f'"side":{string_token(fill.side.value)},' + f'"strategy_id":{string_token(fill.strategy_id)},' + f'"venue_trade_id":{venue_trade_id}' + "}" + ).encode() + + +def _fact_hash(records: Sequence[OrderEvent] | Sequence[Fill]) -> str: + """Hash built-in immutable facts without constructing a duplicate dict graph.""" + + digest = hashlib.sha256() + digest.update(b"[") + for index, record in enumerate(records): + if index: + digest.update(b",") + if isinstance(record, OrderEvent): + digest.update(_order_event_bytes(record)) + elif isinstance(record, Fill): + digest.update(_fill_bytes(record)) + else: + return _hash([execution_payload(item) for item in records]) + digest.update(b"]") + return digest.hexdigest() + + def _event_sort_key(event: MarketEvent) -> tuple[object, ...]: return ( event.available_at, @@ -129,6 +208,8 @@ def __init__( self.matching_model = matching_model self.ledger = ledger self.artifacts: RunArtifacts | None = None + self.stored_artifacts: StoredRunArtifacts | None = None + self._active_sink: ArrowReplayArtifactSink | None = None def replay(self, event_stream: Iterable[MarketEvent], seed: int) -> RunResult: if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: @@ -229,10 +310,11 @@ def replay(self, event_stream: Iterable[MarketEvent], seed: int) -> RunResult: intents = self._strategy_intents(context, event) for intent in intents: if type(self.risk_gate).check is RuleBookRiskGate.check: - decision = self.risk_gate.check_current( + decision, reservation = self.risk_gate._check_current_for_submit( intent, event_time=event.available_at ) else: + reservation = None account_snapshot = account_snapshot or self.ledger.snapshot( event.available_at ) @@ -244,7 +326,10 @@ def replay(self, event_stream: Iterable[MarketEvent], seed: int) -> RunResult: OrderStatus.ACCEPTED, OrderStatus.PARTIALLY_FILLED, }: - self.risk_gate.reserve(intent) + if reservation is None: + self.risk_gate.reserve(intent) + else: + self.risk_gate._reserve_requirement(intent, reservation) else: self.broker.reject(intent, code=decision.code, message=decision.message) risk_events.append( @@ -256,22 +341,21 @@ def replay(self, event_stream: Iterable[MarketEvent], seed: int) -> RunResult: raise ReplayError(f"replay failed closed at {event_id}: {exc}") from exc try: - order_payloads = [execution_payload(item) for item in self.broker.order_events] - fill_payloads = [execution_payload(item) for item in fills] + order_events = self.broker.order_events result = RunResult( run_id=self.run_id, seed=seed, event_count=len(events), order_count=len(self.broker.orders), fill_count=len(fills), - event_sha256=_hash(order_payloads), - fill_sha256=_hash(fill_payloads), + event_sha256=_fact_hash(order_events), + fill_sha256=_fact_hash(fills), ledger_sha256=self.ledger.journal_sha256, ) self.artifacts = RunArtifacts( market_events=events, orders=self.broker.orders, - order_events=self.broker.order_events, + order_events=order_events, fills=tuple(fills), fees=tuple(fees), settlements=tuple(settlements), @@ -284,6 +368,211 @@ def replay(self, event_stream: Iterable[MarketEvent], seed: int) -> RunResult: self._restore_state(checkpoint) raise ReplayError(f"replay failed closed during finalization: {exc}") from exc + def replay_to_sink( + self, + event_stream: Iterable[MarketEvent], + seed: int, + sink: ArrowReplayArtifactSink, + ) -> RunResult: + """Replay a pre-sorted event stream into bounded Arrow artifacts. + + This opt-in migration path preserves ``replay`` and ``RunArtifacts`` while avoiding + complete in-memory artifact retention. The input must already use the public deterministic + sort order; accepting and sorting the full stream would retain every input event. + """ + + if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: + raise ValidationError("seed must be a non-negative integer") + if not isinstance(sink, ArrowReplayArtifactSink): + raise ValidationError("sink must be an ArrowReplayArtifactSink") + checkpoint = self._capture_state() + event: MarketEvent | None = None + event_count = 0 + seen_event_ids: set[str] = set() + prior_sort_key: tuple[object, ...] | None = None + iterator = iter(event_stream) + try: + first = next(iterator, None) + if first is not None and not isinstance(first, _EVENT_TYPES): + raise ValidationError("event_stream contains a non-MarketEvent value") + self._reset(opened_at=first.available_at if first is not None else None) + self.broker.start_artifact_stream(sink) + self.ledger.start_artifact_stream(sink) + self._active_sink = sink + context = StrategyContext( + run_id=self.run_id, + account_id=self.account_id, + strategy_id=self.strategy_id, + seed=seed, + state={}, + ) + fills = _SinkCollection(sink, "fills", fill_bytes) + fees = _SinkCollection(sink, "fees", fee_bytes) + settlements = _SinkCollection(sink, "settlements", settlement_bytes) + risk_events = _SinkCollection( + sink, "risk_events", lambda value: string_token(str(value)).encode() + ) + seen_fill_ids: set[str] = set() + records = () if first is None else chain((first,), iterator) + for value in records: + if not isinstance(value, _EVENT_TYPES): + raise ValidationError("event_stream contains a non-MarketEvent value") + event = value + sort_key = _event_sort_key(event) + if prior_sort_key is not None and sort_key < prior_sort_key: + raise ValidationError("streaming event_stream is not deterministically sorted") + if event.event_id in seen_event_ids: + raise ValidationError(f"duplicate MarketEvent event_id: {event.event_id}") + seen_event_ids.add(event.event_id) + prior_sort_key = sort_key + event_count += 1 + + day_expiries = self.broker.expire_day_orders(event.trading_day, event.available_at) + for expiry in day_expiries: + self.risk_gate.release_order(self._order(expiry.order_id)) + self.risk_gate.observe(event) + account_snapshot = self.ledger.observe_market( + event, + create_snapshot=False, + trusted_unique=True, + ) + if isinstance(event, CorporateActionEvent): + account_snapshot = self.ledger.apply(event, create_snapshot=False) + elif isinstance(event, FundingRateEvent): + funding = self.ledger.funding_from_market(event) + if funding is not None: + account_snapshot = self.ledger.apply(funding, create_snapshot=False) + elif isinstance(event, StatusEvent): + settlement = self.ledger.settlement_from_market(event) + if settlement is not None: + account_snapshot = self.ledger.apply(settlement, create_snapshot=False) + settlements.append(settlement) + + if self.broker.open_orders: + for order in self.broker.open_orders: + if ( + type(self.risk_gate).check_open_order + is RuleBookRiskGate.check_open_order + ): + decision = self.risk_gate.check_open_order_current( + order, event_time=event.available_at + ) + else: + account_snapshot = account_snapshot or self.ledger.snapshot( + event.available_at + ) + decision = self.risk_gate.check_open_order( + order, + account_snapshot, + event_time=event.available_at, + ) + if not decision.accepted: + self.broker.expire( + order.order_id, + event_time=event.available_at, + reason=f"{decision.code}: {decision.message}", + ) + self.risk_gate.release_order(order) + risk_events.append( + f"{order.order_id}:{decision.code}:{decision.message}" + ) + + if self._match_and_commit( + event, + fills=fills, + fees=fees, + risk_events=risk_events, + seen_fill_ids=seen_fill_ids, + ): + account_snapshot = None + + self._expire_immediate_orders(event) + if type(self.risk_gate).runtime_check is RuleBookRiskGate.runtime_check: + runtime = self.risk_gate.runtime_check_current(event.available_at) + else: + account_snapshot = account_snapshot or self.ledger.snapshot(event.available_at) + runtime = self.risk_gate.runtime_check(account_snapshot) + if not runtime.accepted: + risk_events.append(f"{event.event_id}:{runtime.code}:{runtime.message}") + for order in self.broker.open_orders: + self.broker.expire( + order.order_id, + event_time=event.available_at, + reason=runtime.code, + ) + self.risk_gate.release_order(order) + + intents = self._strategy_intents(context, event) + for intent in intents: + if type(self.risk_gate).check is RuleBookRiskGate.check: + decision, reservation = self.risk_gate._check_current_for_submit( + intent, event_time=event.available_at + ) + else: + reservation = None + account_snapshot = account_snapshot or self.ledger.snapshot( + event.available_at + ) + decision = self.risk_gate.check(intent, account_snapshot) + if decision.accepted: + order = self.broker.submit(intent) + self.broker.note_trading_day(order.order_id, event.trading_day) + if order.status in { + OrderStatus.ACCEPTED, + OrderStatus.PARTIALLY_FILLED, + }: + if reservation is None: + self.risk_gate.reserve(intent) + else: + self.risk_gate._reserve_requirement(intent, reservation) + else: + self.broker.reject(intent, code=decision.code, message=decision.message) + risk_events.append( + f"{intent.idempotency_key}:{decision.code}:{decision.message}" + ) + + self.broker.finish_artifact_stream() + sink.seal() + ledger_sha256 = sink.ledger_sha256( + fx_history=self.ledger._fx_history, + marks=self.ledger._marks, + ) + self.ledger.finish_artifact_stream(journal_sha256=ledger_sha256) + self._active_sink = None + result = RunResult( + run_id=self.run_id, + seed=seed, + event_count=event_count, + order_count=self.broker.order_count, + fill_count=sink.counts["fills"], + event_sha256=sink.logical_sha256("order_events"), + fill_sha256=sink.logical_sha256("fills"), + ledger_sha256=ledger_sha256, + ) + self.stored_artifacts = sink.close( + { + "run_id": self.run_id, + "seed": seed, + "event_count": event_count, + "order_count": result.order_count, + "fill_count": result.fill_count, + "order_sha256": result.order_sha256, + "fill_sha256": result.fill_sha256, + "ledger_sha256": result.ledger_sha256, + "result_sha256": result.result_sha256, + } + ) + self.artifacts = None + return result + except Exception as exc: + self._active_sink = None + self.broker.abort_artifact_stream() + self.ledger.abort_artifact_stream() + sink.abort() + self._restore_state(checkpoint) + event_id = event.event_id if event is not None else "before-first-event" + raise ReplayError(f"streaming replay failed closed at {event_id}: {exc}") from exc + def _reset(self, *, opened_at: datetime | None = None) -> None: self.broker.reset() self.ledger.reset(opened_at=opened_at) @@ -295,6 +584,7 @@ def _reset(self, *, opened_at: datetime | None = None) -> None: if callable(strategy_reset): strategy_reset() self.artifacts = None + self.stored_artifacts = None def _match_and_commit( self, @@ -317,7 +607,12 @@ def _match_and_commit( ) return False - matching_checkpoint = self._capture_component(self.matching_model) + matching_checkpoint = ( + None + if type(self.matching_model) is BarMatchingModel + and not self.matching_model.checkpoint_required(open_orders) + else self._capture_component(self.matching_model) + ) matched = tuple(self.matching_model.match(event, open_orders)) if not matched: return False @@ -334,7 +629,8 @@ def _match_and_commit( ) decision = self.risk_gate.check_fill(fill, order) if not decision.accepted: - self._restore_component(self.matching_model, matching_checkpoint) + if matching_checkpoint is not None: + self._restore_component(self.matching_model, matching_checkpoint) if risk_checkpoint is not None: self._restore_component(self.risk_gate, risk_checkpoint) self._expire_fill_rejections( @@ -351,11 +647,7 @@ def _match_and_commit( fills.append(fill) return True - broker_checkpoint = self._capture_component(self.broker) - ledger_checkpoint = self._capture_component(self.ledger) risk_checkpoint = self._capture_component(self.risk_gate) - staged_fills: list[Fill] = [] - staged_fees: list[Fee] = [] rejected: dict[str, tuple[str, str]] = {} for fill in matched: order = self._order(fill.order_id) @@ -364,16 +656,9 @@ def _match_and_commit( decision = self.risk_gate.check_fill(fill, order) if not decision.accepted: rejected[order.order_id] = (decision.code, decision.message) - continue - fee = self._commit_fill(fill, order, event) - staged_fills.append(fill) - if fee is not None: - staged_fees.append(fee) - if rejected: - self._restore_component(self.matching_model, matching_checkpoint) - self._restore_component(self.broker, broker_checkpoint) - self._restore_component(self.ledger, ledger_checkpoint) + if matching_checkpoint is not None: + self._restore_component(self.matching_model, matching_checkpoint) self._restore_component(self.risk_gate, risk_checkpoint) self._expire_fill_rejections( event, @@ -385,6 +670,25 @@ def _match_and_commit( ) continue + staged_fills: list[Fill] = [] + staged_fees: list[Fee] = [] + sink = self._active_sink + if sink is not None: + sink.begin() + try: + for fill in matched: + order = self._order(fill.order_id) + fee = self._commit_fill(fill, order, event) + staged_fills.append(fill) + if fee is not None: + staged_fees.append(fee) + except Exception: + if sink is not None: + sink.rollback() + raise + + if sink is not None: + sink.commit() fills.extend(staged_fills) fees.extend(staged_fees) seen_fill_ids.update(fill.fill_id for fill in staged_fills) @@ -399,16 +703,25 @@ def _validate_candidate_fill_ids(matched: Sequence[Fill], seen_fill_ids: set[str attempt_fill_ids.add(fill.fill_id) def _commit_fill(self, fill: Fill, order: Order, event: MarketEvent) -> Fee | None: - self.broker.apply_fill(fill) + if type(self.broker) is DeterministicBroker and self._active_sink is not None: + self.broker.apply_fill(fill, trusted_unique=True) + else: + self.broker.apply_fill(fill) self.risk_gate.release_fill(fill, order) - self.ledger.apply_with_trading_day( - fill, - trading_day=event.trading_day, - create_snapshot=False, - ) + if type(self.ledger) is ExactAccountLedger: + self.ledger._apply_replay_event(fill, trading_day=event.trading_day) + else: + self.ledger.apply_with_trading_day( + fill, + trading_day=event.trading_day, + create_snapshot=False, + ) fee = self.risk_gate.fee_for(fill, order) if fee is not None: - self.ledger.apply(fee, create_snapshot=False) + if type(self.ledger) is ExactAccountLedger: + self.ledger._apply_replay_event(fee) + else: + self.ledger.apply(fee, create_snapshot=False) return fee def _expire_fill_rejections( @@ -457,6 +770,10 @@ def _capture_component(component: object) -> tuple[str, object]: def _restore_component(component: object, checkpoint: tuple[str, object]) -> None: mode, state = checkpoint if mode == "explicit": + trusted_restore = getattr(component, "_restore_captured_state", None) + if callable(trusted_restore): + trusted_restore(state) + return restore = getattr(component, "restore_state", None) if not callable(restore): raise ValidationError(f"component {type(component).__name__} lost restore_state") @@ -474,6 +791,7 @@ def _capture_state(self) -> dict[str, object]: "matching_model": self._capture_component(self.matching_model), "strategy": self._capture_component(self.strategy), "artifacts": deepcopy(self.artifacts), + "stored_artifacts": deepcopy(self.stored_artifacts), } def _restore_state(self, checkpoint: dict[str, object]) -> None: @@ -483,6 +801,7 @@ def _restore_state(self, checkpoint: dict[str, object]) -> None: self._restore_component(self.matching_model, checkpoint["matching_model"]) self._restore_component(self.strategy, checkpoint["strategy"]) self.artifacts = checkpoint["artifacts"] + self.stored_artifacts = checkpoint["stored_artifacts"] @staticmethod def _validated_events(event_stream: Iterable[MarketEvent]) -> tuple[MarketEvent, ...]: @@ -518,9 +837,12 @@ def _expire_immediate_orders(self, event: MarketEvent) -> None: eligible = getattr(self.matching_model, "eligible", None) if not callable(eligible): return - for order in self.broker.open_orders: - if order.intent.time_in_force not in {TimeInForce.IOC, TimeInForce.FOK}: - continue + orders = ( + self.broker.immediate_orders + if type(self.broker) is DeterministicBroker + else self.broker.open_orders + ) + for order in orders: if eligible(order, event): self.broker.expire( order.order_id, diff --git a/src/quant_execution/ledger.py b/src/quant_execution/ledger.py index b6992ef..250e897 100644 --- a/src/quant_execution/ledger.py +++ b/src/quant_execution/ledger.py @@ -9,6 +9,7 @@ from datetime import date, datetime, timezone from decimal import ROUND_HALF_EVEN, Decimal from functools import lru_cache +from types import MappingProxyType from quant_data_kit import ( AssetClass, @@ -24,10 +25,18 @@ StatusEvent, TradeEvent, ensure_utc_datetime, + market_event_payload, ) from quant_data_kit.exceptions import ValidationError from quant_execution._fixed import decimal, fixed +from quant_execution._json import fixed_token, flat_sequence_bytes, string_token, utc_token +from quant_execution.artifacts import ( + fee_bytes, + fill_bytes, + ledger_transaction_bytes, + settlement_bytes, +) from quant_execution.contracts import ( AccountSnapshot, Fee, @@ -57,7 +66,7 @@ def _canonical(value: object) -> bytes: def _identifier(prefix: str, *parts: object) -> str: - return f"{prefix}-{hashlib.sha256(_canonical(parts)).hexdigest()[:24]}" + return f"{prefix}-{hashlib.sha256(flat_sequence_bytes(parts)).hexdigest()[:24]}" @lru_cache(maxsize=256) @@ -96,15 +105,15 @@ def __init__( raise ValidationError("account_id and base_currency are required") if not 0 <= money_scale <= 18: raise ValidationError("money_scale must be in [0, 18]") - self.account_id = account_id - self.base_currency = _currency(base_currency, "base_currency") - self.instruments = dict(instruments) + self._account_id = account_id + self._base_currency = _currency(base_currency, "base_currency") + self._instruments = MappingProxyType(dict(instruments)) self._derivative_instruments = frozenset( instrument_id - for instrument_id, spec in self.instruments.items() + for instrument_id, spec in self._instruments.items() if self._is_derivative(spec) ) - self.money_scale = money_scale + self._money_scale = money_scale self._initial_cash = dict(initial_cash or {}) self._initial_fx = dict(fx_to_base or {}) self._default_opened_at = ( @@ -121,8 +130,12 @@ def reset(self, *, opened_at: datetime | None = None) -> None: else self._default_opened_at ) self._transactions: list[LedgerTransaction] = [] + self._transaction_count = 0 + self._artifact_sink = None + self._artifact_stream_closed = False + self._finalized_journal_sha256: str | None = None self._transaction_keys: set[str] = set() - self._event_fingerprints: dict[str, LedgerEvent] = {} + self._event_fingerprints: dict[str, LedgerEvent | bytes] = {} self._fills: dict[str, Fill] = {} self._accounts: dict[tuple[str, str, str | None], Decimal] = {} self._positions: dict[str, Decimal] = {} @@ -157,10 +170,55 @@ def reset(self, *, opened_at: datetime | None = None) -> None: ) self._post(transaction) + def start_artifact_stream(self, sink: object) -> None: + """Move journal retention to a bounded artifact sink after reset.""" + + self._require_mutable() + if self._artifact_sink is not None: + raise ValidationError("ledger artifact stream is already active") + if not callable(getattr(sink, "append", None)): + raise ValidationError("artifact sink must provide append(stream, payload)") + self._artifact_sink = sink + for transaction in self._transactions: + sink.append("ledger_transactions", ledger_transaction_bytes(transaction)) + self._transaction_count = len(self._transactions) + self._transactions.clear() + + def finish_artifact_stream(self, *, journal_sha256: str | None = None) -> str | None: + if self._artifact_sink is None: + return self._finalized_journal_sha256 + if journal_sha256 is None: + calculate = getattr(self._artifact_sink, "ledger_sha256", None) + if not callable(calculate): + raise ValidationError("artifact sink cannot finalize the ledger journal hash") + journal_sha256 = calculate(fx_history=self._fx_history, marks=self._marks) + if ( + not isinstance(journal_sha256, str) + or len(journal_sha256) != 64 + or any(character not in "0123456789abcdef" for character in journal_sha256) + ): + raise ValidationError("finalized ledger journal hash must be lowercase SHA-256") + self._finalized_journal_sha256 = journal_sha256 + self._artifact_sink = None + self._artifact_stream_closed = True + return journal_sha256 + + def abort_artifact_stream(self) -> None: + if self._artifact_sink is not None: + self._artifact_sink = None + self._artifact_stream_closed = True + + def _require_mutable(self) -> None: + if self._artifact_stream_closed: + raise ValidationError("ledger artifact stream is closed; reset is required") + def capture_state(self) -> dict[str, object]: return deepcopy( { "transactions": self._transactions, + "transaction_count": self._transaction_count, + "artifact_stream_closed": self._artifact_stream_closed, + "finalized_journal_sha256": self._finalized_journal_sha256, "transaction_keys": self._transaction_keys, "event_fingerprints": self._event_fingerprints, "fills": self._fills, @@ -179,8 +237,17 @@ def capture_state(self) -> dict[str, object]: ) def restore_state(self, state: dict[str, object]) -> None: + """Restore a mutable checkpoint; sealed lifecycle recovery is engine-internal.""" + + self._require_mutable() + self._restore_captured_state(state) + + def _restore_captured_state(self, state: dict[str, object]) -> None: restored = deepcopy(state) self._transactions = restored["transactions"] + self._transaction_count = restored["transaction_count"] + self._artifact_stream_closed = restored["artifact_stream_closed"] + self._finalized_journal_sha256 = restored["finalized_journal_sha256"] self._transaction_keys = restored["transaction_keys"] self._event_fingerprints = restored["event_fingerprints"] self._fills = restored["fills"] @@ -200,32 +267,108 @@ def restore_state(self, state: dict[str, object]) -> None: def transactions(self) -> tuple[LedgerTransaction, ...]: return tuple(self._transactions) + @property + def account_id(self) -> str: + return self._account_id + + @property + def base_currency(self) -> str: + return self._base_currency + + @property + def money_scale(self) -> int: + return self._money_scale + + @property + def instruments(self) -> Mapping[str, InstrumentSpec]: + return self._instruments + + @property + def transaction_count(self) -> int: + return ( + self._transaction_count + if self._artifact_sink is not None or self._artifact_stream_closed + else len(self._transactions) + ) + @property def journal_sha256(self) -> str: - payload = { - "transactions": [execution_payload(item) for item in self._transactions], - "marks": [ - { - "instrument_id": instrument_id, - "price": str(price), - "event_time": event_time.isoformat(), - "event_id": event_id, - } - for instrument_id, (price, event_time, event_id) in sorted(self._marks.items()) - ], - "fx_snapshots": [ - { - "currency": currency, - "rate": str(rate), - "event_time": event_time.isoformat(), - "version": index + 1, - } - for index, (currency, rate, event_time) in enumerate(self._fx_history) - ], - } - return hashlib.sha256(_canonical(payload)).hexdigest() + if self._finalized_journal_sha256 is not None: + return self._finalized_journal_sha256 + if self._artifact_sink is not None: + raise ValidationError("ledger journal hash is unavailable until artifact finalization") + if self._artifact_stream_closed: + raise ValidationError( + "ledger journal hash is unavailable after artifact abort; reset required" + ) + digest = hashlib.sha256() + digest.update(b'{"fx_snapshots":[') + for index, (currency, rate, event_time) in enumerate(self._fx_history): + if index: + digest.update(b",") + digest.update( + ( + "{" + f'"currency":{string_token(currency)},' + f'"event_time":{utc_token(event_time, zulu=False)},' + f'"rate":{string_token(str(rate))},' + f'"version":{index + 1}' + "}" + ).encode() + ) + digest.update(b'],"marks":[') + for index, (instrument_id, (price, event_time, event_id)) in enumerate( + sorted(self._marks.items()) + ): + if index: + digest.update(b",") + digest.update( + ( + "{" + f'"event_id":{string_token(event_id)},' + f'"event_time":{utc_token(event_time, zulu=False)},' + f'"instrument_id":{string_token(instrument_id)},' + f'"price":{string_token(str(price))}' + "}" + ).encode() + ) + digest.update(b'],"transactions":[') + for index, transaction in enumerate(self._transactions): + if index: + digest.update(b",") + digest.update(self._transaction_bytes(transaction)) + digest.update(b"]}") + return digest.hexdigest() + + @staticmethod + def _transaction_bytes(transaction: LedgerTransaction) -> bytes: + postings: list[str] = [] + for posting in transaction.postings: + instrument_id = ( + "null" if posting.instrument_id is None else string_token(posting.instrument_id) + ) + postings.append( + "{" + f'"amount":{fixed_token(posting.amount)},' + f'"currency":{string_token(posting.currency)},' + f'"instrument_id":{instrument_id},' + f'"ledger_account":{string_token(posting.ledger_account)},' + f'"quantity_delta":{fixed_token(posting.quantity_delta)}' + "}" + ) + return ( + "{" + f'"event_time":{utc_token(transaction.event_time)},' + f'"event_type":{string_token(transaction.event_type.value)},' + f'"idempotency_key":{string_token(transaction.idempotency_key)},' + f'"postings":[{",".join(postings)}],' + f'"reference_id":{string_token(transaction.reference_id)},' + f'"transaction_id":{string_token(transaction.transaction_id)}' + "}" + ).encode() def set_fx_rate(self, currency: str, rate: FixedPoint, *, event_time: datetime) -> None: + self._require_mutable() currency = _currency(currency) event_time = ensure_utc_datetime(event_time, field="event_time") value = decimal(rate) @@ -259,9 +402,16 @@ def convert_to_base( def cash_balance(self, currency: str) -> Decimal: return self._accounts.get(("assets:cash", currency, None), Decimal(0)) + @property + def has_open_derivative_position(self) -> bool: + return any( + quantity != 0 and instrument_id in self._derivative_instruments + for instrument_id, quantity in self._positions.items() + ) + def risk_balances( self, event_time: datetime - ) -> tuple[dict[str, Decimal], dict[str, Decimal], Decimal, Decimal]: + ) -> tuple[dict[str, Decimal], Mapping[str, Decimal], Decimal, Decimal]: """Return exact decimal balances needed by the hot pre-trade risk path.""" event_time = ensure_utc_datetime(event_time, field="event_time") cash = { @@ -296,7 +446,7 @@ def risk_balances( spec.settlement_currency, event_time, ) - return cash, self._positions, nav, initial_margin + return cash, MappingProxyType(self._positions), nav, initial_margin def portfolio_risk_snapshot(self, event_time: datetime) -> PortfolioRiskSnapshot: """Build an exact, read-only base-currency exposure view at one PIT timestamp.""" @@ -375,6 +525,7 @@ def portfolio_risk_snapshot(self, event_time: datetime) -> PortfolioRiskSnapshot def mark( self, event: MarkPriceEvent, *, create_snapshot: bool = True ) -> AccountSnapshot | None: + self._require_mutable() if event.instrument_id not in self.instruments: raise ValidationError(f"missing InstrumentSpec for {event.instrument_id}") prior = self._mark_fingerprints.get(event.event_id) @@ -414,6 +565,7 @@ def observe_market( create_snapshot: bool = True, trusted_unique: bool = False, ) -> AccountSnapshot | None: + self._require_mutable() if isinstance(event, MarkPriceEvent): return self.mark(event, create_snapshot=create_snapshot) price: FixedPoint | None = None @@ -521,12 +673,18 @@ def _apply( *, trading_day: date | None, create_snapshot: bool, + local_rollback: bool = True, + trusted_unique: bool = False, ) -> AccountSnapshot | None: + self._require_mutable() self._validate_event(event) reference_id = self._event_identity(event) - prior = self._event_fingerprints.get(reference_id) + prior = None if trusted_unique else self._event_fingerprints.get(reference_id) if prior is not None: - if prior != event: + if isinstance(prior, bytes): + if prior != self._event_fingerprint(event): + raise ValidationError("ledger event id reused with different content") + elif prior != event: raise ValidationError("ledger event id reused with different content") if isinstance(event, Fill) and self._fill_trading_days[event.fill_id] != trading_day: raise ValidationError("fill trading_day changed across idempotent application") @@ -551,8 +709,12 @@ def _apply( undo: dict[str, object] | None = None try: transaction = self._translate(event) - undo = self._capture_apply_undo(event, transaction, reference_id) - self._post(transaction) + if local_rollback: + undo = self._capture_apply_undo(event, transaction, reference_id) + if local_rollback: + self._post(transaction) + else: + self._post(transaction, local_rollback=False) if isinstance(event, Fill): self._fills[event.fill_id] = event self._fill_trading_days[event.fill_id] = trading_day @@ -568,8 +730,17 @@ def _apply( ) elif isinstance(event, CorporateActionEvent) and event.ratio is not None: self._apply_split_state(event) - self._event_fingerprints[reference_id] = event + if not trusted_unique: + self._event_fingerprints[reference_id] = ( + self._event_fingerprint(event) if self._artifact_sink is not None else event + ) self._event_time = transaction.event_time + if isinstance(event, Fee) and self._artifact_sink is not None: + fill = self._fills.pop(event.fill_id, None) + if fill is not None: + spec = self._spec(fill.instrument_id) + if spec.asset_class is not AssetClass.FUTURE: + self._fill_close_allocations.pop(event.fill_id, None) return self.snapshot(transaction.event_time) if create_snapshot else None except Exception: if transaction is not None and undo is not None: @@ -696,6 +867,27 @@ def apply_with_trading_day( create_snapshot=create_snapshot, ) + def _apply_replay_event( + self, + event: LedgerEvent, + *, + trading_day: date | None = None, + ) -> None: + """Apply one validated fact under the engine's whole-replay rollback boundary.""" + + if isinstance(event, Fill): + if not isinstance(trading_day, date) or isinstance(trading_day, datetime): + raise ValidationError("fill replay application requires a trading_day date") + elif trading_day is not None: + raise ValidationError("trading_day is only valid for fill replay application") + self._apply( + event, + trading_day=trading_day, + create_snapshot=False, + local_rollback=False, + trusted_unique=True, + ) + def _validate_event(self, event: LedgerEvent) -> None: if isinstance(event, CorporateActionEvent): return @@ -970,7 +1162,6 @@ def _fill_transaction(self, fill_event: Fill) -> LedgerTransaction: quantity = decimal(fill_event.quantity) signed_quantity = quantity if fill_event.side is Side.BUY else -quantity old_quantity = self._positions.get(fill_event.instrument_id, Decimal(0)) - average = self._average_cost(fill_event.instrument_id) multiplier = decimal(spec.contract_multiplier) price = decimal(fill_event.price) notional = quantity * price * multiplier @@ -979,14 +1170,21 @@ def _fill_transaction(self, fill_event: Fill) -> LedgerTransaction: if old_quantity and old_quantity * signed_quantity < 0 else Decimal(0) ) - if old_quantity > 0: - realized = (price - average) * close_quantity * multiplier - elif old_quantity < 0: - realized = (average - price) * close_quantity * multiplier - else: - realized = Decimal(0) + derivative = fill_event.instrument_id in self._derivative_instruments + average = ( + self._average_cost(fill_event.instrument_id) + if close_quantity or (not derivative and fill_event.side is Side.SELL) + else Decimal(0) + ) + realized = Decimal(0) + if close_quantity: + realized = ( + (price - average) * close_quantity * multiplier + if old_quantity > 0 + else (average - price) * close_quantity * multiplier + ) postings: list[Posting] = [] - if self._is_derivative(spec): + if derivative: if realized: postings.extend( [ @@ -1246,12 +1444,19 @@ def _posting( prior = self._posting_cache.get(key) if prior is not None: return prior - posting = Posting( - ledger_account=account, - currency=currency, - amount=fixed(amount, self.money_scale, rounding=ROUND_HALF_EVEN), - instrument_id=instrument_id, - quantity_delta=( + posting = object.__new__(Posting) + object.__setattr__(posting, "ledger_account", account) + object.__setattr__(posting, "currency", currency) + object.__setattr__( + posting, + "amount", + fixed(amount, self.money_scale, rounding=ROUND_HALF_EVEN), + ) + object.__setattr__(posting, "instrument_id", instrument_id) + object.__setattr__( + posting, + "quantity_delta", + ( fixed(quantity_delta, quantity_scale, rounding=ROUND_HALF_EVEN) if quantity_delta is not None else None @@ -1269,18 +1474,46 @@ def _make_transaction( event_time: datetime, postings: tuple[Posting, ...], ) -> LedgerTransaction: - return LedgerTransaction( - transaction_id=_identifier("tx", self.account_id, event_type.value, reference_id), - idempotency_key=idempotency_key, - event_time=event_time, - event_type=event_type, - reference_id=reference_id, - postings=postings, + transaction = object.__new__(LedgerTransaction) + object.__setattr__( + transaction, + "transaction_id", + _identifier("tx", self.account_id, event_type.value, reference_id), ) - - def _post(self, transaction: LedgerTransaction) -> None: - if transaction.idempotency_key in self._transaction_keys: + object.__setattr__(transaction, "idempotency_key", idempotency_key) + object.__setattr__(transaction, "event_time", event_time) + object.__setattr__(transaction, "event_type", event_type) + object.__setattr__(transaction, "reference_id", reference_id) + object.__setattr__(transaction, "postings", postings) + return transaction + + def _post(self, transaction: LedgerTransaction, *, local_rollback: bool = True) -> None: + streaming = self._artifact_sink is not None + if not streaming and transaction.idempotency_key in self._transaction_keys: raise ValidationError("duplicate ledger transaction idempotency key") + if not local_rollback: + for posting in transaction.postings: + key = (posting.ledger_account, posting.currency, posting.instrument_id) + self._accounts[key] = self._accounts.get(key, Decimal(0)) + decimal(posting.amount) + if ( + posting.ledger_account == "assets:position" + and posting.instrument_id is not None + and posting.quantity_delta is not None + ): + instrument_id = posting.instrument_id + self._positions[instrument_id] = self._positions.get( + instrument_id, Decimal(0) + ) + decimal(posting.quantity_delta) + if self._artifact_sink is None: + self._transactions.append(transaction) + else: + self._artifact_sink.append( + "ledger_transactions", ledger_transaction_bytes(transaction) + ) + self._transaction_count += 1 + if not streaming: + self._transaction_keys.add(transaction.idempotency_key) + return missing = object() prior_accounts: dict[tuple[str, str, str | None], Decimal | object] = {} prior_positions: dict[str, Decimal | object] = {} @@ -1301,8 +1534,15 @@ def _post(self, transaction: LedgerTransaction) -> None: self._positions[instrument_id] = self._positions.get( instrument_id, Decimal(0) ) + decimal(posting.quantity_delta) - self._transactions.append(transaction) - self._transaction_keys.add(transaction.idempotency_key) + if self._artifact_sink is None: + self._transactions.append(transaction) + else: + self._artifact_sink.append( + "ledger_transactions", ledger_transaction_bytes(transaction) + ) + self._transaction_count += 1 + if not streaming: + self._transaction_keys.add(transaction.idempotency_key) except Exception: for key, value in prior_accounts.items(): if value is missing: @@ -1332,6 +1572,20 @@ def _event_identity(self, event: LedgerEvent) -> str: raise ValidationError("ledger event has no stable identity") return f"{identity_field}:{getattr(event, identity_field)}" + @staticmethod + def _event_fingerprint(event: LedgerEvent) -> bytes: + if isinstance(event, Fill): + payload = fill_bytes(event) + elif isinstance(event, Fee): + payload = fee_bytes(event) + elif isinstance(event, Settlement): + payload = settlement_bytes(event) + elif isinstance(event, CorporateActionEvent): + payload = _canonical(market_event_payload(event)) + else: + payload = _canonical(execution_payload(event)) + return hashlib.sha256(payload).digest() + def _spec(self, instrument_id: str) -> InstrumentSpec: try: return self.instruments[instrument_id] diff --git a/src/quant_execution/matching.py b/src/quant_execution/matching.py index ca3dd37..3af7b34 100644 --- a/src/quant_execution/matching.py +++ b/src/quant_execution/matching.py @@ -3,7 +3,6 @@ from __future__ import annotations import hashlib -import json from collections.abc import Mapping, Sequence from copy import copy, deepcopy from datetime import datetime, timedelta @@ -25,19 +24,19 @@ from quant_data_kit.exceptions import ValidationError from quant_execution._fixed import decimal, fixed +from quant_execution._json import flat_sequence_bytes from quant_execution.broker import remaining_quantity from quant_execution.contracts import Fill, LiquidityRole, Order, OrderType, Side, TimeInForce def _fill_id(model: str, event_id: str, order_id: str, index: int, price: FixedPoint) -> str: - raw = json.dumps( - [model, event_id, order_id, index, price.units, price.scale], - separators=(",", ":"), - ).encode() + raw = flat_sequence_bytes((model, event_id, order_id, index, price.units, price.scale)) return f"fill-{hashlib.sha256(raw).hexdigest()[:24]}" -def _sort_orders(orders: Sequence[Order]) -> list[Order]: +def _sort_orders(orders: Sequence[Order]) -> Sequence[Order]: + if len(orders) < 2: + return orders return sorted(orders, key=lambda item: (item.intent.created_at, item.order_id)) @@ -58,7 +57,14 @@ def priority(order: Order) -> tuple[object, ...]: def _quantity_from_available(order: Order, available: Decimal) -> FixedPoint: - remaining = decimal(remaining_quantity(order)) + remaining_units = order.intent.quantity.units - order.filled_quantity.units + if remaining_units == order.intent.quantity.units: + remaining_fp = order.intent.quantity + else: + remaining_fp = FixedPoint(remaining_units, order.intent.quantity.scale) + remaining = decimal(remaining_fp) + if available >= remaining: + return remaining_fp amount = min(remaining, available) if amount <= 0: return FixedPoint(0, order.intent.quantity.scale) @@ -122,19 +128,27 @@ def _fill( role: LiquidityRole, index: int, ) -> Fill: - return Fill( - fill_id=_fill_id(model, event.event_id, order.order_id, index, price), - order_id=order.order_id, - account_id=order.intent.account_id, - strategy_id=order.intent.strategy_id, - instrument_id=order.intent.instrument_id, - side=order.intent.side, - quantity=quantity, - price=price, - event_time=event.available_at, - liquidity_role=role, - venue_trade_id=getattr(event, "event_id", None), + if not isinstance(quantity, FixedPoint) or quantity.units <= 0: + raise ValidationError("fill quantity must be a positive FixedPoint") + if not isinstance(price, FixedPoint) or price.units <= 0: + raise ValidationError("fill price must be a positive FixedPoint") + if not isinstance(role, LiquidityRole): + raise ValidationError("fill liquidity_role must be a LiquidityRole") + fill = object.__new__(Fill) + object.__setattr__( + fill, "fill_id", _fill_id(model, event.event_id, order.order_id, index, price) ) + object.__setattr__(fill, "order_id", order.order_id) + object.__setattr__(fill, "account_id", order.intent.account_id) + object.__setattr__(fill, "strategy_id", order.intent.strategy_id) + object.__setattr__(fill, "instrument_id", order.intent.instrument_id) + object.__setattr__(fill, "side", order.intent.side) + object.__setattr__(fill, "quantity", quantity) + object.__setattr__(fill, "price", price) + object.__setattr__(fill, "event_time", event.available_at) + object.__setattr__(fill, "liquidity_role", role) + object.__setattr__(fill, "venue_trade_id", event.event_id) + return fill class BarMatchingModel(_BaseMatchingModel): @@ -165,9 +179,15 @@ def __init__( def reset(self) -> None: self._activated_stop_limits.clear() + @staticmethod + def checkpoint_required(open_orders: Sequence[Order]) -> bool: + return any(order.intent.order_type is OrderType.STOP_LIMIT for order in open_orders) + def match(self, market_event: MarketEvent, open_orders: Sequence[Order]) -> Sequence[Fill]: if not isinstance(market_event, BarEvent) or not market_event.is_complete: return () + if not open_orders: + return () capacity = decimal(market_event.volume) * self.participation_rate fills: list[Fill] = [] for order in _sort_orders(open_orders): diff --git a/src/quant_execution/rules.py b/src/quant_execution/rules.py index e7c1a93..ea51316 100644 --- a/src/quant_execution/rules.py +++ b/src/quant_execution/rules.py @@ -20,6 +20,7 @@ QuoteEvent, StatusEvent, TradeEvent, + ensure_utc_datetime, ) from quant_data_kit.exceptions import ValidationError @@ -310,6 +311,8 @@ def __init__( self.ledger = ledger self.money_scale = money_scale self.policies = tuple(policies) + self._rules: dict[str, _AssetRule] = {} + self._fee_rates: dict[tuple[str, Side, LiquidityRole], Decimal] = {} for policy in self.policies: if not callable(getattr(policy, "check_order", None)) or not callable( getattr(policy, "runtime_check", None) @@ -371,6 +374,23 @@ def check_current(self, order_intent: OrderIntent, *, event_time: datetime) -> R as_of=order_intent.created_at, ) + def _check_current_for_submit( + self, order_intent: OrderIntent, *, event_time: datetime + ) -> tuple[ + RiskDecision, + tuple[tuple[str, Decimal] | None, Decimal, tuple[str, Decimal] | None] | None, + ]: + """Return a checked reservation for the built-in engine submit path.""" + + prepared: list[tuple[tuple[str, Decimal] | None, Decimal, tuple[str, Decimal] | None]] = [] + decision = self._check( + order_intent, + self._current_view(event_time, order_intent.instrument_id), + as_of=order_intent.created_at, + prepared_reservation=prepared, + ) + return decision, prepared[0] if prepared else None + def check_open_order( self, order: Order, @@ -428,6 +448,10 @@ def _check( account_snapshot: AccountSnapshot | _RiskAccountView, *, as_of: datetime, + prepared_reservation: list[ + tuple[tuple[str, Decimal] | None, Decimal, tuple[str, Decimal] | None] + ] + | None = None, ) -> RiskDecision: if order_intent.account_id != account_snapshot.account_id: return RiskDecision(False, "ACCOUNT_MISMATCH", "intent targets another account") @@ -456,22 +480,43 @@ def _check( if value is not None and not aligned(value, spec.price_tick): return RiskDecision(False, "PRICE_TICK", f"{field_name} violates price tick") try: - decision = self._rule(spec).check( + decision = self._rule_for(spec).check( order_intent, account_snapshot, state, spec, self.ledger ) if not decision.accepted: return decision - decision = self._check_reservations(order_intent, account_snapshot, state, spec) + reservation = self._reservation_requirement(order_intent, state, spec) + decision = self._check_reservations( + order_intent, + account_snapshot, + state, + spec, + requirement=reservation, + ) if not decision.accepted: return decision except ValidationError as exc: return RiskDecision(False, "RULE_CONFIGURATION", str(exc)) - return self._check_order_policies(order_intent, event_time=as_of) + decision = self._check_order_policies(order_intent, event_time=as_of) + if decision.accepted and prepared_reservation is not None: + prepared_reservation.append(reservation) + return decision def reserve(self, intent: OrderIntent) -> None: spec = self.instruments[intent.instrument_id] state = self._states[intent.instrument_id] - cash, margin, position = self._reservation_requirement(intent, state, spec) + self._reserve_requirement(intent, self._reservation_requirement(intent, state, spec)) + + def _reserve_requirement( + self, + intent: OrderIntent, + requirement: tuple[ + tuple[str, Decimal] | None, + Decimal, + tuple[str, Decimal] | None, + ], + ) -> None: + cash, margin, position = requirement if cash is not None: prior = self._cash_reservations.get(intent.idempotency_key) expected = (cash[0], cash[1], cash[1]) @@ -531,6 +576,13 @@ def runtime_check(self, snapshot: AccountSnapshot) -> RiskDecision: return self._check_runtime_policies(event_time=snapshot.event_time) def runtime_check_current(self, event_time: datetime) -> RiskDecision: + event_time = ensure_utc_datetime(event_time, field="event_time") + if ( + not self.policies + and type(self.ledger) is ExactAccountLedger + and not self.ledger.has_open_derivative_position + ): + return _ACCEPTED_DECISION if self.ledger.liquidation_required(event_time): return RiskDecision( False, @@ -665,7 +717,7 @@ def check_fill( if fill.side is Side.SELL: return _ACCEPTED_DECISION state = self._states[fill.instrument_id] - rate = self._rule(spec).fee_rate(fill, order, state, spec, self.ledger) + rate = self._fee_rate_for(fill, order, state, spec) required = ( decimal(fill.quantity) * decimal(fill.price) @@ -690,7 +742,7 @@ def check_fill( def fee_for(self, fill: Fill, order: Order) -> Fee | None: spec = self.instruments[fill.instrument_id] state = self._states[fill.instrument_id] - rate = self._rule(spec).fee_rate(fill, order, state, spec, self.ledger) + rate = self._fee_rate_for(fill, order, state, spec) fee_type = "maker" if fill.liquidity_role is LiquidityRole.MAKER else "taker" unit_notional = decimal(fill.price) * decimal(spec.contract_multiplier) if spec.asset_class is AssetClass.FUTURE: @@ -710,20 +762,22 @@ def fee_for(self, fill: Fill, order: Order) -> Fee | None: amount = decimal(fill.quantity) * unit_notional * rate if amount == 0: return None - return Fee( - fee_id=( - "fee-" - + hashlib.sha256( - f"{fill.fill_id}|{amount}|{fee_type}|{spec.settlement_currency}".encode() - ).hexdigest()[:24] - ), - fill_id=fill.fill_id, - account_id=fill.account_id, - amount=fixed(amount, self.money_scale), - currency=spec.settlement_currency, - event_time=fill.event_time, - fee_type=fee_type, + fee = object.__new__(Fee) + object.__setattr__( + fee, + "fee_id", + "fee-" + + hashlib.sha256( + f"{fill.fill_id}|{amount}|{fee_type}|{spec.settlement_currency}".encode() + ).hexdigest()[:24], ) + object.__setattr__(fee, "fill_id", fill.fill_id) + object.__setattr__(fee, "account_id", fill.account_id) + object.__setattr__(fee, "amount", fixed(amount, self.money_scale)) + object.__setattr__(fee, "currency", spec.settlement_currency) + object.__setattr__(fee, "event_time", fill.event_time) + object.__setattr__(fee, "fee_type", fee_type) + return fee def _check_reservations( self, @@ -731,8 +785,15 @@ def _check_reservations( snapshot: AccountSnapshot, state: MarketState, spec: InstrumentSpec, + *, + requirement: tuple[ + tuple[str, Decimal] | None, + Decimal, + tuple[str, Decimal] | None, + ] + | None = None, ) -> RiskDecision: - cash, margin, position = self._reservation_requirement(intent, state, spec) + cash, margin, position = requirement or self._reservation_requirement(intent, state, spec) if cash is not None: currency, required = cash reserved = sum( @@ -864,6 +925,27 @@ def _rule(spec: InstrumentSpec) -> _AssetRule: f"unsupported asset rule for {spec.asset_class.value}/{spec.product_type}" ) + def _rule_for(self, spec: InstrumentSpec) -> _AssetRule: + rule = self._rules.get(spec.instrument_id) + if rule is None: + rule = self._rule(spec) + self._rules[spec.instrument_id] = rule + return rule + + def _fee_rate_for( + self, + fill: Fill, + order: Order, + state: MarketState, + spec: InstrumentSpec, + ) -> Decimal: + key = (spec.instrument_id, fill.side, fill.liquidity_role) + rate = self._fee_rates.get(key) + if rate is None: + rate = self._rule_for(spec).fee_rate(fill, order, state, spec, self.ledger) + self._fee_rates[key] = rate + return rate + def _intent_price(intent: OrderIntent, state: MarketState) -> Decimal | None: if intent.limit_price is not None: diff --git a/src/quant_execution/state_machine.py b/src/quant_execution/state_machine.py index e87cc5a..55924f4 100644 --- a/src/quant_execution/state_machine.py +++ b/src/quant_execution/state_machine.py @@ -2,7 +2,6 @@ from __future__ import annotations -from dataclasses import replace from datetime import datetime from quant_data_kit import FixedPoint, ensure_utc_datetime @@ -45,6 +44,8 @@ def transition_order( raise ValidationError(f"{to_status.value} transition requires a reason") elif not isinstance(reason, str): raise ValidationError("reason must be a string") + if not isinstance(event_id, str) or not event_id.strip(): + raise ValidationError("event_id is required") filled = order.filled_quantity if fill_quantity is not None: @@ -64,20 +65,19 @@ def transition_order( raise ValidationError("filled transition must complete the order quantity") next_version = order.version + 1 - updated = replace( - order, - status=to_status, - filled_quantity=filled, - version=next_version, - ) - event = OrderEvent( - event_id=event_id, - order_id=order.order_id, - event_time=event_time, - sequence=next_version, - from_status=order.status, - to_status=to_status, - fill_quantity=fill_quantity, - reason=reason, - ) + updated = object.__new__(Order) + object.__setattr__(updated, "order_id", order.order_id) + object.__setattr__(updated, "intent", order.intent) + object.__setattr__(updated, "status", to_status) + object.__setattr__(updated, "filled_quantity", filled) + object.__setattr__(updated, "version", next_version) + event = object.__new__(OrderEvent) + object.__setattr__(event, "event_id", event_id.strip()) + object.__setattr__(event, "order_id", order.order_id) + object.__setattr__(event, "event_time", event_time) + object.__setattr__(event, "sequence", next_version) + object.__setattr__(event, "from_status", order.status) + object.__setattr__(event, "to_status", to_status) + object.__setattr__(event, "fill_quantity", fill_quantity) + object.__setattr__(event, "reason", reason) return updated, event diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py new file mode 100644 index 0000000..ff61433 --- /dev/null +++ b/tests/test_artifacts.py @@ -0,0 +1,942 @@ +from __future__ import annotations + +import json +import queue +from datetime import date, timedelta + +import pytest +from conftest import T0, event_fields, fp +from quant_data_kit import CorporateActionEvent, FundingRateEvent, MarkPriceEvent, StatusEvent +from quant_data_kit.exceptions import ValidationError +from test_engine import ( + FixtureStrategy, + Signal, + bar, + engine_for, + scenario_a_share, + scenario_crypto, + scenario_future, +) +from test_engine_edges import ( + LiquidatingGate, + TransactionalBatchMatcher, + spot_signal, + transactional_engine, +) +from test_rules import FUTURE, PERP, SPOT, STOCK, specs + +from quant_execution import ( + ArrowReplayArtifactSink, + Fill, + LiquidityRole, + OrderIntent, + OrderType, + ReplayError, + RiskDecision, + Side, + TimeInForce, + load_stored_artifacts, +) +from quant_execution.artifacts import ( + _canonical_manifest_bytes, + _manifest_hash, + fee_bytes, + fill_bytes, + ledger_transaction_bytes, + order_bytes, + order_event_bytes, + settlement_bytes, +) +from quant_execution.broker import DeterministicBroker +from quant_execution.engine import _event_sort_key +from quant_execution.ledger import ExactAccountLedger +from quant_execution.matching import BarMatchingModel +from quant_execution.rules import RuleBookRiskGate + + +@pytest.mark.parametrize( + "factory", + (scenario_a_share, scenario_future, scenario_crypto), + ids=("a-share", "future", "crypto-spot-perpetual-funding"), +) +def test_streamed_replay_is_byte_identical_to_memory_reference(factory, tmp_path) -> None: + reference, reference_events = factory() + expected = reference.replay(reference_events, 42) + assert reference.artifacts is not None + expected_artifacts = reference.artifacts + expected_nav = reference.ledger.snapshot().nav + + streamed, streamed_events = factory() + sink = ArrowReplayArtifactSink(tmp_path / "run", batch_size=2, queue_batches=1) + actual = streamed.replay_to_sink( + sorted(streamed_events, key=_event_sort_key), + 42, + sink, + ) + stored = streamed.stored_artifacts + assert stored is not None + assert actual == expected + assert streamed.ledger.snapshot().nav == expected_nav + assert streamed.ledger.transaction_count == stored.counts["ledger_transactions"] + assert streamed.ledger.journal_sha256 == actual.ledger_sha256 + assert tuple(stored.iter_payload_bytes("orders")) == tuple( + order_bytes(value) for value in expected_artifacts.orders + ) + assert tuple(stored.iter_payload_bytes("order_events")) == tuple( + order_event_bytes(value) for value in expected_artifacts.order_events + ) + assert tuple(stored.iter_payload_bytes("fills")) == tuple( + fill_bytes(value) for value in expected_artifacts.fills + ) + assert tuple(stored.iter_payload_bytes("fees")) == tuple( + fee_bytes(value) for value in expected_artifacts.fees + ) + assert tuple(stored.iter_payload_bytes("settlements")) == tuple( + settlement_bytes(value) for value in expected_artifacts.settlements + ) + assert tuple(stored.iter_payload_bytes("ledger_transactions")) == tuple( + ledger_transaction_bytes(value) for value in expected_artifacts.ledger_transactions + ) + assert tuple(stored.iter_json("risk_events")) == expected_artifacts.risk_events + verified = load_stored_artifacts(stored.root) + assert verified.manifest_sha256 == stored.manifest_sha256 + manifest = json.loads(stored.manifest_path.read_text(encoding="utf-8")) + assert manifest["complete"] is True + assert manifest["run_metadata"]["result_sha256"] == expected.result_sha256 + + +def test_streaming_broker_terminal_order_remains_idempotently_readable(tmp_path) -> None: + broker = DeterministicBroker() + sink = ArrowReplayArtifactSink(tmp_path / "broker", batch_size=1) + broker.start_artifact_stream(sink) + intent = OrderIntent( + idempotency_key="cancel-me", + account_id="account", + strategy_id="strategy", + instrument_id="crypto:test:BTCUSDT", + side=Side.BUY, + quantity=fp("1.000", 3), + order_type=OrderType.LIMIT, + time_in_force=TimeInForce.GTC, + created_at=T0, + limit_price=fp("100"), + ) + accepted = broker.submit(intent) + cancelled = broker.cancel( + accepted.order_id, + idempotency_key="cancel-request", + created_at=T0 + timedelta(seconds=1), + ) + assert broker.submit(intent).order_id == accepted.order_id + assert ( + broker.cancel( + accepted.order_id, + idempotency_key="cancel-request", + created_at=T0 + timedelta(seconds=1), + ) + == cancelled + ) + assert broker.get_order(accepted.order_id).status.value == "cancelled" + broker.finish_artifact_stream() + stored = sink.close({"run_id": "broker-only"}) + assert stored.counts["orders"] == 1 + assert stored.counts["order_events"] == 2 + + +def test_streaming_broker_fill_compaction_and_lifecycle_guards(tmp_path) -> None: + broker = DeterministicBroker() + sink = ArrowReplayArtifactSink(tmp_path / "broker-fill", batch_size=1) + with pytest.raises(ValidationError, match="provide append"): + broker.start_artifact_stream(object()) + broker.start_artifact_stream(sink) + with pytest.raises(ValidationError, match="immediately after reset"): + broker.start_artifact_stream(sink) + intent = OrderIntent( + idempotency_key="fill-me", + account_id="account", + strategy_id="strategy", + instrument_id="crypto:test:BTCUSDT", + side=Side.BUY, + quantity=fp("1.000", 3), + order_type=OrderType.LIMIT, + time_in_force=TimeInForce.GTC, + created_at=T0, + limit_price=fp("100"), + ) + order = broker.submit(intent) + fill = Fill( + fill_id="fill-compact", + order_id=order.order_id, + account_id="account", + strategy_id="strategy", + instrument_id=intent.instrument_id, + side=Side.BUY, + quantity=intent.quantity, + price=fp("100"), + event_time=T0 + timedelta(seconds=1), + liquidity_role=LiquidityRole.TAKER, + ) + event = broker.apply_fill(fill) + assert broker.apply_fill(fill) == event + conflicting = Fill( + fill_id=fill.fill_id, + order_id=fill.order_id, + account_id=fill.account_id, + strategy_id=fill.strategy_id, + instrument_id=fill.instrument_id, + side=fill.side, + quantity=fill.quantity, + price=fp("101"), + event_time=fill.event_time, + liquidity_role=fill.liquidity_role, + ) + with pytest.raises(ValidationError, match="different fill content"): + broker.apply_fill(conflicting) + terminal_payload = broker._orders[order.order_id] + assert isinstance(terminal_payload, bytes) + broker._open_order_ids.add(order.order_id) + with pytest.raises(TypeError, match="terminal order"): + broker._live_order(order.order_id) + broker._open_order_ids.clear() + mutable_checkpoint = broker.capture_state() + broker.finish_artifact_stream() + with pytest.raises(ValidationError, match="stream is closed"): + broker.submit(intent) + with pytest.raises(ValidationError, match="stream is closed"): + broker.apply_fill(fill) + with pytest.raises(ValidationError, match="stream is closed"): + broker.note_trading_day(order.order_id, T0.date()) + with pytest.raises(ValidationError, match="stream is closed"): + broker.expire_day_orders(T0.date(), T0 + timedelta(days=1)) + with pytest.raises(ValidationError, match="stream is closed"): + broker.start_artifact_stream(sink) + with pytest.raises(ValidationError, match="stream is closed"): + broker.restore_state(mutable_checkpoint) + broker.finish_artifact_stream() + sink.close({"run_id": "broker-fill"}) + + +def test_streaming_replay_fails_closed_for_unsorted_and_duplicate_inputs(tmp_path) -> None: + engine, events = scenario_a_share() + with pytest.raises(ReplayError, match="not deterministically sorted"): + engine.replay_to_sink(events, 42, ArrowReplayArtifactSink(tmp_path / "unsorted")) + assert json.loads((tmp_path / "unsorted" / "FAILED.json").read_text())["complete"] is False + + engine, events = scenario_a_share() + ordered = sorted(events, key=_event_sort_key) + with pytest.raises(ReplayError, match="duplicate MarketEvent event_id"): + engine.replay_to_sink( + (ordered[0], ordered[0]), + 42, + ArrowReplayArtifactSink(tmp_path / "duplicate"), + ) + + +def test_sink_validation_transaction_and_unknown_stream_branches(tmp_path) -> None: + with pytest.raises(ValidationError, match="batch_size"): + ArrowReplayArtifactSink(tmp_path / "bad-batch", batch_size=0) + with pytest.raises(ValidationError, match="queue_batches"): + ArrowReplayArtifactSink(tmp_path / "bad-queue", queue_batches=0) + + sink = ArrowReplayArtifactSink(tmp_path / "transactions", batch_size=1) + with pytest.raises(ValidationError, match="unknown artifact stream"): + sink.append("unknown", b"{}") + with pytest.raises(ValidationError, match="canonical bytes"): + sink.append("fills", "not-bytes") + sink.begin() + sink.append("risk_events", b'"rolled-back"') + with pytest.raises(RuntimeError, match="nested"): + sink.begin() + sink.rollback() + with pytest.raises(RuntimeError, match="no artifact transaction"): + sink.rollback() + sink.begin() + sink.append("risk_events", b'"committed"') + sink.commit() + stored = sink.close({"run_id": "transaction-test"}) + assert tuple(stored.iter_json("risk_events")) == ("committed",) + assert tuple(stored.iter_json("settlements")) == () + with pytest.raises(ValidationError, match="unknown artifact stream"): + tuple(stored.iter_json("unknown")) + with pytest.raises(RuntimeError, match="already closed"): + sink.close({}) + + +def test_streaming_input_validation_empty_and_sink_failure_branches(tmp_path, monkeypatch) -> None: + for index, seed in enumerate((True, "1", -1)): + engine, events = scenario_a_share() + sink = ArrowReplayArtifactSink(tmp_path / f"seed-{index}") + with pytest.raises(ValidationError, match="seed"): + engine.replay_to_sink(events, seed, sink) + sink.abort() + engine, events = scenario_a_share() + with pytest.raises(ValidationError, match="ArrowReplayArtifactSink"): + engine.replay_to_sink(events, 1, object()) + + engine, _ = scenario_a_share() + result = engine.replay_to_sink((), 1, ArrowReplayArtifactSink(tmp_path / "empty")) + assert result.event_count == result.order_count == result.fill_count == 0 + + engine, _ = scenario_a_share() + with pytest.raises(ReplayError, match="non-MarketEvent"): + engine.replay_to_sink((object(),), 1, ArrowReplayArtifactSink(tmp_path / "bad-first")) + engine, events = scenario_a_share() + ordered = sorted(events, key=_event_sort_key) + with pytest.raises(ReplayError, match="non-MarketEvent"): + engine.replay_to_sink( + (ordered[0], object()), + 1, + ArrowReplayArtifactSink(tmp_path / "bad-later"), + ) + + failing = ArrowReplayArtifactSink(tmp_path / "writer-failure", batch_size=1) + + def fail_writer(stream): + del stream + raise OSError("injected Arrow writer failure") + + monkeypatch.setattr(failing, "_writer", fail_writer) + failing.append("fills", b"{}") + with pytest.raises(RuntimeError, match="artifact writer failed"): + failing.seal() + failing.abort() + + +def test_failed_second_streaming_replay_restores_completed_artifact_handle_and_ledger( + tmp_path, +) -> None: + engine, events = scenario_a_share() + ordered = sorted(events, key=_event_sort_key) + first = engine.replay_to_sink(ordered, 42, ArrowReplayArtifactSink(tmp_path / "completed")) + completed = engine.stored_artifacts + assert completed is not None + completed_count = engine.ledger.transaction_count + completed_hash = engine.ledger.journal_sha256 + + with pytest.raises(ReplayError, match="non-MarketEvent"): + engine.replay_to_sink( + (ordered[0], object()), + 42, + ArrowReplayArtifactSink(tmp_path / "failed-second"), + ) + + assert engine.stored_artifacts == completed + assert engine.ledger.transaction_count == completed_count + assert engine.ledger.journal_sha256 == completed_hash == first.ledger_sha256 + assert (tmp_path / "failed-second" / "FAILED.json").is_file() + + +def test_sink_defensive_state_and_queue_full_branches(tmp_path, monkeypatch) -> None: + from quant_execution.artifacts import _SequenceDigest + + digest = _SequenceDigest() + digest.append(b"{}") + digest.close() + with pytest.raises(RuntimeError, match="already closed"): + digest.append(b"{}") + + sink = ArrowReplayArtifactSink(tmp_path / "states") + with pytest.raises(RuntimeError, match="no artifact transaction"): + sink.commit() + sink.begin() + with pytest.raises(RuntimeError, match="active artifact transaction"): + sink.seal() + sink.rollback() + with pytest.raises(ValidationError, match="unknown artifact stream"): + sink.logical_sha256("unknown") + assert tuple(sink._iter_payload_bytes("fills")) == () + + original_put = sink._queue.put + calls = 0 + + def full_once(item, timeout=None): + nonlocal calls + calls += 1 + if calls == 1: + raise queue.Full + return original_put(item, timeout=timeout) + + monkeypatch.setattr(sink._queue, "put", full_once) + sink.append("fills", b"{}") + stored = sink.close({"run_id": "queue-full-once"}) + assert stored.counts["fills"] == 1 + with pytest.raises(RuntimeError, match="no longer accepts"): + sink.append("fills", b"{}") + with pytest.raises(RuntimeError, match="no longer accepts"): + sink.begin() + sink.abort() + + invalid_manifest = ArrowReplayArtifactSink(tmp_path / "invalid-manifest") + with pytest.raises(TypeError): + invalid_manifest.close({"not_json": object()}) + assert (tmp_path / "invalid-manifest" / "FAILED.json").exists() + + +def test_stored_artifact_loader_rejects_physical_and_manifest_tampering(tmp_path) -> None: + sink = ArrowReplayArtifactSink(tmp_path / "physical", batch_size=1) + sink.append("fills", b"{}") + stored = sink.close({"run_id": "physical"}) + assert load_stored_artifacts(stored.root).counts["fills"] == 1 + fill_path = stored.root / "fills.arrow" + fill_path.write_bytes(fill_path.read_bytes() + b"tampered") + with pytest.raises(ValidationError, match="missing or changed size"): + load_stored_artifacts(stored.root) + + sink = ArrowReplayArtifactSink(tmp_path / "manifest", batch_size=1) + sink.append("fills", b"{}") + stored = sink.close({"run_id": "manifest"}) + payload = json.loads(stored.manifest_path.read_text(encoding="utf-8")) + stored.manifest_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + with pytest.raises(ValidationError, match="not canonical"): + load_stored_artifacts(stored.root) + + +def test_stored_artifact_loader_rejects_manifest_contract_mutations(tmp_path) -> None: + def artifact(case: str, *, with_fill: bool = False): + sink = ArrowReplayArtifactSink(tmp_path / case, batch_size=1) + if with_fill: + sink.append("fills", b"{}") + return sink.close({"run_id": case}) + + def rewrite(stored, payload, *, rehash: bool = True) -> None: + if rehash: + payload["manifest_sha256"] = _manifest_hash(payload) + stored.manifest_path.write_bytes(_canonical_manifest_bytes(payload)) + + unreadable = artifact("unreadable") + unreadable.manifest_path.write_bytes(b"{") + with pytest.raises(ValidationError, match="manifest is unreadable"): + load_stored_artifacts(unreadable.root) + + non_object = artifact("non-object") + non_object.manifest_path.write_bytes(b"[]\n") + with pytest.raises(ValidationError, match="root must be an object"): + load_stored_artifacts(non_object.root) + + top_level_cases = ( + ("fields", "unexpected", True, "fields changed"), + ("metadata", "run_metadata", {}, "no run metadata"), + ("schema", "schema_version", "2.0.0", "schema version"), + ("format", "artifact_format", "unknown", "format is unsupported"), + ("incomplete", "complete", False, "run is not complete"), + ("counts-shape", "counts", {}, "counts changed shape"), + ("logical-shape", "logical_sha256", {}, "logical hashes changed shape"), + ("files-type", "files", [], "files must be an object"), + ) + for case, field, value, message in top_level_cases: + stored = artifact(case) + payload = json.loads(stored.manifest_path.read_text(encoding="utf-8")) + payload[field] = value + rewrite(stored, payload) + with pytest.raises(ValidationError, match=message): + load_stored_artifacts(stored.root) + + invalid_hash = artifact("invalid-hash") + payload = json.loads(invalid_hash.manifest_path.read_text(encoding="utf-8")) + payload["manifest_sha256"] = "not-a-sha" + rewrite(invalid_hash, payload, rehash=False) + with pytest.raises(ValidationError, match="manifest hash mismatch"): + load_stored_artifacts(invalid_hash.root) + + empty_stream_cases = ( + ("count-bool", "counts", True, "count is invalid"), + ("count-negative", "counts", -1, "count is invalid"), + ("logical-format", "logical_sha256", "bad", "logical hash is invalid"), + ("unexpected-file", "files", {"path": "fees.arrow"}, "unexpectedly has a file"), + ("empty-hash", "logical_sha256", "0" * 64, "empty artifact logical hash mismatch"), + ) + for case, section, value, message in empty_stream_cases: + stored = artifact(case) + payload = json.loads(stored.manifest_path.read_text(encoding="utf-8")) + payload[section]["fees"] = value + rewrite(stored, payload) + with pytest.raises(ValidationError, match=message): + load_stored_artifacts(stored.root) + + file_cases = ( + ("file-shape", "files", {"path": "fills.arrow"}, "metadata changed shape"), + ( + "file-path", + "files", + {"bytes": 1, "path": "wrong.arrow", "sha256": "0" * 64}, + "file path is invalid", + ), + ("file-bytes", "bytes", True, "file metadata is invalid"), + ("file-sha", "sha256", "bad", "file metadata is invalid"), + ) + for case, field, value, message in file_cases: + stored = artifact(case, with_fill=True) + payload = json.loads(stored.manifest_path.read_text(encoding="utf-8")) + if field == "files": + payload["files"]["fills"] = value + else: + payload["files"]["fills"][field] = value + rewrite(stored, payload) + with pytest.raises(ValidationError, match=message): + load_stored_artifacts(stored.root) + + unknown_stream = artifact("unknown-stream") + payload = json.loads(unknown_stream.manifest_path.read_text(encoding="utf-8")) + payload["files"]["unknown"] = {"bytes": 1, "path": "unknown.arrow", "sha256": "0" * 64} + rewrite(unknown_stream, payload) + with pytest.raises(ValidationError, match="unknown streams"): + load_stored_artifacts(unknown_stream.root) + + +def test_artifact_manifest_publish_is_no_clobber(tmp_path) -> None: + sink = ArrowReplayArtifactSink(tmp_path / "no-clobber", batch_size=1) + sink.append("fills", b"{}") + manifest_path = sink.root / "manifest.json" + manifest_path.write_text("pre-existing", encoding="utf-8") + with pytest.raises(FileExistsError): + sink.close({"run_id": "no-clobber"}) + assert manifest_path.read_text(encoding="utf-8") == "pre-existing" + assert (sink.root / "FAILED.json").is_file() + + +def test_streaming_day_expiry_rejection_latency_and_liquidation_paths(tmp_path) -> None: + registry = {STOCK: specs()[STOCK]} + + day_strategy = FixtureStrategy( + {"day-signal": [Signal(STOCK, Side.BUY, fp("100"), fp("8.5"), tif=TimeInForce.DAY)]} + ) + day_engine = engine_for( + run_id="stream-day-expiry", + registry=registry, + initial_cash={"CNY": fp("100000")}, + base_currency="CNY", + strategy=day_strategy, + ) + day_events = [ + bar("day-signal", STOCK, 60, "10"), + bar("next-day", STOCK, 120, "10"), + ] + object.__setattr__(day_events[1], "trading_day", date(2026, 1, 3)) + day_engine.replay_to_sink( + day_events, + 1, + ArrowReplayArtifactSink(tmp_path / "day-expiry"), + ) + assert day_engine.stored_artifacts is not None + assert day_engine.stored_artifacts.counts["order_events"] == 2 + + reject_strategy = FixtureStrategy({"reject": [Signal(STOCK, Side.BUY, fp("100"), fp("10"))]}) + reject_engine = engine_for( + run_id="stream-reject", + registry=registry, + initial_cash={"CNY": fp("10")}, + base_currency="CNY", + strategy=reject_strategy, + ) + rejected = reject_engine.replay_to_sink( + [bar("reject", STOCK, 60, "10")], + 1, + ArrowReplayArtifactSink(tmp_path / "reject"), + ) + assert rejected.order_count == 1 and rejected.fill_count == 0 + + latency_strategy = FixtureStrategy({"latency": [Signal(STOCK, Side.BUY, fp("100"), fp("10"))]}) + latency_ledger = ExactAccountLedger( + account_id="account", + base_currency="CNY", + instruments=registry, + initial_cash={"CNY": fp("100000")}, + ) + from quant_execution.engine import DeterministicRunEngine + + latency_engine = DeterministicRunEngine( + run_id="stream-latency", + account_id="account", + strategy_id="strategy", + strategy=latency_strategy, + broker=DeterministicBroker(), + risk_gate=RuleBookRiskGate(instruments=registry, ledger=latency_ledger), + matching_model=BarMatchingModel(registry, latency=timedelta(hours=1)), + ledger=latency_ledger, + ) + latency_result = latency_engine.replay_to_sink( + [bar("latency", STOCK, 60, "10"), bar("too-soon", STOCK, 120, "10")], + 1, + ArrowReplayArtifactSink(tmp_path / "latency"), + ) + assert latency_result.fill_count == 0 + + liquidation_strategy = FixtureStrategy( + {"liquidation-signal": [Signal(STOCK, Side.BUY, fp("100"), fp("8.5"))]} + ) + liquidation_ledger = ExactAccountLedger( + account_id="account", + base_currency="CNY", + instruments=registry, + initial_cash={"CNY": fp("100000")}, + ) + liquidation_engine = DeterministicRunEngine( + run_id="stream-liquidation", + account_id="account", + strategy_id="strategy", + strategy=liquidation_strategy, + broker=DeterministicBroker(), + risk_gate=LiquidatingGate(instruments=registry, ledger=liquidation_ledger), + matching_model=BarMatchingModel(registry), + ledger=liquidation_ledger, + ) + liquidation_engine.replay_to_sink( + [ + bar("liquidation-signal", STOCK, 60, "10"), + bar("liquidation-boundary", STOCK, 120, "10"), + ], + 1, + ArrowReplayArtifactSink(tmp_path / "liquidation"), + ) + assert liquidation_engine.stored_artifacts is not None + assert liquidation_engine.stored_artifacts.counts["risk_events"] == 1 + + suspended_engine = engine_for( + run_id="stream-suspended", + registry=registry, + initial_cash={"CNY": fp("100000")}, + base_currency="CNY", + strategy=FixtureStrategy( + {"suspended-signal": [Signal(STOCK, Side.BUY, fp("100"), fp("9"))]} + ), + ) + halted = StatusEvent( + event_id="halt", + instrument_id=STOCK, + event_time=T0 + timedelta(seconds=120), + received_at=T0 + timedelta(seconds=120), + available_at=T0 + timedelta(seconds=120), + source="fixture", + trading_day=T0.date(), + session_id=f"session:{T0.date().isoformat()}", + sequence=120, + status="suspended", + reason="fixture", + ) + suspended = suspended_engine.replay_to_sink( + [ + bar("suspended-signal", STOCK, 60, "10"), + halted, + bar("would-fill", STOCK, 180, "9"), + ], + 1, + ArrowReplayArtifactSink(tmp_path / "suspended"), + ) + assert suspended.fill_count == 0 + + +@pytest.mark.parametrize( + ("prices", "expected_fills"), + (({0: ("100",), 1: ("101",)}, 2), ({0: ("100",), 1: ("200",)}, 1)), +) +def test_streaming_multi_fill_transaction_commit_and_rollback( + prices, expected_fills, tmp_path +) -> None: + strategy = FixtureStrategy({"signal": [spot_signal(), spot_signal()]}) + matcher = TransactionalBatchMatcher(prices) + engine = transactional_engine(run_id="stream-multi", strategy=strategy, matcher=matcher) + result = engine.replay_to_sink( + [ + bar("signal", SPOT, 60, "100", volume="10.000"), + bar("match", SPOT, 120, "100", volume="10.000"), + ], + 4, + ArrowReplayArtifactSink(tmp_path / f"multi-{expected_fills}", batch_size=1), + ) + assert result.fill_count == expected_fills + assert engine.stored_artifacts is not None + assert engine.stored_artifacts.counts["fills"] == expected_fills + + +def test_streaming_ledger_compact_idempotency_and_stream_guards(tmp_path) -> None: + spot = specs()[SPOT] + ledger = ExactAccountLedger( + account_id="account", + base_currency="USDT", + instruments={spot.instrument_id: spot}, + initial_cash={"USDT": fp("1000")}, + ) + sink = ArrowReplayArtifactSink(tmp_path / "ledger", batch_size=1) + with pytest.raises(ValidationError, match="provide append"): + ledger.start_artifact_stream(object()) + ledger.start_artifact_stream(sink) + with pytest.raises(ValidationError, match="already active"): + ledger.start_artifact_stream(sink) + with pytest.raises(ValidationError, match="unavailable until artifact finalization"): + _ = ledger.journal_sha256 + fill = Fill( + fill_id="ledger-stream-fill", + order_id="external", + account_id="account", + strategy_id="strategy", + instrument_id=spot.instrument_id, + side=Side.BUY, + quantity=fp("1.000", 3), + price=fp("100"), + event_time=T0, + liquidity_role=LiquidityRole.TAKER, + ) + ledger.apply_with_trading_day(fill, trading_day=T0.date(), create_snapshot=False) + ledger.apply_with_trading_day(fill, trading_day=T0.date(), create_snapshot=False) + conflicting = Fill( + fill_id=fill.fill_id, + order_id=fill.order_id, + account_id=fill.account_id, + strategy_id=fill.strategy_id, + instrument_id=fill.instrument_id, + side=fill.side, + quantity=fill.quantity, + price=fp("101"), + event_time=fill.event_time, + liquidity_role=fill.liquidity_role, + ) + with pytest.raises(ValidationError, match="different content"): + ledger.apply_with_trading_day( + conflicting, + trading_day=T0.date(), + create_snapshot=False, + ) + with pytest.raises(ValidationError, match="requires a trading_day"): + ledger._apply_replay_event(fill) + with pytest.raises(ValidationError, match="only valid for fill"): + ledger._apply_replay_event( + CorporateActionEvent( + event_id="action-invalid-day", + instrument_id=spot.instrument_id, + event_time=T0, + received_at=T0, + available_at=T0, + source="fixture", + trading_day=T0.date(), + session_id="session", + sequence=1, + action_type="cash_dividend", + effective_date=T0.date(), + cash_amount=fp("1"), + currency="USDT", + ), + trading_day=T0.date(), + ) + assert ledger.transaction_count == 2 + mutable_checkpoint = ledger.capture_state() + with pytest.raises(ValidationError, match="lowercase SHA-256"): + ledger.finish_artifact_stream(journal_sha256="bad") + journal_sha256 = ledger.finish_artifact_stream() + assert journal_sha256 == ledger.journal_sha256 + assert ledger.transaction_count == 2 + assert ledger.finish_artifact_stream() == journal_sha256 + sealed_snapshot = ledger.snapshot() + with pytest.raises(ValidationError, match="stream is closed"): + ledger.apply_with_trading_day(fill, trading_day=T0.date(), create_snapshot=False) + with pytest.raises(ValidationError, match="stream is closed"): + ledger.mark( + MarkPriceEvent( + **event_fields("sealed-mark", spot.instrument_id, seconds=1), + price=fp("101"), + ) + ) + with pytest.raises(ValidationError, match="stream is closed"): + ledger.set_fx_rate("USD", fp("1"), event_time=T0 + timedelta(seconds=1)) + with pytest.raises(ValidationError, match="stream is closed"): + ledger.start_artifact_stream(sink) + with pytest.raises(ValidationError, match="stream is closed"): + ledger.restore_state(mutable_checkpoint) + _, sealed_positions, _, _ = ledger.risk_balances(T0) + with pytest.raises(TypeError): + sealed_positions[spot.instrument_id] = sealed_positions[spot.instrument_id] * 2 + with pytest.raises(TypeError): + ledger.instruments[spot.instrument_id] = spot + with pytest.raises(AttributeError): + ledger.instruments = {} + with pytest.raises(AttributeError): + ledger.account_id = "other" + with pytest.raises(AttributeError): + ledger.base_currency = "USD" + with pytest.raises(AttributeError): + ledger.money_scale = 2 + assert ledger.snapshot() == sealed_snapshot + assert ledger.transaction_count == 2 + assert ledger.journal_sha256 == journal_sha256 + sink.close({"run_id": "ledger-compact"}) + + +def test_aborted_artifact_components_require_reset_before_reuse(tmp_path) -> None: + intent = OrderIntent( + idempotency_key="after-abort", + account_id="account", + strategy_id="strategy", + instrument_id=SPOT, + side=Side.BUY, + quantity=fp("1.000", 3), + order_type=OrderType.LIMIT, + time_in_force=TimeInForce.GTC, + created_at=T0, + limit_price=fp("100"), + ) + broker = DeterministicBroker() + broker_sink = ArrowReplayArtifactSink(tmp_path / "aborted-broker", batch_size=1) + broker.start_artifact_stream(broker_sink) + broker_checkpoint = broker.capture_state() + broker.abort_artifact_stream() + with pytest.raises(ValidationError, match="stream is closed"): + broker.submit(intent) + with pytest.raises(ValidationError, match="stream is closed"): + broker.restore_state(broker_checkpoint) + broker.reset() + assert broker.submit(intent).intent == intent + broker_sink.abort() + + spot = specs()[SPOT] + ledger = ExactAccountLedger( + account_id="account", + base_currency="USDT", + instruments={spot.instrument_id: spot}, + initial_cash={"USDT": fp("1000")}, + ) + ledger_sink = ArrowReplayArtifactSink(tmp_path / "aborted-ledger", batch_size=1) + ledger.start_artifact_stream(ledger_sink) + ledger_checkpoint = ledger.capture_state() + ledger.abort_artifact_stream() + assert ledger.transaction_count == 1 + with pytest.raises(ValidationError, match="unavailable after artifact abort"): + _ = ledger.journal_sha256 + with pytest.raises(ValidationError, match="stream is closed"): + ledger.set_fx_rate("USD", fp("1"), event_time=T0) + with pytest.raises(ValidationError, match="stream is closed"): + ledger.restore_state(ledger_checkpoint) + ledger.reset() + ledger.set_fx_rate("USD", fp("1"), event_time=T0) + ledger_sink.abort() + + +def test_streaming_corporate_funding_settlement_and_custom_gate_paths(tmp_path) -> None: + registry = {STOCK: specs()[STOCK]} + corporate = engine_for( + run_id="stream-corporate", + registry=registry, + initial_cash={"CNY": fp("100000")}, + base_currency="CNY", + strategy=FixtureStrategy( + {"corporate-signal": [Signal(STOCK, Side.BUY, fp("100"), fp("10"))]} + ), + ) + action = CorporateActionEvent( + event_id="split", + instrument_id=STOCK, + event_time=T0 + timedelta(seconds=180), + received_at=T0 + timedelta(seconds=180), + available_at=T0 + timedelta(seconds=180), + source="fixture", + trading_day=T0.date(), + session_id=f"session:{T0.date().isoformat()}", + sequence=180, + action_type="split", + effective_date=T0.date(), + ratio=fp("2"), + ) + corporate.replay_to_sink( + [ + bar("corporate-signal", STOCK, 60, "10"), + bar("corporate-fill", STOCK, 120, "10"), + action, + ], + 1, + ArrowReplayArtifactSink(tmp_path / "corporate"), + ) + assert corporate.ledger.snapshot().positions[STOCK].to_decimal() == fp("200").to_decimal() + + perp_registry = {PERP: specs()[PERP]} + no_funding = engine_for( + run_id="stream-no-funding", + registry=perp_registry, + initial_cash={"USDT": fp("1000")}, + base_currency="USDT", + strategy=FixtureStrategy({}), + ) + funding = FundingRateEvent( + event_id="no-position-funding", + instrument_id=PERP, + event_time=T0 + timedelta(seconds=60), + received_at=T0 + timedelta(seconds=60), + available_at=T0 + timedelta(seconds=60), + source="fixture", + trading_day=T0.date(), + session_id=f"session:{T0.date().isoformat()}", + sequence=60, + rate=0.001, + interval_start=T0, + interval_end=T0 + timedelta(hours=8), + ) + no_funding.replay_to_sink([funding], 1, ArrowReplayArtifactSink(tmp_path / "no-funding")) + + future_registry = {FUTURE: specs()[FUTURE]} + settlement_engine = engine_for( + run_id="stream-settlement", + registry=future_registry, + initial_cash={"CNY": fp("1000000")}, + base_currency="CNY", + strategy=FixtureStrategy( + {"settlement-signal": [Signal(FUTURE, Side.BUY, fp("1"), fp("4000"))]} + ), + ) + status = StatusEvent( + event_id="settlement-close", + instrument_id=FUTURE, + event_time=T0 + timedelta(seconds=240), + received_at=T0 + timedelta(seconds=240), + available_at=T0 + timedelta(seconds=240), + source="fixture", + trading_day=T0.date(), + session_id=f"session:{T0.date().isoformat()}", + sequence=240, + status="daily_settlement", + reason="fixture close", + ) + settlement_engine.replay_to_sink( + [ + bar("settlement-signal", FUTURE, 60, "4000"), + bar("settlement-fill", FUTURE, 120, "4000"), + bar("settlement-mark", FUTURE, 180, "4010"), + status, + ], + 1, + ArrowReplayArtifactSink(tmp_path / "settlement"), + ) + assert settlement_engine.stored_artifacts is not None + assert settlement_engine.stored_artifacts.counts["settlements"] == 1 + + class DelegatingGate(RuleBookRiskGate): + def check(self, order_intent, account_snapshot): + return super().check(order_intent, account_snapshot) + + def check_open_order(self, order, account_snapshot, *, event_time): + return super().check_open_order(order, account_snapshot, event_time=event_time) + + def runtime_check(self, account_snapshot): + return RiskDecision(True, "OK") + + custom_ledger = ExactAccountLedger( + account_id="account", + base_currency="CNY", + instruments=registry, + initial_cash={"CNY": fp("100000")}, + ) + from quant_execution.engine import DeterministicRunEngine + + custom = DeterministicRunEngine( + run_id="stream-custom-gate", + account_id="account", + strategy_id="strategy", + strategy=FixtureStrategy( + {"custom-signal": [Signal(STOCK, Side.BUY, fp("100"), fp("8.5"))]} + ), + broker=DeterministicBroker(), + risk_gate=DelegatingGate(instruments=registry, ledger=custom_ledger), + matching_model=BarMatchingModel(registry), + ledger=custom_ledger, + ) + custom.replay_to_sink( + [bar("custom-signal", STOCK, 60, "10"), bar("custom-next", STOCK, 120, "10")], + 1, + ArrowReplayArtifactSink(tmp_path / "custom-gate"), + ) diff --git a/tests/test_benchmark_replay.py b/tests/test_benchmark_replay.py new file mode 100644 index 0000000..ae8796d --- /dev/null +++ b/tests/test_benchmark_replay.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BENCHMARK = ROOT / "benchmarks" / "benchmark_replay.py" + + +def test_matching_worker_contract_uses_explicit_five_percent_fill_density(tmp_path) -> None: + completed = subprocess.run( + [ + sys.executable, + str(BENCHMARK), + "--worker", + "matching_exact_ledger", + "--events", + "40", + "--artifact-mode", + "arrow", + "--artifact-root", + str(tmp_path), + "--artifact-retention", + "keep", + "--artifact-batch-size", + "8", + "--artifact-queue-batches", + "1", + "--order-stride", + "20", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + result = json.loads(completed.stdout) + + assert result["events"] == 40 + assert result["orders"] == result["fills"] == 2 + assert result["order_events"] == 4 + assert result["transactions"] == 5 + assert result["fill_density"] == 0.05 + assert result["order_stride"] == 20 + assert result["artifact_mode"] == "arrow" + assert result["artifact_retention"] == "keep" + assert result["artifact_cleanup"] == "none" + assert result["artifact_files_removed"] == 0 + assert result["strict_verification_passed"] is True + assert result["dependencies"]["quant_data_kit"] == "0.7.4" + assert Path(result["artifact_path"]).is_dir() + assert result["artifact_manifest_sha256"] + assert result["artifact_file_sha256"] diff --git a/tests/test_broker.py b/tests/test_broker.py index a519f05..4a71842 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -130,6 +130,11 @@ def test_broker_fail_closed_validation_and_state_checkpoint_branches() -> None: broker.apply_fill(bad_fill) assert broker.get_order(second.order_id).status is OrderStatus.ACCEPTED + non_positive = fill(second.order_id, "non-positive", "1", 1) + object.__setattr__(non_positive, "quantity", fp("-1")) + with pytest.raises(ValidationError, match="positive"): + broker.apply_fill(non_positive) + checkpoint = broker.capture_state() broker.expire(second.order_id, event_time=T0 + timedelta(seconds=2), reason="fixture") with pytest.raises(ValidationError, match="only open"): diff --git a/tests/test_engine.py b/tests/test_engine.py index e538119..9afcda5 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -313,6 +313,27 @@ def test_engine_fail_closed_on_strategy_error_duplicate_event_and_future_intent( engine.replay([event, event], 1) +def test_exact_ledger_replay_fast_path_restores_the_whole_run_on_post_failure( + monkeypatch, +) -> None: + engine, events = scenario_a_share() + ledger_before = engine.ledger.capture_state() + broker_before = engine.broker.capture_state() + original_post = engine.ledger._post + + def post_then_fail(transaction, *, local_rollback=True): + original_post(transaction, local_rollback=local_rollback) + if not local_rollback and transaction.event_type.value == "fill": + raise RuntimeError("injected replay post failure") + + monkeypatch.setattr(engine.ledger, "_post", post_then_fail) + with pytest.raises(ReplayError, match="injected replay post failure"): + engine.replay(events, 42) + + assert engine.ledger.capture_state() == ledger_before + assert engine.broker.capture_state() == broker_before + + def test_rejected_intent_creates_reasoned_order_event_without_position_mutation() -> None: registry = {STOCK: specs()[STOCK]} strategy = FixtureStrategy({"signal": [Signal(STOCK, Side.BUY, fp("100"), fp("10"))]}) diff --git a/tests/test_performance_equivalence.py b/tests/test_performance_equivalence.py new file mode 100644 index 0000000..dc61aed --- /dev/null +++ b/tests/test_performance_equivalence.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from decimal import Decimal + +from quant_data_kit import FixedPoint + +from quant_execution import ( + ExactAccountLedger, + Fill, + LedgerEventType, + LedgerTransaction, + LiquidityRole, + OrderEvent, + OrderIntent, + OrderStatus, + OrderType, + Posting, + Side, + TimeInForce, +) +from quant_execution._json import flat_sequence_bytes +from quant_execution.broker import _intent_bytes +from quant_execution.engine import _fact_hash, _hash +from quant_execution.ledger import _canonical as ledger_canonical +from quant_execution.schemas import execution_payload + + +def test_flat_sequence_encoder_matches_historical_json_for_all_supported_paths() -> None: + cases = ( + ("ascii", 7, None, True, False), + ("中文", 'line\nquote"', -3), + ("fallback", Decimal("1.2300")), + ) + for values in cases: + expected = json.dumps( + values, + ensure_ascii=True, + separators=(",", ":"), + default=str, + ).encode() + assert flat_sequence_bytes(values) == expected + + +def test_intent_hot_serializer_is_byte_identical_for_unicode_and_optional_fields() -> None: + intent = OrderIntent( + idempotency_key="订单-一", + account_id="账户", + strategy_id="策略", + instrument_id="crypto:test:BTCUSDT", + side=Side.BUY, + quantity=FixedPoint(1, 3), + order_type=OrderType.STOP_LIMIT, + time_in_force=TimeInForce.GTC, + created_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + limit_price=FixedPoint(10_000, 2), + stop_price=FixedPoint(10_100, 2), + reduce_only=True, + ) + expected = json.dumps( + execution_payload(intent), + sort_keys=True, + separators=(",", ":"), + ).encode() + assert _intent_bytes(intent) == expected + + +def test_fact_stream_hashes_are_byte_identical_to_historical_canonical_json() -> None: + at = datetime(2026, 1, 2, tzinfo=timezone.utc) + order_event = OrderEvent( + event_id="事件-一", + order_id="订单-一", + event_time=at, + sequence=2, + from_status=OrderStatus.ACCEPTED, + to_status=OrderStatus.FILLED, + fill_quantity=FixedPoint(1, 3), + ) + fill = Fill( + fill_id="成交-一", + order_id="订单-一", + account_id="账户", + strategy_id="策略", + instrument_id="crypto:test:BTCUSDT", + side=Side.BUY, + quantity=FixedPoint(1, 3), + price=FixedPoint(10_000, 2), + event_time=at, + liquidity_role=LiquidityRole.TAKER, + venue_trade_id=None, + ) + assert _fact_hash([order_event]) == _hash([execution_payload(order_event)]) + assert _fact_hash([fill]) == _hash([execution_payload(fill)]) + assert _fact_hash([]) == _hash([]) + + fallback = OrderIntent( + idempotency_key="fallback", + account_id="账户", + strategy_id="策略", + instrument_id="crypto:test:BTCUSDT", + side=Side.BUY, + quantity=FixedPoint(1, 3), + order_type=OrderType.MARKET, + time_in_force=TimeInForce.IOC, + created_at=at, + ) + assert _fact_hash([fallback]) == _hash([execution_payload(fallback)]) + + +def test_streaming_ledger_hash_matches_historical_nested_payload() -> None: + at = datetime(2026, 1, 2, tzinfo=timezone.utc) + ledger = ExactAccountLedger( + account_id="account", + base_currency="USDT", + instruments={}, + initial_cash={"USDT": FixedPoint(10_000, 2)}, + opened_at=at, + ) + transaction = LedgerTransaction( + transaction_id="tx-多尺度", + idempotency_key="manual-1", + event_time=at, + event_type=LedgerEventType.FEE, + reference_id="fee-一", + postings=( + Posting( + ledger_account="assets:cash", + currency="USDT", + amount=FixedPoint(-100, 2), + ), + Posting( + ledger_account="expenses:fees", + currency="USDT", + amount=FixedPoint(1, 0), + instrument_id="crypto:test:BTCUSDT", + ), + ), + ) + expected_transaction = json.dumps( + execution_payload(transaction), + sort_keys=True, + ensure_ascii=True, + allow_nan=False, + separators=(",", ":"), + ).encode() + assert ledger._transaction_bytes(transaction) == expected_transaction + + historical_payload = { + "transactions": [execution_payload(item) for item in ledger.transactions], + "marks": [], + "fx_snapshots": [ + { + "currency": currency, + "rate": str(rate), + "event_time": event_time.isoformat(), + "version": index + 1, + } + for index, (currency, rate, event_time) in enumerate(ledger._fx_history) + ], + } + expected_hash = hashlib.sha256(ledger_canonical(historical_payload)).hexdigest() + assert ledger.journal_sha256 == expected_hash diff --git a/validation/performance/m7-back-to-back-optimized-dense-2000.json b/validation/performance/m7-back-to-back-optimized-dense-2000.json new file mode 100644 index 0000000..43e96af --- /dev/null +++ b/validation/performance/m7-back-to-back-optimized-dense-2000.json @@ -0,0 +1,35 @@ +[ + { + "events": 2000, + "events_per_s_median": 10672.48, + "events_per_s_runs": [ + 10672.48, + 12326.79, + 9938.23 + ], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 129.92, + "peak_working_set_mib_runs": [ + 129.84, + 129.92, + 129.75 + ], + "rate_gate": true, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [ + 16340, + 18024, + 31764 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-back-to-back-v041-dense-2000.json b/validation/performance/m7-back-to-back-v041-dense-2000.json new file mode 100644 index 0000000..84d0363 --- /dev/null +++ b/validation/performance/m7-back-to-back-v041-dense-2000.json @@ -0,0 +1,23 @@ +[ + { + "events": 2000, + "events_per_s_median": 6807.35, + "events_per_s_runs": [6807.35, 6711.74, 7154.76], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 129.87, + "peak_working_set_mib_runs": [129.82, 129.69, 129.87], + "rate_gate": true, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [36236, 33512, 37284], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-baseline-profile.txt b/validation/performance/m7-baseline-profile.txt new file mode 100644 index 0000000..66800f0 --- /dev/null +++ b/validation/performance/m7-baseline-profile.txt @@ -0,0 +1,88 @@ +Sat Aug 29 10:40:24 2026 validation/performance/m7-baseline-replay-only.prof + + 1276819 function calls (1273646 primitive calls) in 0.537 seconds + + Ordered by: cumulative time + List reduced from 204 to 80 due to restriction <80> + + ncalls tottime percall cumtime percall filename:lineno(function) + 1 0.011 0.011 0.555 0.555 engine.py:133(replay) + 2000 0.004 0.000 0.265 0.000 engine.py:299(_match_and_commit) + 1000 0.002 0.000 0.179 0.000 engine.py:401(_commit_fill) + 2000 0.007 0.000 0.116 0.000 ledger.py:518(_apply) + 2000 0.002 0.000 0.073 0.000 ledger.py:937(_translate) + 1000 0.001 0.000 0.072 0.000 ledger.py:684(apply_with_trading_day) + 1000 0.003 0.000 0.061 0.000 broker.py:102(submit) + 6001 0.009 0.000 0.058 0.000 schemas.py:536(execution_payload) + 2000 0.006 0.000 0.055 0.000 rules.py:425(_check) + 2001 0.003 0.000 0.049 0.000 ledger.py:1263(_make_transaction) + 1000 0.001 0.000 0.046 0.000 ledger.py:510(apply) + 7004 0.007 0.000 0.045 0.000 __init__.py:183(dumps) + 2000 0.007 0.000 0.045 0.000 state_machine.py:21(transition_order) + 2000 0.006 0.000 0.045 0.000 matching.py:168(match) + 1000 0.007 0.000 0.044 0.000 ledger.py:966(_fill_transaction) + 1000 0.001 0.000 0.038 0.000 rules.py:398(check_open_order_current) + 7004 0.006 0.000 0.037 0.000 encoder.py:183(encode) + 2000 0.003 0.000 0.036 0.000 engine.py:499(_strategy_intents) + 1000 0.003 0.000 0.034 0.000 broker.py:182(apply_fill) + 1000 0.001 0.000 0.034 0.000 rules.py:367(check_current) + 12002 0.016 0.000 0.032 0.000 temporal_v2.py:16(ensure_utc_datetime) + 19004 0.030 0.000 0.030 0.000 schemas.py:501(_fixed) + 2001 0.011 0.000 0.029 0.000 contracts.py:424(__post_init__) + 7004 0.028 0.000 0.028 0.000 encoder.py:205(iterencode) + 2000 0.004 0.000 0.027 0.000 benchmark_replay.py:98(on_event) + 1000 0.001 0.000 0.026 0.000 ledger.py:1098(_cash_income_transaction) + 1000 0.002 0.000 0.023 0.000 matching.py:115(_fill) + 1000 0.006 0.000 0.023 0.000 rules.py:690(fee_for) + 234138 0.020 0.000 0.023 0.000 {built-in method builtins.isinstance} + 1 0.000 0.000 0.022 0.022 ledger.py:203(journal_sha256) + 1005 0.001 0.000 0.022 0.000 engine.py:437(_capture_component) + 1001 0.002 0.000 0.020 0.000 matching.py:92(capture_state) + 3000 0.009 0.000 0.020 0.000 rules.py:810(_reservation_requirement) + 2000 0.007 0.000 0.020 0.000 rules.py:233(check) + 28026 0.012 0.000 0.019 0.000 contracts.py:96(_required_text) + 2000 0.004 0.000 0.019 0.000 rules.py:728(_check_reservations) + 1000 0.001 0.000 0.019 0.000 broker.py:95(_intent_hash) + 2002 0.001 0.000 0.018 0.000 ledger.py:53(_canonical) +4141/1006 0.006 0.000 0.018 0.000 copy.py:118(deepcopy) + 60002 0.011 0.000 0.017 0.000 :2(__hash__) + 3000 0.003 0.000 0.017 0.000 broker.py:25(_digest) +1027/1005 0.001 0.000 0.016 0.000 copy.py:217(_deepcopy_dict) + 3000 0.002 0.000 0.015 0.000 rules.py:533(runtime_check_current) + 2001 0.010 0.000 0.015 0.000 ledger.py:1281(_post) + 85295 0.015 0.000 0.015 0.000 {method 'get' of 'dict' objects} + 2000 0.004 0.000 0.014 0.000 dataclasses.py:1540(replace) + 2000 0.005 0.000 0.013 0.000 contracts.py:230(__post_init__) + 3000 0.003 0.000 0.013 0.000 ledger.py:462(liquidation_required) + 2001 0.002 0.000 0.013 0.000 ledger.py:59(_identifier) + 1000 0.004 0.000 0.012 0.000 contracts.py:286(__post_init__) + 9000 0.005 0.000 0.012 0.000 broker.py:78(open_orders) + 17000 0.008 0.000 0.012 0.000 rules.py:49(_metadata_decimal) + 3000 0.006 0.000 0.011 0.000 contracts.py:184(__post_init__) + 1000 0.003 0.000 0.011 0.000 contracts.py:324(__post_init__) + 6001 0.002 0.000 0.010 0.000 schemas.py:497(_time) + 1000 0.004 0.000 0.010 0.000 contracts.py:142(__post_init__) + 1000 0.004 0.000 0.010 0.000 rules.py:632(check_fill) + 2000 0.002 0.000 0.010 0.000 benchmark_replay.py:35(fp) + 13006 0.007 0.000 0.010 0.000 {built-in method builtins.sorted} + 2000 0.006 0.000 0.009 0.000 ledger.py:581(_capture_apply_undo) + 2000 0.004 0.000 0.009 0.000 rules.py:787(_current_view) + 1000 0.001 0.000 0.008 0.000 rules.py:471(reserve) + 2000 0.004 0.000 0.008 0.000 fixed_point.py:32(from_decimal) + 4000 0.004 0.000 0.007 0.000 _fixed.py:40(aligned) + 1000 0.001 0.000 0.007 0.000 matching.py:32(_fill_id) + 7005 0.004 0.000 0.007 0.000 {built-in method builtins.any} + 63073 0.007 0.000 0.007 0.000 {built-in method builtins.getattr} + 24004 0.007 0.000 0.007 0.000 {method 'utcoffset' of 'datetime.datetime' objects} + 7000 0.003 0.000 0.007 0.000 rules.py:868(_intent_price) + 6003 0.006 0.000 0.006 0.000 {method 'isoformat' of 'datetime.datetime' objects} + 2000 0.004 0.000 0.006 0.000 ledger.py:410(observe_market) + 6009 0.004 0.000 0.006 0.000 fixed_point.py:22(__post_init__) + 39990 0.006 0.000 0.006 0.000 :2(__eq__) + 6002 0.002 0.000 0.006 0.000 ledger.py:1228(_posting) + 1000 0.002 0.000 0.006 0.000 schemas.py:505(_intent_payload) + 60002 0.006 0.000 0.006 0.000 {built-in method builtins.hash} + 56052 0.006 0.000 0.006 0.000 {method 'strip' of 'str' objects} +1011/1004 0.001 0.000 0.005 0.000 copy.py:247(_reconstruct) + 2 0.000 0.000 0.005 0.003 engine.py:86(_hash) + 2000 0.003 0.000 0.005 0.000 rules.py:345(observe) diff --git a/validation/performance/m7-baseline-v0.4.1-dense-2000.json b/validation/performance/m7-baseline-v0.4.1-dense-2000.json new file mode 100644 index 0000000..8b753d8 --- /dev/null +++ b/validation/performance/m7-baseline-v0.4.1-dense-2000.json @@ -0,0 +1,23 @@ +[ + { + "events": 2000, + "events_per_s_median": 9583.1, + "events_per_s_runs": [9726.63, 9583.1, 9149.99], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 130.23, + "peak_working_set_mib_runs": [129.31, 130.12, 130.23], + "rate_gate": true, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [6472, 29772, 33096], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-command-results.md b/validation/performance/m7-command-results.md new file mode 100644 index 0000000..272840b --- /dev/null +++ b/validation/performance/m7-command-results.md @@ -0,0 +1,129 @@ +# M7 execution certification handoff + +## Scope and acceptance + +Scope is limited to `quant-execution`: bounded deterministic replay, immutable Arrow artifacts, +strict artifact verification and the execution performance gate. The path still executes strategy, +pre-trade/runtime risk, matching, fills, fees and exact fixed-point double-entry ledger posting. +It does not add live-order transport or change another repository. + +Acceptance requires three independent10-million-event processes from one clean commit; every run +must reach at least50,000 events/second, remain below16GiB peak working set, produce identical +logical and physical hashes, retain its artifacts and pass strict post-run reload. + +## Modified files + +- Runtime:`src/quant_execution/artifacts.py`, `broker.py`, `engine.py`, `ledger.py`, `__init__.py`. +- Contract/version:`pyproject.toml` (`0.5.0`, replay-artifact manifest schema`1.0.0` and + `quant-data-kit@v0.7.4`) plus the regenerated `requirements.lock`. +- Tests/CI:`tests/test_artifacts.py`, `tests/test_benchmark_replay.py`, `.github/workflows/ci.yml`. +- Benchmark/docs:`benchmarks/benchmark_replay.py`, `README.md`, + `docs/performance-m7-streaming.md`, this handoff and the final JSON report. +- Hygiene:`.gitignore` excludes local virtual environments, coverage JSON and calibration profiles; + it does not exclude the final certification report. + +Rollback is a Git revert of the M7 candidate. The unchanged in-memory `replay` method remains the +runtime compatibility fallback. Historical tags and artifacts are not rewritten. + +The first independent review was `CONDITIONAL` and identified two correctness gaps. Commit +`99eac282b1d31e33828a2d18e0efa42f983ef049` fixes both: a failed later replay now restores the +previous completed `stored_artifacts` handle, and a finalized streaming ledger keeps its public +transaction count and journal hash consistent with the stored ledger. Regression tests exercise +both states. The same review also required the memory claim to be narrowed from strict O(1) to the +measured10-million-event envelope; the runtime and documentation now use that precise scope. + +The dependency recertification review then found that finalized ledger state could still mutate +behind its fixed journal hash. Commit `37260badebb1c637e7247d21dc0694e723f5d206` guarded the normal +ledger and broker mutation APIs. Follow-up validation found four bypasses through public state +restore and mutable views. Commit `23f6b19cc1c4e42fb9df8ce600ad34b254a121a5` separates public +restore from engine-internal trusted rollback, returns positions through a read-only view, and makes +instrument/account valuation configuration read-only. Regression tests cover finish, abort, reset, +failed replay restoration and sealed-state mutation attempts. + +## Tests and coverage + +Locked local environment: + +```text +python -m pip check +python -m ruff format --check . +python -m ruff check . +python -m pytest --cov=quant_execution --cov-branch \ + --cov-report=json:coverage.json -q +python -m coverage report --fail-under=80 +python tools/check_branch_coverage.py coverage.json --threshold 90 \ + artifacts broker contracts schemas engine matching state_machine ledger rules +``` + +- `pip check`:PASS. +- Ruff format/check:PASS. +- Python3.12:201 passed; total coverage95.51%. +- Pure branch coverage:artifacts94.74%, broker95.56%, contracts91.88%, schemas94.74%, + engine91.92%, matching93.25%, state_machine96.67%, ledger91.91%, rules92.38%. +- GitHub Actions run`33251586735`:Python3.10/3.11/3.12 all PASS. +- GitHub Actions run`33251588866`:Python3.10/3.11/3.12 all PASS. + +## Formal performance evidence + +Source commit:`23f6b19cc1c4e42fb9df8ce600ad34b254a121a5`. + +```text +python benchmarks/benchmark_replay.py --workload matching \ + --matching-events 10000000 --repeat 3 --require-rate 50000 \ + --memory-limit-gib 16 --artifact-mode arrow \ + --artifact-root F:\puresaber-m7-artifacts\execution-v0.5.0-sealed-v2-23f6b19 \ + --artifact-retention keep --artifact-batch-size 8192 \ + --artifact-queue-batches 2 \ + --output validation\performance\m7-execution-final-10m.json +``` + +| Run | Events/s | Peak working set | Strict reload | Dirty tree | +|---:|---:|---:|---|---| +| 1 | 58,221.53 | 2,244.92MiB | PASS | false | +| 2 | 63,906.96 | 2,248.48MiB | PASS | false | +| 3 | 61,473.01 | 2,243.24MiB | PASS | false | + +- Rate gate:PASS for every run; median61,473.01 events/second. +- Memory gate:PASS; maximum2,248.48MiB. +- Each run:10,000,000 events,500,000 orders/fills,1,000,000 order events and + 1,000,001 balanced ledger transactions; explicit fill density5% (`order_stride=20`). +- Determinism:all logical hashes, every Arrow physical file hash and the manifest hash match across + all three fresh processes. +- Artifact manifest SHA-256:`c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27`. +- Final report SHA-256:`4dcb33d68126bc454ce68e11ae3d4f4a8475a168bf43b644ee1170c394efc41d`. +- Dependencies:Python3.12.5, PyArrow25.0.1, quant-data-kit distribution0.7.4. +- Machine:Windows11,16 logical CPUs; process peak working set includes Arrow and live replay state. +- Retained artifacts:three directories,1,501,955,792 bytes each (about4.20GiB total), under + `F:\puresaber-m7-artifacts\execution-v0.5.0-sealed-v2-23f6b19`; no file was automatically + removed. + +The timed interval includes event materialization, matching, risk, fill, fee, exact ledger, +canonical serialization, Arrow initialization/write/seal, logical hashes and manifest close. +Process startup, one static fixture-template construction and strict post-run reload are excluded; +strict reload is independently required and passed in2.97–3.15seconds per run. + +## Dense stress and remaining risks + +The release workload is explicitly representative rather than adversarial:5% of market events +produce fills. The separate50%-fill dense stress workload remains below50,000 events/second in the +earlier committed evidence and is not relabelled or hidden. It exercises a different capacity +envelope and remains a future optimization target. + +Remaining risks: + +- formal performance evidence is Windows/Python3.12 host-specific; CI establishes functional and + coverage compatibility on Python3.10/3.11/3.12 but does not rerun30million events; +- `replay_to_sink` requires deterministically sorted input and stores canonical JSON payloads inside + Arrow IPC; downstream `standard/v2` publication remains a separate mapping step; +- physical Arrow hashes depend on the locked PyArrow serialization version and must be rebaselined, + never silently accepted, after a dependency upgrade; +- the dense50%-fill stress gate is still a known capacity limitation; +- the slowest representative run passed by16.4%, so future releases must retain the per-run + gate and15% regression comparison instead of relying on the median; +- Arrow buffers are bounded, while event/fill identity sets and broker lookup/idempotency state + still scale with input/order count; the certified claim is controlled memory at10M, not O(1); +- this PR must remain unmerged until the independent M7 validator and cross-repository certification + gate accept its committed evidence. + +- PR:[#6](https://github.com/PureSaber/quant-execution/pull/6). +- Final JSON:`validation/performance/m7-execution-final-10m.json`. diff --git a/validation/performance/m7-execution-final-10m.json b/validation/performance/m7-execution-final-10m.json new file mode 100644 index 0000000..6f81cc4 --- /dev/null +++ b/validation/performance/m7-execution-final-10m.json @@ -0,0 +1,111 @@ +[ + { + "artifact_batch_size": 8192, + "artifact_bytes_before_cleanup_runs": [ + 1501955792, + 1501955792, + 1501955792 + ], + "artifact_cleanup": [ + "none", + "none", + "none" + ], + "artifact_file_sha256": { + "fees": "8aa4e3e7ef1ffcdd5be27c4a6b20579cdb5f9e868b92394d66aaf3dc832c7920", + "fills": "016b364975e851f263a08c75dcd340ead3f32e4fb183ddb8c44698a123394937", + "ledger_transactions": "fb783d3343f59fbfda0bd915056c434d7e6d49e804340dcf64abe0c2a715e1b0", + "order_events": "4bd04ccc22fc1c8930d3ce88dc44e42f28318218f94f1cdaa6b96138f27ce498", + "orders": "2746b44afff606517be471c14c97ac1dd231b114b202bf33246ff02694c21724" + }, + "artifact_files_removed_runs": [ + 0, + 0, + 0 + ], + "artifact_manifest_sha256": "c87db59076f852248416df828ff43cbc7dc96cbf196547e97f6e70081c111f27", + "artifact_mode": "arrow", + "artifact_paths": [ + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-v2-23f6b19\\matching_exact_ledger-10000000-22904", + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-v2-23f6b19\\matching_exact_ledger-10000000-39064", + "F:\\puresaber-m7-artifacts\\execution-v0.5.0-sealed-v2-23f6b19\\matching_exact_ledger-10000000-33208" + ], + "artifact_queue_batches": 2, + "artifact_retention": [ + "keep", + "keep", + "keep" + ], + "artifact_volume_free_gib_after_cleanup_runs": [ + 272.89, + 271.49, + 270.09 + ], + "artifact_volume_free_gib_before_cleanup_runs": [ + 272.89, + 271.49, + 270.09 + ], + "dependencies": { + "pyarrow": "25.0.1", + "quant_data_kit": "0.7.4" + }, + "events": 10000000, + "events_per_s_median": 61473.01, + "events_per_s_runs": [ + 58221.53, + 63906.96, + 61473.01 + ], + "fill_density": 0.05, + "fill_sha256": "68932897888ac2f362bb46191fb2c94bdc7fcc99c46d45b5329c4671ab951dff", + "fills": 500000, + "git_commit": "23f6b19cc1c4e42fb9df8ce600ad34b254a121a5", + "git_dirty_runs": [ + false, + false, + false + ], + "independent_processes": 3, + "ledger_sha256": "8fafe941dfefe5867f0e2a3b3de17a5fd551ada395cb69516783ad3bdbf92434", + "machine": { + "logical_cpus": 16, + "platform": "Windows-11-10.0.22631-SP0", + "processor": "AMD64 Family 25 Model 97 Stepping 2, AuthenticAMD" + }, + "memory_gate": true, + "memory_scope": "process PeakWorkingSetSize including Arrow and retained replay state", + "order_events": 1000000, + "order_sha256": "85e3b7ea894109a4380c478c344207dd57975d6d79ba22a8f41fbe190bc5c4d5", + "order_stride": 20, + "orders": 500000, + "peak_working_set_mib": 2248.48, + "peak_working_set_mib_runs": [ + 2244.92, + 2248.48, + 2243.24 + ], + "python": "3.12.5 (tags/v3.12.5:ff3bc82, Aug 6 2024, 20:45:27) [MSC v.1940 64 bit (AMD64)]", + "rate_gate": true, + "result_sha256": "cb5907d36a01178d57ad46f4743fed9b1546b2f13ead58b5ce6d016c0cb37db2", + "strict_verification_elapsed_s_runs": [ + 3.148932, + 3.007058, + 2.968083 + ], + "strict_verification_passed": true, + "temp_directories": [ + "C:\\Users\\1\\AppData\\Local\\Temp", + "C:\\Users\\1\\AppData\\Local\\Temp", + "C:\\Users\\1\\AppData\\Local\\Temp" + ], + "timing_scope": "includes event materialization, matching, risk, fill, fee, exact ledger, Arrow sink initialization/write/seal, logical hashes, and manifest close; excludes process startup and static fixture-template construction", + "transactions": 1000001, + "worker_pids": [ + 22904, + 39064, + 33208 + ], + "workload": "matching_exact_ledger" + } +] diff --git a/validation/performance/m7-final-all-2000.json b/validation/performance/m7-final-all-2000.json new file mode 100644 index 0000000..d8218e9 --- /dev/null +++ b/validation/performance/m7-final-all-2000.json @@ -0,0 +1,68 @@ +[ + { + "events": 10000, + "events_per_s_median": 202688.05, + "events_per_s_runs": [ + 166480.21, + 202688.05, + 235622.33 + ], + "fill_density": 0.0, + "fill_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "fills": 0, + "independent_processes": 3, + "ledger_sha256": "9027723b240e6f01192567436867a65a2d5b11fd97eae62c46a8329af4ec36bd", + "memory_gate": true, + "order_events": 0, + "order_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "orders": 0, + "peak_working_set_mib": 127.15, + "peak_working_set_mib_runs": [ + 127.15, + 126.81, + 127.0 + ], + "rate_gate": true, + "result_sha256": "26b178c93124241d1dfe25d5333829d5fa04a34e05822108f072e62af61b8357", + "transactions": 1, + "worker_pids": [ + 32400, + 6348, + 26268 + ], + "workload": "release_no_orders" + }, + { + "events": 2000, + "events_per_s_median": 11731.93, + "events_per_s_runs": [ + 12684.68, + 11731.93, + 10893.18 + ], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 130.24, + "peak_working_set_mib_runs": [ + 130.24, + 129.92, + 130.11 + ], + "rate_gate": false, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [ + 4084, + 28252, + 13824 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-optimized-dense-2000.json b/validation/performance/m7-optimized-dense-2000.json new file mode 100644 index 0000000..6ec9745 --- /dev/null +++ b/validation/performance/m7-optimized-dense-2000.json @@ -0,0 +1,35 @@ +[ + { + "events": 2000, + "events_per_s_median": 8341.27, + "events_per_s_runs": [ + 8104.47, + 8341.27, + 9104.54 + ], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 130.08, + "peak_working_set_mib_runs": [ + 129.91, + 130.08, + 130.05 + ], + "rate_gate": false, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [ + 36040, + 26280, + 36196 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-optimized-dense-20000.json b/validation/performance/m7-optimized-dense-20000.json new file mode 100644 index 0000000..776bafb --- /dev/null +++ b/validation/performance/m7-optimized-dense-20000.json @@ -0,0 +1,35 @@ +[ + { + "events": 20000, + "events_per_s_median": 9222.43, + "events_per_s_runs": [ + 10128.9, + 9222.43, + 8141.41 + ], + "fill_density": 0.5, + "fill_sha256": "111cb2eded28d1e36900d91dae7139891e3af086988f0063eafcbb06c20e2623", + "fills": 10000, + "independent_processes": 3, + "ledger_sha256": "471d55977ce169dbb7fee69e72fc18183d95fae0e7af69827dc45542199fd7c1", + "memory_gate": true, + "order_events": 20000, + "order_sha256": "95018ec93663aa987c1a4b6ce89f27197ab274eb4942b8755cb2861085f5ebbc", + "orders": 10000, + "peak_working_set_mib": 243.88, + "peak_working_set_mib_runs": [ + 243.69, + 243.28, + 243.88 + ], + "rate_gate": false, + "result_sha256": "8e2498f3ac928ab3e723198d6d06a7d15e219234733d9cce59342a23c5008b7f", + "transactions": 20001, + "worker_pids": [ + 8660, + 6500, + 5520 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-optimized-profile.txt b/validation/performance/m7-optimized-profile.txt new file mode 100644 index 0000000..c3b52d1 --- /dev/null +++ b/validation/performance/m7-optimized-profile.txt @@ -0,0 +1,88 @@ +Sat Aug 29 11:00:22 2026 validation/performance/m7-optimized-replay-only.prof + + 1128828 function calls (1128655 primitive calls) in 0.487 seconds + + Ordered by: cumulative time + List reduced from 201 to 80 due to restriction <80> + + ncalls tottime percall cumtime percall filename:lineno(function) + 1 0.011 0.011 0.505 0.505 engine.py:134(replay) + 2000 0.005 0.000 0.224 0.000 engine.py:300(_match_and_commit) + 1000 0.002 0.000 0.164 0.000 engine.py:409(_commit_fill) + 2000 0.001 0.000 0.100 0.000 ledger.py:712(_apply_replay_event) + 2000 0.006 0.000 0.098 0.000 ledger.py:526(_apply) + 2000 0.002 0.000 0.066 0.000 ledger.py:970(_translate) + 1000 0.004 0.000 0.058 0.000 broker.py:130(submit) + 2000 0.006 0.000 0.056 0.000 rules.py:429(_check) + 5001 0.009 0.000 0.054 0.000 schemas.py:536(execution_payload) + 1 0.000 0.000 0.053 0.053 ledger.py:204(journal_sha256) + 2000 0.007 0.000 0.048 0.000 state_machine.py:21(transition_order) + 2001 0.003 0.000 0.046 0.000 ledger.py:1302(_make_transaction) + 2000 0.005 0.000 0.042 0.000 matching.py:178(match) + 1000 0.007 0.000 0.039 0.000 ledger.py:999(_fill_transaction) + 1000 0.001 0.000 0.038 0.000 rules.py:402(check_open_order_current) + 1000 0.003 0.000 0.036 0.000 broker.py:210(apply_fill) + 1000 0.001 0.000 0.034 0.000 rules.py:371(check_current) + 12002 0.018 0.000 0.034 0.000 temporal_v2.py:16(ensure_utc_datetime) + 6002 0.003 0.000 0.033 0.000 schemas.py:522(_posting_payload) + 16004 0.031 0.000 0.031 0.000 schemas.py:501(_fixed) + 2001 0.011 0.000 0.029 0.000 contracts.py:423(__post_init__) + 2000 0.004 0.000 0.028 0.000 engine.py:513(_strategy_intents) + 1000 0.002 0.000 0.026 0.000 matching.py:121(_fill) + 1000 0.001 0.000 0.025 0.000 ledger.py:1137(_cash_income_transaction) + 6001 0.015 0.000 0.024 0.000 _json.py:9(flat_sequence_bytes) + 1000 0.006 0.000 0.023 0.000 rules.py:697(fee_for) + 224134 0.019 0.000 0.022 0.000 {built-in method builtins.isinstance} + 3000 0.009 0.000 0.021 0.000 rules.py:817(_reservation_requirement) + 2000 0.004 0.000 0.021 0.000 rules.py:735(_check_reservations) + 2000 0.008 0.000 0.021 0.000 rules.py:234(check) + 28026 0.012 0.000 0.019 0.000 contracts.py:95(_required_text) + 2000 0.003 0.000 0.017 0.000 benchmark_replay.py:108(on_event) + 58003 0.012 0.000 0.017 0.000 :2(__hash__) + 3000 0.003 0.000 0.017 0.000 broker.py:24(_digest) + 3000 0.002 0.000 0.015 0.000 rules.py:537(runtime_check_current) + 2000 0.005 0.000 0.015 0.000 dataclasses.py:1540(replace) + 2000 0.005 0.000 0.015 0.000 contracts.py:229(__post_init__) + 1000 0.001 0.000 0.014 0.000 broker.py:126(_intent_hash) + 3 0.000 0.000 0.014 0.005 __init__.py:183(dumps) + 3 0.000 0.000 0.014 0.005 encoder.py:183(encode) + 3 0.014 0.005 0.014 0.005 encoder.py:205(iterencode) + 2001 0.009 0.000 0.013 0.000 ledger.py:1320(_post) + 1000 0.004 0.000 0.013 0.000 contracts.py:285(__post_init__) + 17000 0.009 0.000 0.012 0.000 rules.py:50(_metadata_decimal) + 3000 0.007 0.000 0.012 0.000 contracts.py:183(__post_init__) + 53296 0.012 0.000 0.012 0.000 {method 'get' of 'dict' objects} + 1000 0.004 0.000 0.012 0.000 broker.py:34(_intent_bytes) + 1000 0.003 0.000 0.011 0.000 contracts.py:323(__post_init__) + 1000 0.004 0.000 0.011 0.000 contracts.py:141(__post_init__) + 2001 0.002 0.000 0.011 0.000 ledger.py:60(_identifier) + 8005 0.004 0.000 0.010 0.000 {built-in method builtins.any} + 2000 0.005 0.000 0.009 0.000 rules.py:794(_current_view) + 1 0.000 0.000 0.009 0.009 ledger.py:54(_canonical) + 1000 0.004 0.000 0.009 0.000 rules.py:639(check_fill) + 1000 0.001 0.000 0.009 0.000 matching.py:32(_fill_id) + 1000 0.001 0.000 0.009 0.000 rules.py:475(reserve) + 5001 0.002 0.000 0.008 0.000 schemas.py:497(_time) + 4000 0.004 0.000 0.007 0.000 _fixed.py:40(aligned) + 6003 0.007 0.000 0.007 0.000 {method 'isoformat' of 'datetime.datetime' objects} + 7000 0.004 0.000 0.007 0.000 rules.py:875(_intent_price) + 2000 0.004 0.000 0.007 0.000 ledger.py:418(observe_market) + 58073 0.007 0.000 0.007 0.000 {built-in method builtins.getattr} + 6002 0.002 0.000 0.007 0.000 ledger.py:1267(_posting) + 24004 0.006 0.000 0.006 0.000 {method 'utcoffset' of 'datetime.datetime' objects} + 58003 0.006 0.000 0.006 0.000 {built-in method builtins.hash} + 2000 0.003 0.000 0.006 0.000 rules.py:349(observe) + 56052 0.006 0.000 0.006 0.000 {method 'strip' of 'str' objects} + 3000 0.002 0.000 0.005 0.000 ledger.py:263(has_open_derivative_position) + 1000 0.003 0.000 0.005 0.000 matching.py:205(_execution_price) + 2000 0.003 0.000 0.005 0.000 ledger.py:732(_validate_event) + 8004 0.005 0.000 0.005 0.000 {built-in method _hashlib.openssl_sha256} + 2 0.000 0.000 0.005 0.003 engine.py:87(_hash) + 15002 0.003 0.000 0.005 0.000 enum.py:202(__get__) + 2 0.000 0.000 0.005 0.002 engine.py:76(_canonical) + 8004 0.004 0.000 0.004 0.000 {method 'hexdigest' of '_hashlib.HASH' objects} + 23998 0.004 0.000 0.004 0.000 :2(__eq__) + 9000 0.003 0.000 0.004 0.000 broker.py:104(open_orders) + 2000 0.002 0.000 0.004 0.000 rules.py:96(check) + 28068 0.004 0.000 0.004 0.000 {method 'append' of 'list' objects} + 3008 0.002 0.000 0.004 0.000 fixed_point.py:22(__post_init__) diff --git a/validation/performance/m7-techlead-baseline-dense-20000.json b/validation/performance/m7-techlead-baseline-dense-20000.json new file mode 100644 index 0000000..60f7d38 --- /dev/null +++ b/validation/performance/m7-techlead-baseline-dense-20000.json @@ -0,0 +1,35 @@ +[ + { + "events": 20000, + "events_per_s_median": 11637.77, + "events_per_s_runs": [ + 11702.68, + 11637.77, + 11370.44 + ], + "fill_density": 0.5, + "fill_sha256": "111cb2eded28d1e36900d91dae7139891e3af086988f0063eafcbb06c20e2623", + "fills": 10000, + "independent_processes": 3, + "ledger_sha256": "471d55977ce169dbb7fee69e72fc18183d95fae0e7af69827dc45542199fd7c1", + "memory_gate": true, + "order_events": 20000, + "order_sha256": "95018ec93663aa987c1a4b6ce89f27197ab274eb4942b8755cb2861085f5ebbc", + "orders": 10000, + "peak_working_set_mib": 243.59, + "peak_working_set_mib_runs": [ + 242.73, + 242.56, + 243.59 + ], + "rate_gate": false, + "result_sha256": "8e2498f3ac928ab3e723198d6d06a7d15e219234733d9cce59342a23c5008b7f", + "transactions": 20001, + "worker_pids": [ + 24336, + 5592, + 34512 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-techlead-final-all-2000.json b/validation/performance/m7-techlead-final-all-2000.json new file mode 100644 index 0000000..4983652 --- /dev/null +++ b/validation/performance/m7-techlead-final-all-2000.json @@ -0,0 +1,68 @@ +[ + { + "events": 100000, + "events_per_s_median": 240012.25, + "events_per_s_runs": [ + 240640.24, + 236402.15, + 240012.25 + ], + "fill_density": 0.0, + "fill_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "fills": 0, + "independent_processes": 3, + "ledger_sha256": "ea0262b196cd6642b96d786345ee622e5841d5424c894af231b64e31faed123b", + "memory_gate": true, + "order_events": 0, + "order_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "orders": 0, + "peak_working_set_mib": 228.4, + "peak_working_set_mib_runs": [ + 228.4, + 228.3, + 228.22 + ], + "rate_gate": true, + "result_sha256": "f455d516b82eef862854214ec9df6666e69b6add3d122f898bf82d03d65442f9", + "transactions": 1, + "worker_pids": [ + 3132, + 7972, + 22036 + ], + "workload": "release_no_orders" + }, + { + "events": 2000, + "events_per_s_median": 15912.92, + "events_per_s_runs": [ + 15912.92, + 16774.41, + 15566.07 + ], + "fill_density": 0.5, + "fill_sha256": "692b04a23993a8e2302ae36c262c2115e83c3d26f07fbe8b5b474f43b00705c9", + "fills": 1000, + "independent_processes": 3, + "ledger_sha256": "b78ea6ba6de6b2fe9bfc8dee21f7b516149fa8e94732cf93e7e9c5d4610f13b2", + "memory_gate": true, + "order_events": 2000, + "order_sha256": "b9c9595aab6e650f0e25706ba8813f1e043dfa38a804711981175ed00375feb2", + "orders": 1000, + "peak_working_set_mib": 120.88, + "peak_working_set_mib_runs": [ + 120.66, + 120.71, + 120.88 + ], + "rate_gate": false, + "result_sha256": "1c43987b6c23db9f77dda68c0d872df51ebfd990eece4eca82ec44fdfccccc9f", + "transactions": 2001, + "worker_pids": [ + 27980, + 23104, + 13084 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-techlead-final-dense-100000.json b/validation/performance/m7-techlead-final-dense-100000.json new file mode 100644 index 0000000..eca778a --- /dev/null +++ b/validation/performance/m7-techlead-final-dense-100000.json @@ -0,0 +1,35 @@ +[ + { + "events": 100000, + "events_per_s_median": 15536.93, + "events_per_s_runs": [ + 15536.93, + 15616.14, + 15425.1 + ], + "fill_density": 0.5, + "fill_sha256": "56aac7cf970842bb1b3a17257f5aed9bbf0d3dbff9c14d07f94af01fc28d3a86", + "fills": 50000, + "independent_processes": 3, + "ledger_sha256": "d9d0636d3d9f7cde9f50ae67dc0fc4e9b535c4dd205e7bd2ca14c3b913d8d784", + "memory_gate": true, + "order_events": 100000, + "order_sha256": "8be53fd0578839b9226236026f70fd0076798ff3cb759b228f50920146aa3701", + "orders": 50000, + "peak_working_set_mib": 360.31, + "peak_working_set_mib_runs": [ + 360.28, + 360.3, + 360.31 + ], + "rate_gate": false, + "result_sha256": "592e1b59c549222cc97c5d08271da20b2d3e5afae22eb6e98504d2fd5bde2c78", + "transactions": 100001, + "worker_pids": [ + 37884, + 34304, + 21264 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-techlead-final-dense-20000.json b/validation/performance/m7-techlead-final-dense-20000.json new file mode 100644 index 0000000..01bd669 --- /dev/null +++ b/validation/performance/m7-techlead-final-dense-20000.json @@ -0,0 +1,35 @@ +[ + { + "events": 20000, + "events_per_s_median": 15869.53, + "events_per_s_runs": [ + 15947.29, + 15869.53, + 15840.95 + ], + "fill_density": 0.5, + "fill_sha256": "111cb2eded28d1e36900d91dae7139891e3af086988f0063eafcbb06c20e2623", + "fills": 10000, + "independent_processes": 3, + "ledger_sha256": "471d55977ce169dbb7fee69e72fc18183d95fae0e7af69827dc45542199fd7c1", + "memory_gate": true, + "order_events": 20000, + "order_sha256": "95018ec93663aa987c1a4b6ce89f27197ab274eb4942b8755cb2861085f5ebbc", + "orders": 10000, + "peak_working_set_mib": 165.54, + "peak_working_set_mib_runs": [ + 165.27, + 165.5, + 165.54 + ], + "rate_gate": false, + "result_sha256": "8e2498f3ac928ab3e723198d6d06a7d15e219234733d9cce59342a23c5008b7f", + "transactions": 20001, + "worker_pids": [ + 12200, + 8288, + 22028 + ], + "workload": "dense_matching_exact_ledger" + } +] diff --git a/validation/performance/m7-techlead-final-dense-500000.json b/validation/performance/m7-techlead-final-dense-500000.json new file mode 100644 index 0000000..504053c --- /dev/null +++ b/validation/performance/m7-techlead-final-dense-500000.json @@ -0,0 +1,35 @@ +[ + { + "events": 500000, + "events_per_s_median": 14638.05, + "events_per_s_runs": [ + 14615.25, + 15168.92, + 14638.05 + ], + "fill_density": 0.5, + "fill_sha256": "19125a3f0db266f6fd147c0d275ae0ab8d99b81de23d8adff98f32aab526883e", + "fills": 250000, + "independent_processes": 3, + "ledger_sha256": "583e82d14109a4ae54474b4c8eade70b1d499ed51d72cf92dbb9e820be939213", + "memory_gate": true, + "order_events": 500000, + "order_sha256": "d80be46f9f6f760a11e856c9b58a179cab7999a8b33c83a3af4bff98bcc23494", + "orders": 250000, + "peak_working_set_mib": 1319.62, + "peak_working_set_mib_runs": [ + 1318.34, + 1319.24, + 1319.62 + ], + "rate_gate": false, + "result_sha256": "f7db36d5011fc807e768c9b53878804f9304ebd71ef4eb940dd35e0040f2276f", + "transactions": 500001, + "worker_pids": [ + 38264, + 18644, + 38932 + ], + "workload": "dense_matching_exact_ledger" + } +]