diff --git a/README.md b/README.md index 61fcbb4..aa54cc3 100644 --- a/README.md +++ b/README.md @@ -35,14 +35,14 @@ SYNTHETIC DATA - deterministic generator, not a real venue. policy style fill maker vs_arrival_bps vs_vwap_bps is_bps participation ----------- ---------- ------- ------ -------------- ----------- ------- ------------- -twap passive 100.00% 96.3% -7.9369 -0.8977 -7.9369 2.198% -twap aggressive 100.00% 5.1% -6.9336 0.1063 -6.9336 2.198% -vwap passive 100.00% 96.3% -7.9001 -0.8608 -7.9001 2.198% -vwap aggressive 100.00% 5.3% -6.8957 0.1442 -6.8957 2.198% -pov passive 100.00% 100.0% -5.0705 1.9707 -5.0705 2.198% -pov aggressive 100.00% 3.4% -3.3543 3.6881 -3.3543 2.198% -is_baseline passive 100.00% 97.4% -7.1082 -0.0685 -7.1082 2.198% -is_baseline aggressive 100.00% 4.8% -6.1079 0.9325 -6.1079 2.198% +twap passive 100.00% 96.3% -7.9369 -0.8977 -8.0246 2.198% +twap aggressive 100.00% 5.1% -6.9336 0.1063 -5.4720 2.198% +vwap passive 100.00% 96.3% -7.9001 -0.8608 -7.9874 2.198% +vwap aggressive 100.00% 5.3% -6.8957 0.1442 -5.4376 2.198% +pov passive 100.00% 100.0% -5.0705 1.9707 -5.2204 2.198% +pov aggressive 100.00% 3.4% -3.3543 3.6881 -1.8626 2.198% +is_baseline passive 100.00% 97.4% -7.1082 -0.0685 -7.2143 2.198% +is_baseline aggressive 100.00% 4.8% -6.1079 0.9325 -4.6396 2.198% ``` **Read the first two cost columns together.** Passive TWAP beats both benchmarks; @@ -52,6 +52,10 @@ market move between arrival and the volume-weighted price during the order windo The platform ships with `sql/02_benchmark_disagreement.sql` and presents both cost columns together. +`vs_arrival_bps` and `vs_vwap_bps` compare execution prices. `is_bps` includes +explicit fees, net rebates and any unfilled opportunity cost. The quoted tables +and report are [checked against the public CLI](tests/unit/test_documentation.py). + Participation divides filled quantity by market volume observed inside the parent execution window. Warm-up and later markout events do not enter that denominator. diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index b62931e..45eca76 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -73,39 +73,41 @@ python -m tradeforge.interfaces.cli run --policy twap --style passive ``` ``` -metric value ------------------------------- ----------------------------- -policy twap -side BUY -fill_ratio 1 -participation_rate 0.02198 -cost_vs_arrival_bps -7.9369 -cost_vs_vwap_bps -0.8977 -implementation_shortfall_bps -8.0246 -price_shortfall_bps -7.9369 -maker_fill_ratio 0.9634 -queue_mode APPROXIMATE -latency_basis scenario -counterfactual_mode replay_approximation +metric value +---------------------------- -------------------- +cost_vs_arrival_bps -7.9369 +cost_vs_twap_bps -0.8667 +cost_vs_vwap_bps -0.8977 +counterfactual_mode replay_approximation +fill_ratio 1 +implementation_shortfall_bps -8.0246 +latency_basis scenario +maker_fill_ratio 0.9634 +participation_rate 0.022 +policy twap +price_shortfall_bps -7.9369 +queue_mode APPROXIMATE +side BUY Benchmarks (ticks) -benchmark value basis ---------------- ------- ---------------------------------- -arrival_mid 10000 first mid at/after window start -interval_vwap 9992.96 volume weighted over 8417 prints -interval_twap 9992.93 time weighted mid, trapezoidal -terminal_mid 9976.5 last mid inside the window +benchmark value basis +------------- --------- ------------------------------- +arrival_mid 10000 first mid at/after window start +interval_vwap 9992.9602 8417 prints +interval_twap 9992.9292 time weighted mid +interval_mid 9992.9867 unweighted mean of mids +terminal_mid 9976.5 last mid inside the window Cost attribution (bps, positive = worse) -component bps ------------------------ --------- -is_filled -7.9369 -spread_cost -0.479944 -fees -0.0877492 -timing -7.45728 -residual (unexplained) 0.0880677 -opportunity_cost -23.5 -is_total -7.9369 +component bps +---------------------- ------- +is_filled -8.0246 +spread_cost -0.4799 +fees -0.0877 +timing -7.4573 +residual (unexplained) 0.0003 +opportunity_cost -23.5 +is_total -8.0246 Markouts (bps, positive = adverse) horizon_s measurable of_fills volume_weighted median @@ -121,8 +123,9 @@ Four things worth noticing: 1. `queue_mode` is `APPROXIMATE`. On L2 data the queue position is not observable, so the number is an estimate. The report says so at the top, not in a footnote. -2. The attribution closes exactly: `is_filled` equals the sum of the four - components. `residual` is labelled *unexplained* and is small here (0.09 bps). +2. Before display rounding, `is_filled` equals the sum of the four components, + including fees. `residual` is labelled *unexplained* and is small here + (0.0003 bps). When it is large the report says the decomposition is not describing the execution. 3. `opportunity_cost` is −23.5 bps and contributes nothing, because the fill diff --git a/tests/unit/test_documentation.py b/tests/unit/test_documentation.py index 8d50fba..b60d56d 100644 --- a/tests/unit/test_documentation.py +++ b/tests/unit/test_documentation.py @@ -9,11 +9,8 @@ costs the reader the time to discover the docs are wrong before they can start looking for the real answer. The checks here are cheap and catch exactly that. -**What is not checked.** The prose, the numbers in the tables, and the Python -snippets in the guides are not executed. Verifying every snippet would mean -maintaining a doctest harness for code that is deliberately illustrative; the -`make demo` table is checked because it is the repository's headline claim and -because it is cheap to check. +The quoted demo tables and getting-started report are checked against the +public CLI. Prose and illustrative Python snippets are not executed. """ from __future__ import annotations @@ -30,6 +27,7 @@ EXAMPLES = sorted((ROOT / "examples").glob("*.py")) GUIDES = sorted((ROOT / "docs/guides").glob("*.md")) DOCS = [ROOT / "README.md", *GUIDES] +GETTING_STARTED = ROOT / "docs/guides/getting-started.md" #: Prose that happens to contain the word "make", not a target reference. _PROSE = {"the", "a", "it", "them", "sure", "this", "that", "no"} @@ -51,6 +49,53 @@ def makefile_targets() -> set[str]: return set(re.findall(r"^([a-z][a-z0-9-]*):", makefile, re.M)) +def quoted_block(path: Path, header: str) -> str: + blocks = [block for block in code_spans(path) if block.startswith("```")] + pattern = rf"(? dict[tuple[str, str], dict[str, str]]: + lines = text.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith("policy ")) + columns = lines[start].split() + rows = {} + for line in lines[start + 2 :]: + cells = line.split() + if not cells: + break + assert len(cells) == len(columns), f"malformed demo row: {line!r}" + key = tuple(cells[:2]) + assert key not in rows, f"duplicate demo row: {key}" + rows[key] = dict(zip(columns[2:], cells[2:], strict=True)) + assert rows, "demo table has no rows" + return rows + + +@pytest.fixture(scope="module") +def cli_runs() -> dict[str, str]: + """Exercise the documented commands once, including their rendered columns.""" + outputs = {} + for name, args in { + "demo": ["demo"], + "run": ["run", "--policy", "twap", "--style", "passive"], + }.items(): + result = subprocess.run( + [sys.executable, "-m", "tradeforge.interfaces.cli", *args], + capture_output=True, + text=True, + cwd=ROOT, + env={"PYTHONPATH": str(ROOT / "src"), "PATH": "/usr/bin:/bin"}, + timeout=240, + check=False, + ) + assert result.returncode == 0, f"{name} failed:\n{result.stderr[-1500:]}" + outputs[name] = result.stdout + return outputs + + @pytest.fixture(scope="module") def example_runs() -> dict[str, subprocess.CompletedProcess]: """Run each example once and share the result. @@ -176,45 +221,53 @@ def test_every_documented_module_is_runnable(self): class TestTheHeadlineClaim: """`getting-started.md` quotes the `make demo` table as the repository's point. - The guide's argument is that every strategy is about 7 bps cheaper than the - arrival price and about 3 bps more expensive than the interval VWAP, and that - both are correct because the market drifted. That argument rests on the - numbers, so a drift in the generator would make the guide misleading rather - than merely stale. + The aggressive TWAP benchmark comparison changes sign because the market + drifted. Price-only columns, fee-inclusive shortfall and participation must + all match the actual output; checking only frozen price numbers missed stale + cash-cost columns and the guide's attribution table. """ - @staticmethod - def _demo_rows() -> dict[tuple[str, str], dict[str, float]]: - from tradeforge.application.harness import ExecutionHarness, RunRequest - from tradeforge.infrastructure.config import load_configs, validate_required - - configs = load_configs(ROOT / "configs") - validate_required(configs) - harness = ExecutionHarness(configs) - rows = {} - for policy in ("twap", "vwap", "pov", "is_baseline"): - for style in ("passive", "aggressive"): - context = harness.run(RunRequest(policy=policy, style=style, seed=20260908)) - assert context.tca is not None - metrics = context.tca.metrics - rows[(policy, style)] = { - "fill_ratio": metrics.fill_ratio, - "maker_fill_ratio": metrics.maker_fill_ratio, - "vs_arrival_bps": metrics.cost_vs_arrival_bps, - } - return rows - - def test_the_quoted_numbers_still_hold(self): - rows = self._demo_rows() + def test_the_quoted_numbers_still_hold(self, cli_runs): + rows = demo_rows(cli_runs["demo"]) quoted = { ("twap", "passive"): (1.0, 0.963, -7.9369), ("twap", "aggressive"): (1.0, 0.051, -6.9336), } - for key, (fill, maker, is_bps) in quoted.items(): + for key, (fill, maker, arrival_bps) in quoted.items(): row = rows[key] - assert row["fill_ratio"] == pytest.approx(fill, abs=5e-4), key - assert row["maker_fill_ratio"] == pytest.approx(maker, abs=5e-4), key - assert row["vs_arrival_bps"] == pytest.approx(is_bps, abs=5e-4), key + assert float(row["fill"].rstrip("%")) / 100 == pytest.approx(fill, abs=5e-4), key + assert float(row["maker"].rstrip("%")) / 100 == pytest.approx(maker, abs=5e-4), key + assert float(row["vs_arrival_bps"]) == pytest.approx(arrival_bps, abs=5e-4), key + + @pytest.mark.parametrize("doc,n_rows", [(ROOT / "README.md", 8), (GETTING_STARTED, 2)]) + def test_documented_demo_columns_match_the_cli(self, cli_runs, doc, n_rows): + actual = demo_rows(cli_runs["demo"]) + quoted = demo_rows(quoted_block(doc, "vs_arrival_bps")) + assert len(actual) == 8 + assert list(quoted) == list(actual)[:n_rows], doc.name + expected_columns = set(next(iter(actual.values()))) + if n_rows == 2: + expected_columns.remove("participation") + for key, row in quoted.items(): + assert set(row) == expected_columns, (doc.name, key) + for column, value in row.items(): + assert float(value.rstrip("%")) == pytest.approx( + float(actual[key][column].rstrip("%")), abs=5e-5 + ), (doc.name, key, column) + + def test_documented_report_matches_the_cli(self, cli_runs): + quoted = quoted_block(GETTING_STARTED, "Cost attribution (bps, positive = worse)") + actual = cli_runs["run"].split("\nProvenance\n", 1)[0] + + def normalized_lines(text): + return [ + " ".join(line.split()) + for line in text.splitlines() + if line.strip() and not re.fullmatch(r"[-\s]+", line) + ] + + # Padding is presentation; row names, values and their order are the contract. + assert normalized_lines(quoted) == normalized_lines(actual) def test_the_guide_calls_its_table_an_excerpt(self): """The guide shows two of eight rows and omits the participation column.