docs: reconcile fee-inclusive demo tables with the public CLI

README.md

@@ -35,14 +35,14 @@ SYNTHETIC DATA - deterministic generator, not a real venue.
 
 policy       style       fill     maker   vs_arrival_bps  vs_vwap_bps  is_bps   participation
 -----------  ----------  -------  ------  --------------  -----------  -------  -------------
-twap         passive     100.00%  96.3%   -7.9369         -0.8977      -7.9369  2.198%
-twap         aggressive  100.00%  5.1%    -6.9336         0.1063       -6.9336  2.198%
-vwap         passive     100.00%  96.3%   -7.9001         -0.8608      -7.9001  2.198%
-vwap         aggressive  100.00%  5.3%    -6.8957         0.1442       -6.8957  2.198%
-pov          passive     100.00%  100.0%  -5.0705         1.9707       -5.0705  2.198%
-pov          aggressive  100.00%  3.4%    -3.3543         3.6881       -3.3543  2.198%
-is_baseline  passive     100.00%  97.4%   -7.1082         -0.0685      -7.1082  2.198%
-is_baseline  aggressive  100.00%  4.8%    -6.1079         0.9325       -6.1079  2.198%
+twap         passive     100.00%  96.3%   -7.9369         -0.8977      -8.0246  2.198%
+twap         aggressive  100.00%  5.1%    -6.9336         0.1063       -5.4720  2.198%
+vwap         passive     100.00%  96.3%   -7.9001         -0.8608      -7.9874  2.198%
+vwap         aggressive  100.00%  5.3%    -6.8957         0.1442       -5.4376  2.198%
+pov          passive     100.00%  100.0%  -5.0705         1.9707       -5.2204  2.198%
+pov          aggressive  100.00%  3.4%    -3.3543         3.6881       -1.8626  2.198%
+is_baseline  passive     100.00%  97.4%   -7.1082         -0.0685      -7.2143  2.198%
+is_baseline  aggressive  100.00%  4.8%    -6.1079         0.9325       -4.6396  2.198%
 ```
 
 **Read the first two cost columns together.** Passive TWAP beats both benchmarks;
@@ -52,6 +52,10 @@ market move between arrival and the volume-weighted price during the order windo
 The platform ships with `sql/02_benchmark_disagreement.sql` and presents both
 cost columns together.
 
+`vs_arrival_bps` and `vs_vwap_bps` compare execution prices. `is_bps` includes
+explicit fees, net rebates and any unfilled opportunity cost. The quoted tables
+and report are [checked against the public CLI](tests/unit/test_documentation.py).
+
 Participation divides filled quantity by market volume observed inside the parent
 execution window. Warm-up and later markout events do not enter that denominator.
 

docs/guides/getting-started.md

@@ -73,39 +73,41 @@ python -m tradeforge.interfaces.cli run --policy twap --style passive
 ```
 
 ```
-metric                          value
-------------------------------  -----------------------------
-policy                          twap
-side                            BUY
-fill_ratio                      1
-participation_rate              0.02198
-cost_vs_arrival_bps             -7.9369
-cost_vs_vwap_bps                -0.8977
-implementation_shortfall_bps    -8.0246
-price_shortfall_bps             -7.9369
-maker_fill_ratio                0.9634
-queue_mode                      APPROXIMATE
-latency_basis                   scenario
-counterfactual_mode             replay_approximation
+metric                        value
+----------------------------  --------------------
+cost_vs_arrival_bps           -7.9369
+cost_vs_twap_bps              -0.8667
+cost_vs_vwap_bps              -0.8977
+counterfactual_mode           replay_approximation
+fill_ratio                    1
+implementation_shortfall_bps  -8.0246
+latency_basis                 scenario
+maker_fill_ratio              0.9634
+participation_rate            0.022
+policy                        twap
+price_shortfall_bps           -7.9369
+queue_mode                    APPROXIMATE
+side                          BUY
 
 Benchmarks (ticks)
-benchmark        value    basis
----------------  -------  ----------------------------------
-arrival_mid      10000    first mid at/after window start
-interval_vwap    9992.96  volume weighted over 8417 prints
-interval_twap    9992.93  time weighted mid, trapezoidal
-terminal_mid     9976.5   last mid inside the window
+benchmark      value      basis
+-------------  ---------  -------------------------------
+arrival_mid    10000      first mid at/after window start
+interval_vwap  9992.9602  8417 prints
+interval_twap  9992.9292  time weighted mid
+interval_mid   9992.9867  unweighted mean of mids
+terminal_mid   9976.5     last mid inside the window
 
 Cost attribution (bps, positive = worse)
-component                bps
------------------------  ---------
-is_filled                -7.9369
-spread_cost              -0.479944
-fees                     -0.0877492
-timing                   -7.45728
-residual (unexplained)   0.0880677
-opportunity_cost         -23.5
-is_total                 -7.9369
+component               bps
+----------------------  -------
+is_filled               -8.0246
+spread_cost             -0.4799
+fees                    -0.0877
+timing                  -7.4573
+residual (unexplained)  0.0003
+opportunity_cost        -23.5
+is_total                -8.0246
 
 Markouts (bps, positive = adverse)
 horizon_s  measurable  of_fills  volume_weighted  median
@@ -121,8 +123,9 @@ Four things worth noticing:
 1. `queue_mode` is `APPROXIMATE`. On L2 data the queue position is not
    observable, so the number is an estimate. The report says so at the top, not
    in a footnote.
-2. The attribution closes exactly: `is_filled` equals the sum of the four
-   components. `residual` is labelled *unexplained* and is small here (0.09 bps).
+2. Before display rounding, `is_filled` equals the sum of the four components,
+   including fees. `residual` is labelled *unexplained* and is small here
+   (0.0003 bps).
    When it is large the report says the decomposition is not describing the
    execution.
 3. `opportunity_cost` is −23.5 bps and contributes nothing, because the fill

tests/unit/test_documentation.py

@@ -9,11 +9,8 @@
 costs the reader the time to discover the docs are wrong before they can start
 looking for the real answer. The checks here are cheap and catch exactly that.
 
-**What is not checked.** The prose, the numbers in the tables, and the Python
-snippets in the guides are not executed. Verifying every snippet would mean
-maintaining a doctest harness for code that is deliberately illustrative; the
-`make demo` table is checked because it is the repository's headline claim and
-because it is cheap to check.
+The quoted demo tables and getting-started report are checked against the
+public CLI. Prose and illustrative Python snippets are not executed.
 """
 
 from __future__ import annotations
@@ -30,6 +27,7 @@
 EXAMPLES = sorted((ROOT / "examples").glob("*.py"))
 GUIDES = sorted((ROOT / "docs/guides").glob("*.md"))
 DOCS = [ROOT / "README.md", *GUIDES]
+GETTING_STARTED = ROOT / "docs/guides/getting-started.md"
 
 #: Prose that happens to contain the word "make", not a target reference.
 _PROSE = {"the", "a", "it", "them", "sure", "this", "that", "no"}
@@ -51,6 +49,53 @@ def makefile_targets() -> set[str]:
     return set(re.findall(r"^([a-z][a-z0-9-]*):", makefile, re.M))
 
 
+def quoted_block(path: Path, header: str) -> str:
+    blocks = [block for block in code_spans(path) if block.startswith("```")]
+    pattern = rf"(?<!\w){re.escape(header)}(?!\w)"
+    matches = [block.strip("`\n") for block in blocks if re.search(pattern, block)]
+    assert len(matches) == 1, f"{path.name}: expected one table containing {header!r}"
+    return matches[0]
+
+
+def demo_rows(text: str) -> dict[tuple[str, str], dict[str, str]]:
+    lines = text.splitlines()
+    start = next(i for i, line in enumerate(lines) if line.startswith("policy "))
+    columns = lines[start].split()
+    rows = {}
+    for line in lines[start + 2 :]:
+        cells = line.split()
+        if not cells:
+            break
+        assert len(cells) == len(columns), f"malformed demo row: {line!r}"
+        key = tuple(cells[:2])
+        assert key not in rows, f"duplicate demo row: {key}"
+        rows[key] = dict(zip(columns[2:], cells[2:], strict=True))
+    assert rows, "demo table has no rows"
+    return rows
+
+
+@pytest.fixture(scope="module")
+def cli_runs() -> dict[str, str]:
+    """Exercise the documented commands once, including their rendered columns."""
+    outputs = {}
+    for name, args in {
+        "demo": ["demo"],
+        "run": ["run", "--policy", "twap", "--style", "passive"],
+    }.items():
+        result = subprocess.run(
+            [sys.executable, "-m", "tradeforge.interfaces.cli", *args],
+            capture_output=True,
+            text=True,
+            cwd=ROOT,
+            env={"PYTHONPATH": str(ROOT / "src"), "PATH": "/usr/bin:/bin"},
+            timeout=240,
+            check=False,
+        )
+        assert result.returncode == 0, f"{name} failed:\n{result.stderr[-1500:]}"
+        outputs[name] = result.stdout
+    return outputs
+
+
 @pytest.fixture(scope="module")
 def example_runs() -> dict[str, subprocess.CompletedProcess]:
     """Run each example once and share the result.
@@ -176,45 +221,53 @@ def test_every_documented_module_is_runnable(self):
 class TestTheHeadlineClaim:
     """`getting-started.md` quotes the `make demo` table as the repository's point.
 
-    The guide's argument is that every strategy is about 7 bps cheaper than the
-    arrival price and about 3 bps more expensive than the interval VWAP, and that
-    both are correct because the market drifted. That argument rests on the
-    numbers, so a drift in the generator would make the guide misleading rather
-    than merely stale.
+    The aggressive TWAP benchmark comparison changes sign because the market
+    drifted. Price-only columns, fee-inclusive shortfall and participation must
+    all match the actual output; checking only frozen price numbers missed stale
+    cash-cost columns and the guide's attribution table.
     """
 
-    @staticmethod
-    def _demo_rows() -> dict[tuple[str, str], dict[str, float]]:
-        from tradeforge.application.harness import ExecutionHarness, RunRequest
-        from tradeforge.infrastructure.config import load_configs, validate_required
-
-        configs = load_configs(ROOT / "configs")
-        validate_required(configs)
-        harness = ExecutionHarness(configs)
-        rows = {}
-        for policy in ("twap", "vwap", "pov", "is_baseline"):
-            for style in ("passive", "aggressive"):
-                context = harness.run(RunRequest(policy=policy, style=style, seed=20260908))
-                assert context.tca is not None
-                metrics = context.tca.metrics
-                rows[(policy, style)] = {
-                    "fill_ratio": metrics.fill_ratio,
-                    "maker_fill_ratio": metrics.maker_fill_ratio,
-                    "vs_arrival_bps": metrics.cost_vs_arrival_bps,
-                }
-        return rows
-
-    def test_the_quoted_numbers_still_hold(self):
-        rows = self._demo_rows()
+    def test_the_quoted_numbers_still_hold(self, cli_runs):
+        rows = demo_rows(cli_runs["demo"])
         quoted = {
             ("twap", "passive"): (1.0, 0.963, -7.9369),
             ("twap", "aggressive"): (1.0, 0.051, -6.9336),
         }
-        for key, (fill, maker, is_bps) in quoted.items():
+        for key, (fill, maker, arrival_bps) in quoted.items():
             row = rows[key]
-            assert row["fill_ratio"] == pytest.approx(fill, abs=5e-4), key
-            assert row["maker_fill_ratio"] == pytest.approx(maker, abs=5e-4), key
-            assert row["vs_arrival_bps"] == pytest.approx(is_bps, abs=5e-4), key
+            assert float(row["fill"].rstrip("%")) / 100 == pytest.approx(fill, abs=5e-4), key
+            assert float(row["maker"].rstrip("%")) / 100 == pytest.approx(maker, abs=5e-4), key
+            assert float(row["vs_arrival_bps"]) == pytest.approx(arrival_bps, abs=5e-4), key
+
+    @pytest.mark.parametrize("doc,n_rows", [(ROOT / "README.md", 8), (GETTING_STARTED, 2)])
+    def test_documented_demo_columns_match_the_cli(self, cli_runs, doc, n_rows):
+        actual = demo_rows(cli_runs["demo"])
+        quoted = demo_rows(quoted_block(doc, "vs_arrival_bps"))
+        assert len(actual) == 8
+        assert list(quoted) == list(actual)[:n_rows], doc.name
+        expected_columns = set(next(iter(actual.values())))
+        if n_rows == 2:
+            expected_columns.remove("participation")
+        for key, row in quoted.items():
+            assert set(row) == expected_columns, (doc.name, key)
+            for column, value in row.items():
+                assert float(value.rstrip("%")) == pytest.approx(
+                    float(actual[key][column].rstrip("%")), abs=5e-5
+                ), (doc.name, key, column)
+
+    def test_documented_report_matches_the_cli(self, cli_runs):
+        quoted = quoted_block(GETTING_STARTED, "Cost attribution (bps, positive = worse)")
+        actual = cli_runs["run"].split("\nProvenance\n", 1)[0]
+
+        def normalized_lines(text):
+            return [
+                " ".join(line.split())
+                for line in text.splitlines()
+                if line.strip() and not re.fullmatch(r"[-\s]+", line)
+            ]
+
+        # Padding is presentation; row names, values and their order are the contract.
+        assert normalized_lines(quoted) == normalized_lines(actual)
 
     def test_the_guide_calls_its_table_an_excerpt(self):
         """The guide shows two of eight rows and omits the participation column.