From 129335fdedc550ceed2911a8ea15775a3327e22e Mon Sep 17 00:00:00 2001 From: RLungWu Date: Wed, 26 Aug 2026 18:42:46 +0800 Subject: [PATCH 1/2] docs(m2): record comparison value semantics --- .../ADR-0004-comparison-value-semantics.md | 122 +++++++++++ docs/milestones/M2-local-paired-comparison.md | 196 ++++++++++++++++++ 2 files changed, 318 insertions(+) create mode 100644 docs/adr/ADR-0004-comparison-value-semantics.md create mode 100644 docs/milestones/M2-local-paired-comparison.md diff --git a/docs/adr/ADR-0004-comparison-value-semantics.md b/docs/adr/ADR-0004-comparison-value-semantics.md new file mode 100644 index 0000000..3457456 --- /dev/null +++ b/docs/adr/ADR-0004-comparison-value-semantics.md @@ -0,0 +1,122 @@ +# ADR-0004: Define Comparison Value Semantics and Evidence Requirements + +## Status + +Accepted + +## Context + +RunGuard compares a baseline experiment with a candidate experiment using +canonical `Run` objects. A run can contain both metric history and a scalar +`summary_metrics` value. These values do not necessarily have the same +meaning: a source system may report a summary as the latest value, the best +value, or another source-defined aggregate. + +The comparison engine also needs to define what happens when runs cannot be +paired cleanly. Silent value selection, dictionary overwrites, or silently +discarded runs could make an experiment appear more stable than its evidence +supports. + +## Decision + +RunGuard will separate the storage of metric observations from the policy used +to select a comparison value. + +### Comparison value + +`summary_metrics` is treated as a source-reported scalar observation. It is not +assumed to represent a best checkpoint or a final checkpoint. + +M2 will compare the requested metric from `summary_metrics` only. If the +metric is absent or its value is missing, the comparison fails explicitly. +There is no automatic fallback from summary metrics to metric history. + +Best-checkpoint and final-checkpoint selection are separate policies and will +be introduced only when their semantics and configuration are defined. + +### Difference and metric direction + +The comparison result will preserve the raw difference: + +```text +raw_difference = candidate - baseline +``` + +It will also calculate a direction-aware improvement difference. A positive +improvement means that the candidate is better: + +```text +higher-is-better: improvement = candidate - baseline +lower-is-better: improvement = baseline - candidate +``` + +Both values are retained so that interpretation does not overwrite the +observed data. + +### Pairing and validation + +Runs are paired by seed. A comparison fails explicitly when: + +* a selected run has no seed; +* the baseline and candidate seed sets do not match; +* a selected run lacks the requested summary metric or has a missing value; +* more than one run has the same variant and seed. + +The comparison engine must not silently discard runs or overwrite duplicate +pairing keys. Pair results are ordered by seed for deterministic output. + +### Evidence and provenance + +Every pair result must retain its seed, baseline and candidate run IDs, both +metric values, the raw difference, and the direction-aware improvement. The +aggregate result must retain the group, variants, metric, direction, and value +source. + +## Alternatives Considered + +**Always use the final history value.** Rejected for M2 because different +sources may have different step semantics, and some local formats do not +contain history. Defining final-checkpoint behavior is deferred to an explicit +policy. + +**Always use the best history value.** Rejected for M2 because it requires a +metric direction and introduces checkpoint-selection behavior that can make +results optimistic. It also belongs to a later policy layer. + +**Try summary, then final, then best as fallbacks.** Rejected because the +selection rule would be hidden in the comparison engine and could produce +different meanings for apparently identical inputs. + +**Allow incomplete pairs and compare the available runs.** Rejected as the +default because changing the pair denominator silently can turn missing data +into apparent evidence. An explicit incomplete-pair policy may be added later. + +**Store only one signed difference.** Rejected because lower-is-better metrics +would make it difficult to distinguish the observed candidate-minus-baseline +value from the interpreted improvement. + +## Consequences + +Positive: + +* Comparison values have an explicit and inspectable source. +* Best and final checkpoint semantics are not inferred from ambiguous source + data. +* Positive-pair rate has one consistent interpretation for both metric + directions. +* Invalid or ambiguous pairing is visible instead of being hidden by input + order or dictionary behavior. +* Results can be traced back to the exact source runs. + +Negative: + +* M2 may reject data that could be useful for exploratory analysis. +* Users must understand that a source-reported summary is not necessarily a + best or final checkpoint. +* Structured results contain more fields than a list of numeric differences. +* Future checkpoint policies will need their own configuration and tests. + +## Related Documents + +* [M2 — Local Paired Comparison](../milestones/M2-local-paired-comparison.md) +* [ADR-0001: Use a Modular Monolith Architecture](ADR-0001-modular-monolith.md) diff --git a/docs/milestones/M2-local-paired-comparison.md b/docs/milestones/M2-local-paired-comparison.md new file mode 100644 index 0000000..e6427b5 --- /dev/null +++ b/docs/milestones/M2-local-paired-comparison.md @@ -0,0 +1,196 @@ +# M2 — Local Paired Comparison + +## Status + +Active + +## Goal + +Compare a baseline and candidate experiment using canonical local data and +paired seeds, while preserving enough detail to explain how every aggregate +value was produced. + +## User Value + +An ML engineer can compare two local experiment variants without relying on +input ordering or silently dropping runs. The result reports both aggregate +statistics and the source runs that produced each pair. + +## Context + +M1 introduced `ExperimentGroup` and the canonical `Run` model. The existing +M0 comparison code still reads the temporary flat JSON shape directly and +returns only a list of numeric differences. M2 moves comparison to the +canonical model and adds an explicit result model. + +The canonical model stores summary metrics separately from metric history. A +summary metric is treated as a scalar value reported by the source system. It +is not assumed to represent the best checkpoint or the final checkpoint. +Selecting a best or final checkpoint from history is deferred to a later +checkpoint-selection policy. + +## Scope + +M2 includes: + +* baseline and candidate selection by `variant`; +* pairing by `seed`; +* missing-pair detection; +* comparison of source-reported `summary_metrics`; +* mean paired difference; +* median paired difference; +* positive-pair rate; +* explicit higher-is-better and lower-is-better direction; +* structured comparison and pair results; +* source run IDs in every pair result; +* deterministic ordering; +* human-readable CLI output; +* synthetic scenarios for stable improvement, mixed outcomes, single-seed + improvement, and invalid pairing. + +## Non-Goals + +M2 does not include: + +* best-checkpoint selection; +* final-checkpoint selection; +* automatic fallback from summary metrics to history; +* confidence intervals; +* outlier influence analysis; +* W&B or other remote ingestion; +* release policies; +* causal interpretation; +* incomplete-pair analysis that silently excludes runs. + +## Comparison Decisions + +| Topic | M2 decision | Reason | +|---|---|---| +| Input boundary | Comparison accepts an `ExperimentGroup`, not raw JSON | Keeps source ingestion separate from analysis and allows all local formats to converge first | +| Value source | Use `Run.summary_metrics[metric]` only | The source-provided scalar is available for M0 and M1 local formats without inventing checkpoint semantics | +| Summary semantics | Preserve the value as `source-reported summary`; do not call it best or final | A source may define its summary as latest, best, or another aggregate | +| Missing metric/value | Reject the comparison with an explicit error | Avoid changing the denominator or silently producing partial evidence | +| Missing seed | A run without a seed cannot participate in pairing and is reported explicitly | Pairing requires an unambiguous key | +| Missing pair | Reject the comparison and report the unmatched seeds | A paired comparison is invalid when the baseline and candidate seed sets differ | +| Duplicate variant and seed | Reject the comparison and report the conflicting run IDs | Dictionary-style overwriting would make results input-order dependent | +| Raw difference | Preserve `candidate - baseline` | Keeps the observed data unchanged | +| Improvement difference | Derive a direction-aware value where positive means candidate is better | Makes positive-pair rate interpretable for both metric directions | +| Positive pair | Count pairs with `improvement_difference > 0`; ties are not positive | Defines a deterministic and conservative interpretation of improvement | +| Ordering | Sort pair results by seed | Makes serialized results and CLI output deterministic | +| Provenance | Store group ID, variants, seed, both run IDs, both values, and both differences | Every aggregate should be traceable to source runs | + +## Result Shape + +The implementation should expose a structured result containing: + +* metric name and direction; +* value source (`summary`); +* baseline and candidate variant names; +* group ID; +* ordered pair records; +* mean difference; +* median difference; +* positive-pair count and rate; +* any comparison validation details needed by the CLI. + +Each pair record should contain: + +* seed; +* baseline run ID; +* candidate run ID; +* baseline metric value; +* candidate metric value; +* raw difference; +* direction-aware improvement difference. + +## Error Handling + +Comparison errors must identify the violated assumption and, when possible, +the affected seeds or run IDs. The implementation must not silently discard +runs, overwrite duplicate keys, or fall back from summary metrics to history. + +## Synthetic Scenarios + +The fixtures and tests should cover at least: + +* stable improvement across all paired seeds; +* mixed positive and negative pairs; +* a single-seed improvement with otherwise no evidence of stable improvement; +* lower-is-better metrics; +* missing candidate seeds; +* missing summary metrics or `None` values; +* duplicate variant and seed records; +* deterministic pair ordering independent of input order. + +## Success Criteria + +* Canonical local input can be compared without importing a source adapter in + the comparison module. +* Stable improvement scenarios produce the expected mean, median, and + positive-pair rate. +* A single positive seed is visible in the pair results and is not described + as stable improvement by the M2 output. +* Higher-is-better and lower-is-better metrics use the same positive-means- + improvement interpretation. +* Missing pairs, missing seeds, missing values, and duplicate pairing keys + produce explicit errors. +* Every pair result identifies both source runs. +* Pair and aggregate results are deterministic. +* `pytest`, `ruff check .`, `mypy src`, and pre-commit checks pass. + +## Risks + +### Ambiguous source summary semantics + +A source-reported summary may mean best, final, or another aggregation. M2 +documents this limitation and preserves `value_source=summary`; explicit best +and final policies remain future work. + +### Overly strict incomplete-data handling + +Rejecting incomplete comparisons may be inconvenient for exploratory work. M2 +chooses strict behavior to protect reproducibility. A later explicit policy +may allow incomplete pairs without changing the default behavior. + +### Confusing raw and direction-aware differences + +Reporting only one signed difference can make lower-is-better metrics hard to +interpret. M2 stores both the raw difference and the direction-aware +improvement difference. + +## Open Questions + +The following are intentionally deferred: + +* How should best-checkpoint selection be configured? +* How should final-checkpoint selection handle missing or non-monotonic steps? +* Should incomplete pairing become an opt-in analysis policy? +* Should a machine-readable JSON CLI format be added alongside human output? + +## Planned Issues + +* Define comparison result and pair models. +* Implement canonical-model pairing and validation. +* Implement mean, median, and positive-pair aggregation. +* Add synthetic comparison fixtures and tests. +* Update the CLI to load canonical local input and display the structured + result. + +## Release Gate + +M2 is ready to complete when: + +* all success criteria are satisfied; +* the canonical comparison path is covered by unit and integration tests; +* known limitations and deferred checkpoint decisions are documented; +* `CHANGELOG.md` contains the `v0.0.3` entry; +* the retrospective is complete; +* the `v0.0.3` release is tagged after the changes are merged to `main`. + +## Decisions Made During Implementation + +To be updated as implementation details are finalized. + +## Retrospective + +Not started. From 40b601d9ea77a055428ae8a5039673b4faa63649 Mon Sep 17 00:00:00 2001 From: RLungWu Date: Thu, 27 Aug 2026 13:45:12 +0800 Subject: [PATCH 2/2] feat(comparison): add structured comparison models --- src/runguard/comparison/models.py | 39 ++++++++++++++++++++ tests/unit/test_comparison_models.py | 54 ++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 src/runguard/comparison/models.py create mode 100644 tests/unit/test_comparison_models.py diff --git a/src/runguard/comparison/models.py b/src/runguard/comparison/models.py new file mode 100644 index 0000000..bfdc869 --- /dev/null +++ b/src/runguard/comparison/models.py @@ -0,0 +1,39 @@ +from enum import StrEnum +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class MetricDirection(StrEnum): + HIGHER_IS_BETTER = "higher-is-better" + LOWER_IS_BETTER = "lower-is-better" + + +class ComparisonPair(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + seed: int + baseline_run_id: str = Field(min_length=1) + candidate_run_id: str = Field(min_length=1) + baseline_value: float + candidate_value: float + raw_difference: float + improvement_difference: float + + +class ComparisonResult(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + group_id: str = Field(min_length=1) + baseline_variant: str = Field(min_length=1) + candidate_variant: str = Field(min_length=1) + metric: str = Field(min_length=1) + value_source: Literal["summary"] = "summary" + direction: MetricDirection + pairs: list[ComparisonPair] + mean_difference: float + median_difference: float + mean_improvement: float + median_improvement: float + positive_pair_count: int + positive_pair_rate: float diff --git a/tests/unit/test_comparison_models.py b/tests/unit/test_comparison_models.py new file mode 100644 index 0000000..d22979f --- /dev/null +++ b/tests/unit/test_comparison_models.py @@ -0,0 +1,54 @@ +import pytest + +from runguard.comparison.models import ( + ComparisonPair, + ComparisonResult, + MetricDirection, +) + + +def test_comparison_pair_preserves_source_run_ids_and_differences() -> None: + pair = ComparisonPair( + seed=1, + baseline_run_id="baseline-seed-1", + candidate_run_id="candidate-seed-1", + baseline_value=0.8, + candidate_value=0.82, + raw_difference=0.02, + improvement_difference=0.02, + ) + + assert pair.seed == 1 + assert pair.baseline_run_id == "baseline-seed-1" + assert pair.candidate_run_id == "candidate-seed-1" + assert pair.raw_difference == pytest.approx(0.02) + assert pair.improvement_difference == pytest.approx(0.02) + + +def test_comparison_result_defaults_to_source_reported_summary() -> None: + pair = ComparisonPair( + seed=1, + baseline_run_id="baseline-seed-1", + candidate_run_id="candidate-seed-1", + baseline_value=0.8, + candidate_value=0.82, + raw_difference=0.02, + improvement_difference=0.02, + ) + result = ComparisonResult( + group_id="experiment-1", + baseline_variant="baseline", + candidate_variant="candidate", + metric="val_f1", + direction=MetricDirection.HIGHER_IS_BETTER, + pairs=[pair], + mean_difference=0.02, + median_difference=0.02, + mean_improvement=0.02, + median_improvement=0.02, + positive_pair_count=1, + positive_pair_rate=1.0, + ) + + assert result.value_source == "summary" + assert result.direction is MetricDirection.HIGHER_IS_BETTER