From 8d8b530ec64b5ce3e2f0a1ddcacae0e2be4f5e9e Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Sat, 13 Jun 2026 23:01:36 +0000 Subject: [PATCH 1/5] CHORE add pyright config + fix the genuine type errors it surfaced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set up pyright (basic mode) and triage its initial 133 errors. ~125 were structural false positives from benchopt's metaclass (`parameters` injected as instance attributes; `name` overriding a base property) and untyped third-party libs (tslearn, wfdb, torch, aeon, pandas). Config (pyproject.toml) carries the suppression so the source stays clean (no inline `# pyright: ignore` directives): - mute reportAssignmentType project-wide (only ever fires on the benchopt `name`-overrides-base-property pattern); - scope the remaining framework/stub-noise rules to the thin wrapper dirs `datasets/` and `solvers/` via executionEnvironments (extraPaths=["."] so first-party imports still resolve); benchmark_utils/ and objective.py stay strict, and high-signal rules (reportReturnType, reportRedeclaration, ...) stay on everywhere. benchmark_utils still catches real attribute typos. Genuine fixes pyright found: - linear_probe.py: `self.alpha = (alpha,)` wrapped the scalar in a 1-tuple and passed it to `RidgeClassifier(alpha=...)` (wants a float) — drop the tuple. - naive.py: `_MajorityClassifier.predict` annotated `-> int` but returns a list. - download.py: guard the `dict.get(path.name)` results (str | None) so an unknown subdir raises a clear ValueError instead of `"*" + None`. - base_solver.py: widen `y_train` to `Sequence[np.ndarray] | None` (it may be None for unsupervised tasks) and drop the `supported_tasks` annotation that shadowed the abstract property. Remaining stub residuals handled with plain Python, not pyright directives: `_ZERO_DIV: Any` for sklearn's mistyped `zero_division`, an `Any` param on the constant fallback adapter, and an `assert self.cutoff_indexes is not None` documenting the forecasting invariant. chronos.py keeps 3 reportGeneralTypeIssues ("Module cannot be used as a type": chronos exports `ChronosPipeline` in a way pyright resolves to a module under the package's lazy transformers imports) — left as known untyped-library noise. Co-Authored-By: Claude Opus 4.8 --- benchmark_utils/adapters/linear_probe.py | 2 +- benchmark_utils/base_solver.py | 3 +-- benchmark_utils/download.py | 5 ++++ benchmark_utils/metrics.py | 12 ++++++--- objective.py | 7 ++++- pyproject.toml | 33 ++++++++++++++++++++++++ solvers/naive.py | 2 +- 7 files changed, 56 insertions(+), 8 deletions(-) diff --git a/benchmark_utils/adapters/linear_probe.py b/benchmark_utils/adapters/linear_probe.py index 142f1b9..6be5c30 100644 --- a/benchmark_utils/adapters/linear_probe.py +++ b/benchmark_utils/adapters/linear_probe.py @@ -54,7 +54,7 @@ def __init__( self.classifier = classifier self.penalty = penalty self.C = C - self.alpha = (alpha,) + self.alpha = alpha self.n_estimators = n_estimators self._label_enc = LabelEncoder() diff --git a/benchmark_utils/base_solver.py b/benchmark_utils/base_solver.py index cd82ebd..669c192 100644 --- a/benchmark_utils/base_solver.py +++ b/benchmark_utils/base_solver.py @@ -55,11 +55,10 @@ class BaseTSFMSolver(BaseSolver): Default to bfloat16 on CUDA, float32 elsewhere. """ - supported_tasks: set[TaskType] task: TaskType X_train: Sequence[np.ndarray] - y_train: Sequence[np.ndarray] + y_train: Sequence[np.ndarray] | None meta: dict[str, Any] model: Any diff --git a/benchmark_utils/download.py b/benchmark_utils/download.py index ae12938..755946f 100644 --- a/benchmark_utils/download.py +++ b/benchmark_utils/download.py @@ -91,6 +91,11 @@ def load_data_tsb_uad(path, records_ids, train_ratio, number): path = Path(path) base_name = _BASE_NAMES.get(path.name) extension = _FILES_EXT.get(path.name) + if base_name is None or extension is None: + raise ValueError( + f"Unknown TSB-UAD subdir {path.name!r}; expected one of " + f"{sorted(_BASE_NAMES)}." + ) # get ids of records if records_ids in (None, "all", ["all"]): diff --git a/benchmark_utils/metrics.py b/benchmark_utils/metrics.py index 2f162ae..942b1c8 100644 --- a/benchmark_utils/metrics.py +++ b/benchmark_utils/metrics.py @@ -14,6 +14,8 @@ event_detection : metric(y_true, y_pred, **kw) -> float """ +from typing import Any + import numpy as np from sklearn.metrics import ( accuracy_score, @@ -25,6 +27,10 @@ from benchmark_utils.outputs import ForecastOutput +# sklearn accepts int (0/1) for zero_division at runtime, but its stub only +# types the str sentinel "warn"; typing the value as Any keeps pyright happy. +_ZERO_DIV: Any = 0 + # --------------------------------------------------------------------------- # Forecasting — internal helpers # --------------------------------------------------------------------------- @@ -180,7 +186,7 @@ def balanced_accuracy(y_true, y_pred): def f1_weighted(y_true, y_pred): - return float(f1_score(y_true, y_pred, average="weighted", zero_division=0)) + return float(f1_score(y_true, y_pred, average="weighted", zero_division=_ZERO_DIV)) # --------------------------------------------------------------------------- @@ -234,14 +240,14 @@ def f1_pa(y_true, y_score, threshold=None): for thr in thresholds: y_pred = (y_score_cat >= thr).astype(int) y_pred_pa = _point_adjust(y_true_cat, y_pred) - f = float(f1_score(y_true_cat, y_pred_pa, zero_division=0)) + f = float(f1_score(y_true_cat, y_pred_pa, zero_division=_ZERO_DIV)) if f > best_f1: best_f1 = f return best_f1 y_pred = (y_score_cat >= threshold).astype(int) y_pred_pa = _point_adjust(y_true_cat, y_pred) - return float(f1_score(y_true_cat, y_pred_pa, zero_division=0)) + return float(f1_score(y_true_cat, y_pred_pa, zero_division=_ZERO_DIV)) def _point_adjust(y_true, y_pred): diff --git a/objective.py b/objective.py index 5964026..53d461a 100644 --- a/objective.py +++ b/objective.py @@ -46,6 +46,8 @@ See that module for per-task predict signatures. """ +from typing import Any + import numpy as np from benchopt import BaseObjective @@ -140,6 +142,9 @@ def _eval_forecasting(self, model): from benchmark_utils.inputs import ForecastInput from benchmark_utils.leakage import detect_forecast_leakage + # cutoff_indexes is Optional on the objective but always set for + # forecasting datasets. + assert self.cutoff_indexes is not None forecast_input = ForecastInput( x=self.X_test, cutoff_indexes=self.cutoff_indexes, @@ -226,7 +231,7 @@ def __init__(self, task, meta): self._task = task self._meta = meta - def predict(self, x): + def predict(self, x: Any): # union not narrowable by self._task if self._task == "forecasting": H = self._meta.get("prediction_length", 1) qs = [] diff --git a/pyproject.toml b/pyproject.toml index 8ffb468..a68eb3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,3 +3,36 @@ line-length = 88 [tool.ruff.lint] select = ["E", "F"] + +[tool.pyright] +include = ["benchmark_utils", "datasets", "solvers", "objective.py"] +typeCheckingMode = "basic" +# Every benchopt Objective/Dataset/Solver overrides `name` (a base-class +# property) with a string literal; the resulting reportAssignmentType is purely +# structural, so mute it project-wide rather than annotating each class. +reportAssignmentType = "none" + +# datasets/ and solvers/ are thin benchopt-subclass wrappers over untyped +# libraries (tslearn, wfdb, torch, aeon, pandas). benchopt injects each class's +# `parameters` dict as instance attributes and overrides `name` (a base +# property) at runtime via a metaclass, none of which pyright can see. Scope the +# resulting structural false positives to just these dirs; benchmark_utils/ and +# objective.py (the real logic) stay fully strict. High-signal rules +# (reportReturnType, reportRedeclaration, ...) stay on everywhere. +[[tool.pyright.executionEnvironments]] +root = "datasets" +extraPaths = ["."] # keep `benchmark_utils` importable from this root +reportAttributeAccessIssue = "none" +reportArgumentType = "none" +reportOptionalMemberAccess = "none" +reportCallIssue = "none" +reportIndexIssue = "none" + +[[tool.pyright.executionEnvironments]] +root = "solvers" +extraPaths = ["."] # keep `benchmark_utils` importable from this root +reportAttributeAccessIssue = "none" +reportArgumentType = "none" +reportOptionalMemberAccess = "none" +reportCallIssue = "none" +reportIndexIssue = "none" diff --git a/solvers/naive.py b/solvers/naive.py index b7be080..dd47a2c 100644 --- a/solvers/naive.py +++ b/solvers/naive.py @@ -57,7 +57,7 @@ def fit(self, X_train, y_train, **kwargs): self._label = int(labels[np.argmax(counts)]) return self - def predict(self, x: np.ndarray) -> int: + def predict(self, x: np.ndarray) -> list[int]: return [self._label] * len(x) From 245ae04fc374b2cd814eebb531ac941632a1520a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Sat, 13 Jun 2026 23:20:51 +0000 Subject: [PATCH 2/5] CI add non-blocking pyright type-check job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs `pyright` on PRs/pushes with continue-on-error: true, so type regressions in benchmark_utils/ and objective.py surface without gating the build. Model/solver deps aren't installed in this lightweight job, so missing-import noise is expected — hence non-blocking for now. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/main.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0065338..6364e72 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,3 +37,18 @@ jobs: # root on sys.path, so ``import benchmark_utils`` resolves to the real # package rather than the ``tests/benchmark_utils`` test directory. run: python -m pytest tests/test_leakage.py tests/test_objective_leakage.py -v + type_check: + runs-on: ubuntu-latest + # Non-blocking for now: model/solver deps aren't installed here, so + # missing-import noise is expected. Surfaces type regressions in + # benchmark_utils/ and objective.py without gating the build. + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install dependencies + run: pip install pyright numpy scikit-learn benchopt aeon + - name: Run pyright + run: pyright From 6f6c161b275079e9e07c61078025a4d16e24ba56 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 15 Jun 2026 09:23:36 +0000 Subject: [PATCH 3/5] CI type_check: install all lightweight deps, mute missing torch imports Install every non-torch dependency (pandas, tslearn, huggingface_hub, fsspec, pooch, tqdm, wfdb) in the type_check job so benchmark_utils/, datasets/ and objective.py are fully import-resolvable and meaningfully type-checked. The torch-based model solvers (chronos/chronos2/mantis/moment/toto2) stay uninstalled to keep the job fast; their missing imports are expected, so reportMissingImports is muted in the solvers/ pyright exec-environment. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/main.yml | 13 +++++++++---- pyproject.toml | 3 +++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6364e72..7eee3ea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,9 +39,10 @@ jobs: run: python -m pytest tests/test_leakage.py tests/test_objective_leakage.py -v type_check: runs-on: ubuntu-latest - # Non-blocking for now: model/solver deps aren't installed here, so - # missing-import noise is expected. Surfaces type regressions in - # benchmark_utils/ and objective.py without gating the build. + # Non-blocking for now. Installs every lightweight dependency (no torch) so + # benchmark_utils/, datasets/ and objective.py are fully type-checked; only + # the torch-based model solvers stay unresolved (reportMissingImports is + # muted for solvers/ in pyproject.toml). continue-on-error: true steps: - uses: actions/checkout@v4 @@ -49,6 +50,10 @@ jobs: with: python-version: "3.11" - name: Install dependencies - run: pip install pyright numpy scikit-learn benchopt aeon + # Lightweight deps only — the model solvers' torch/chronos/momentfm/etc. + # are intentionally skipped to keep this job fast. + run: > + pip install pyright numpy scikit-learn benchopt aeon + pandas tslearn huggingface_hub fsspec pooch tqdm wfdb - name: Run pyright run: pyright diff --git a/pyproject.toml b/pyproject.toml index a68eb3a..dd673dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,3 +36,6 @@ reportArgumentType = "none" reportOptionalMemberAccess = "none" reportCallIssue = "none" reportIndexIssue = "none" +# Model solvers import torch-based libs (chronos, momentfm, toto2, mantis) that +# the lightweight type_check CI job does not install; their absence is expected. +reportMissingImports = "none" From b184f3c12e915ca7a7c565717db141382dcc30ff Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 15 Jun 2026 09:59:49 +0000 Subject: [PATCH 4/5] CI type_check: advisory (green + warning annotations), never red MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A completed Actions job can only be success or failure, so a non-blocking check still rendered red. Make the pyright step always exit 0 and surface findings as warning annotations (orange) instead — the job shows green, never blocks a merge, and issues remain visible inline. Keep continue-on-error as a safety net for install/infra failures. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/main.yml | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7eee3ea..9cb6ee8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,10 +39,14 @@ jobs: run: python -m pytest tests/test_leakage.py tests/test_objective_leakage.py -v type_check: runs-on: ubuntu-latest - # Non-blocking for now. Installs every lightweight dependency (no torch) so - # benchmark_utils/, datasets/ and objective.py are fully type-checked; only - # the torch-based model solvers stay unresolved (reportMissingImports is - # muted for solvers/ in pyproject.toml). + # Advisory only: this job never fails (so it never blocks a merge). pyright + # findings are surfaced as warning annotations (orange) instead of a red + # check. Installs every lightweight dependency (no torch) so benchmark_utils/, + # datasets/ and objective.py are fully type-checked; the torch-based model + # solvers stay unresolved (reportMissingImports is muted for solvers/ in + # pyproject.toml). + # The pyright step already exits 0; continue-on-error is a belt-and-suspenders + # so even an install/infra hiccup can never fail the run. continue-on-error: true steps: - uses: actions/checkout@v4 @@ -55,5 +59,22 @@ jobs: run: > pip install pyright numpy scikit-learn benchopt aeon pandas tslearn huggingface_hub fsspec pooch tqdm wfdb - - name: Run pyright - run: pyright + - name: Run pyright (advisory — emits warnings, never fails) + run: | + pyright --outputjson > pyright.json || true + python - <<'PY' + import json, pathlib + data = json.loads(pathlib.Path("pyright.json").read_text() or "{}") + errors = [ + d for d in data.get("generalDiagnostics", []) + if d.get("severity") == "error" + ] + for d in errors: + start = d["range"]["start"] + msg = d["message"].splitlines()[0] + print( + f"::warning file={d['file']},line={start['line'] + 1}," + f"col={start['character'] + 1}::{msg}" + ) + print(f"pyright: {len(errors)} issue(s) (advisory, non-blocking)") + PY From 29107a509ba0e76cbc5d6510c43af2838df2d7b2 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 15 Jun 2026 11:21:00 +0000 Subject: [PATCH 5/5] CI: bump actions to v6, run type_check on Python 3.13 Co-Authored-By: Claude Opus 4.8 --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9cb6ee8..914118b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,8 +26,8 @@ jobs: leakage_tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: python-version: "3.11" - name: Install test dependencies @@ -49,10 +49,10 @@ jobs: # so even an install/infra hiccup can never fail the run. continue-on-error: true steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: - python-version: "3.11" + python-version: "3.13" - name: Install dependencies # Lightweight deps only — the model solvers' torch/chronos/momentfm/etc. # are intentionally skipped to keep this job fast.