diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3407a35..86f7dcc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,3 +37,44 @@ jobs: # root on sys.path, so ``import benchmark_utils`` resolves to the real # package rather than the ``tests/benchmark_utils`` test directory. run: python -m pytest -v + type_check: + runs-on: ubuntu-latest + # Advisory only: this job never fails (so it never blocks a merge). pyright + # findings are surfaced as warning annotations (orange) instead of a red + # check. Installs every lightweight dependency (no torch) so benchmark_utils/, + # datasets/ and objective.py are fully type-checked; the torch-based model + # solvers stay unresolved (reportMissingImports is muted for solvers/ in + # pyproject.toml). + # The pyright step already exits 0; continue-on-error is a belt-and-suspenders + # so even an install/infra hiccup can never fail the run. + continue-on-error: true + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Install dependencies + # Lightweight deps only — the model solvers' torch/chronos/momentfm/etc. + # are intentionally skipped to keep this job fast. + run: > + pip install pyright numpy scikit-learn benchopt aeon + pandas tslearn huggingface_hub fsspec pooch tqdm wfdb + - name: Run pyright (advisory — emits warnings, never fails) + run: | + pyright --outputjson > pyright.json || true + python - <<'PY' + import json, pathlib + data = json.loads(pathlib.Path("pyright.json").read_text() or "{}") + errors = [ + d for d in data.get("generalDiagnostics", []) + if d.get("severity") == "error" + ] + for d in errors: + start = d["range"]["start"] + msg = d["message"].splitlines()[0] + print( + f"::warning file={d['file']},line={start['line'] + 1}," + f"col={start['character'] + 1}::{msg}" + ) + print(f"pyright: {len(errors)} issue(s) (advisory, non-blocking)") + PY diff --git a/benchmark_utils/adapters/linear_probe.py b/benchmark_utils/adapters/linear_probe.py index 66af11c..f623888 100644 --- a/benchmark_utils/adapters/linear_probe.py +++ b/benchmark_utils/adapters/linear_probe.py @@ -54,7 +54,7 @@ def __init__( self.classifier = classifier self.penalty = penalty self.C = C - self.alpha = (alpha,) + self.alpha = alpha self.n_estimators = n_estimators self._label_enc = LabelEncoder() diff --git a/benchmark_utils/base_solver.py b/benchmark_utils/base_solver.py index f6f3c66..7a38208 100644 --- a/benchmark_utils/base_solver.py +++ b/benchmark_utils/base_solver.py @@ -281,11 +281,10 @@ class BaseTSFMSolver(BaseSolver): Default to bfloat16 on CUDA, float32 elsewhere. """ - supported_tasks: set[TaskType] task: TaskType X_train: Sequence[np.ndarray] - y_train: Sequence[np.ndarray] + y_train: Sequence[np.ndarray] | None meta: dict[str, Any] model: Any diff --git a/benchmark_utils/download_pooch.py b/benchmark_utils/download_pooch.py index ae12938..755946f 100644 --- a/benchmark_utils/download_pooch.py +++ b/benchmark_utils/download_pooch.py @@ -91,6 +91,11 @@ def load_data_tsb_uad(path, records_ids, train_ratio, number): path = Path(path) base_name = _BASE_NAMES.get(path.name) extension = _FILES_EXT.get(path.name) + if base_name is None or extension is None: + raise ValueError( + f"Unknown TSB-UAD subdir {path.name!r}; expected one of " + f"{sorted(_BASE_NAMES)}." + ) # get ids of records if records_ids in (None, "all", ["all"]): diff --git a/benchmark_utils/metrics.py b/benchmark_utils/metrics.py index 4e5b8cd..cfa99bb 100644 --- a/benchmark_utils/metrics.py +++ b/benchmark_utils/metrics.py @@ -14,6 +14,8 @@ event_detection : metric(y_true, y_pred, **kw) -> float """ +from typing import Any + import numpy as np from sklearn.metrics import ( accuracy_score, @@ -25,6 +27,10 @@ from benchmark_utils.outputs import ForecastOutput +# sklearn accepts int (0/1) for zero_division at runtime, but its stub only +# types the str sentinel "warn"; typing the value as Any keeps pyright happy. +_ZERO_DIV: Any = 0 + # --------------------------------------------------------------------------- # Forecasting — internal helpers # --------------------------------------------------------------------------- @@ -180,7 +186,7 @@ def balanced_accuracy(y_true, y_pred): def f1_weighted(y_true, y_pred): - return float(f1_score(y_true, y_pred, average="weighted", zero_division=0)) + return float(f1_score(y_true, y_pred, average="weighted", zero_division=_ZERO_DIV)) # --------------------------------------------------------------------------- @@ -234,14 +240,14 @@ def f1_pa(y_true, y_score, threshold=None): for thr in thresholds: y_pred = (y_score_cat >= thr).astype(int) y_pred_pa = _point_adjust(y_true_cat, y_pred) - f = float(f1_score(y_true_cat, y_pred_pa, zero_division=0)) + f = float(f1_score(y_true_cat, y_pred_pa, zero_division=_ZERO_DIV)) if f > best_f1: best_f1 = f return best_f1 y_pred = (y_score_cat >= threshold).astype(int) y_pred_pa = _point_adjust(y_true_cat, y_pred) - return float(f1_score(y_true_cat, y_pred_pa, zero_division=0)) + return float(f1_score(y_true_cat, y_pred_pa, zero_division=_ZERO_DIV)) def _point_adjust(y_true, y_pred): diff --git a/objective.py b/objective.py index 5da735c..3bfd355 100644 --- a/objective.py +++ b/objective.py @@ -48,6 +48,8 @@ See that module for per-task predict signatures. """ +from typing import Any + import numpy as np from benchopt import BaseObjective @@ -160,6 +162,9 @@ def _eval_forecasting(self, model): from benchmark_utils.inputs import ForecastInput from benchmark_utils.leakage import detect_forecast_leakage + # cutoff_indexes is Optional on the objective but always set for + # forecasting datasets. + assert self.cutoff_indexes is not None forecast_input = ForecastInput( x=self.X_test, cutoff_indexes=self.cutoff_indexes, @@ -246,7 +251,7 @@ def __init__(self, task, meta): self._task = task self._meta = meta - def predict(self, x): + def predict(self, x: Any): # union not narrowable by self._task if self._task == "forecasting": H = self._meta.get("prediction_length", 1) qs = [] diff --git a/pyproject.toml b/pyproject.toml index 8ffb468..dd673dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,3 +3,39 @@ line-length = 88 [tool.ruff.lint] select = ["E", "F"] + +[tool.pyright] +include = ["benchmark_utils", "datasets", "solvers", "objective.py"] +typeCheckingMode = "basic" +# Every benchopt Objective/Dataset/Solver overrides `name` (a base-class +# property) with a string literal; the resulting reportAssignmentType is purely +# structural, so mute it project-wide rather than annotating each class. +reportAssignmentType = "none" + +# datasets/ and solvers/ are thin benchopt-subclass wrappers over untyped +# libraries (tslearn, wfdb, torch, aeon, pandas). benchopt injects each class's +# `parameters` dict as instance attributes and overrides `name` (a base +# property) at runtime via a metaclass, none of which pyright can see. Scope the +# resulting structural false positives to just these dirs; benchmark_utils/ and +# objective.py (the real logic) stay fully strict. High-signal rules +# (reportReturnType, reportRedeclaration, ...) stay on everywhere. +[[tool.pyright.executionEnvironments]] +root = "datasets" +extraPaths = ["."] # keep `benchmark_utils` importable from this root +reportAttributeAccessIssue = "none" +reportArgumentType = "none" +reportOptionalMemberAccess = "none" +reportCallIssue = "none" +reportIndexIssue = "none" + +[[tool.pyright.executionEnvironments]] +root = "solvers" +extraPaths = ["."] # keep `benchmark_utils` importable from this root +reportAttributeAccessIssue = "none" +reportArgumentType = "none" +reportOptionalMemberAccess = "none" +reportCallIssue = "none" +reportIndexIssue = "none" +# Model solvers import torch-based libs (chronos, momentfm, toto2, mantis) that +# the lightweight type_check CI job does not install; their absence is expected. +reportMissingImports = "none" diff --git a/solvers/naive.py b/solvers/naive.py index b7be080..dd47a2c 100644 --- a/solvers/naive.py +++ b/solvers/naive.py @@ -57,7 +57,7 @@ def fit(self, X_train, y_train, **kwargs): self._label = int(labels[np.argmax(counts)]) return self - def predict(self, x: np.ndarray) -> int: + def predict(self, x: np.ndarray) -> list[int]: return [self._label] * len(x)