Add CodSpeed continuous benchmarking

.github/workflows/codspeed.yaml

@@ -0,0 +1,51 @@
+name: CodSpeed
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write # for OpenID Connect authentication with CodSpeed
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+    env:
+      UV_PYTHON: "3.11"
+
+    steps:
+      - name: Clone repo
+        uses: actions/checkout@v4
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Free disk space in action runner
+        uses: endersonmenezes/free-disk-space@v3
+        with:
+          remove_android: true
+          remove_dotnet: true
+          remove_haskell: true
+          rm_cmd: "rmz"
+          rmz_version: "3.1.1"
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v5
+        with:
+          mode: simulation
+          run: uv run pytest benchmarks/ --codspeed

README.md

@@ -10,6 +10,7 @@
   <a href="https://pypi.python.org/pypi/foundationforecast"><img src="https://img.shields.io/pypi/v/foundationforecast.svg" alt="PyPI"></a>
   <a href="https://github.com/TimeCopilot/foundationforecast"><img src="https://img.shields.io/pypi/pyversions/foundationforecast.svg" alt="versions"></a>
   <a href="https://github.com/TimeCopilot/foundationforecast/blob/main/LICENSE"><img src="https://img.shields.io/github/license/TimeCopilot/foundationforecast" alt="license"></a>
+  <a href="https://app.codspeed.io/TimeCopilot/foundationforecast?utm_source=badge"><img src="https://img.shields.io/endpoint?url=https://codspeed.io/badge.json" alt="CodSpeed"/></a>
   <a href="https://discord.gg/7GEdHR6Pfg"><img src="https://img.shields.io/discord/1387291858513821776?label=discord" alt="Join Discord"></a>
 </div>
 

benchmarks/__init__.py

@@ -0,0 +1 @@
+"""CodSpeed performance benchmarks for foundationforecast."""

benchmarks/_models.py

@@ -0,0 +1,86 @@
+"""Lightweight forecasters used to benchmark the FoundationForecast pipeline.
+
+These models are deliberately trivial: the goal is to measure the cost of the
+library's own data plumbing (validation, splitting, merging, quantile handling)
+without downloading or running any foundation model weights.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from utilsforecast.processing import make_future_dataframe
+
+from foundationforecast.core.forecaster import Forecaster, QuantileConverter
+
+
+class ConstantModel(Forecaster):
+    """Return a constant point forecast, plus quantiles when requested."""
+
+    def __init__(self, alias: str = "Constant", value: float = 1.0):
+        self.alias = alias
+        self.value = value
+
+    def forecast(
+        self,
+        df: pd.DataFrame,
+        h: int,
+        freq: str | None = None,
+        level: list[int | float] | None = None,
+        quantiles: list[float] | None = None,
+    ) -> pd.DataFrame:
+        freq = self._maybe_infer_freq(df, freq)
+        qc = QuantileConverter(level=level, quantiles=quantiles)
+        last_times = df.groupby("unique_id")["ds"].max()
+        fcst = make_future_dataframe(
+            uids=last_times.index.tolist(),
+            last_times=last_times,
+            h=h,
+            freq=freq,
+        )
+        fcst[self.alias] = self.value
+        if qc.quantiles:
+            for q in qc.quantiles:
+                fcst[f"{self.alias}-q-{int(q * 100)}"] = fcst[self.alias] + q
+            fcst = qc.maybe_convert_quantiles_to_level(fcst, models=[self.alias])
+        return fcst
+
+
+class SeasonalNaiveModel(Forecaster):
+    """Repeat the last seasonal period, mimicking a real model's output shape."""
+
+    alias = "SeasonalNaive"
+
+    def __init__(self, season_length: int | None = None):
+        self.season_length = season_length
+
+    def forecast(
+        self,
+        df: pd.DataFrame,
+        h: int,
+        freq: str | None = None,
+        level: list[int | float] | None = None,
+        quantiles: list[float] | None = None,
+    ) -> pd.DataFrame:
+        freq = self._maybe_infer_freq(df, freq)
+        season_length = self._maybe_get_seasonality(freq)
+        qc = QuantileConverter(level=level, quantiles=quantiles)
+        results = []
+        for uid, group in df.groupby("unique_id"):
+            y = group["y"].to_numpy()
+            effective_length = min(season_length, len(y))
+            future = make_future_dataframe(
+                uids=[uid],
+                last_times=pd.Series([group["ds"].max()], index=[uid]),
+                h=h,
+                freq=freq,
+            )
+            seasonal_values = y[-effective_length:]
+            future[self.alias] = seasonal_values[np.arange(h) % effective_length]
+            results.append(future)
+        fcst = pd.concat(results, ignore_index=True)
+        if qc.quantiles:
+            for q in qc.quantiles:
+                fcst[f"{self.alias}-q-{int(q * 100)}"] = fcst[self.alias] * (1 + q)
+            fcst = qc.maybe_convert_quantiles_to_level(fcst, models=[self.alias])
+        return fcst

benchmarks/conftest.py

@@ -0,0 +1,55 @@
+"""Shared, deterministic data fixtures for the CodSpeed benchmarks."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+import pytest
+from utilsforecast.data import generate_series
+
+
+def make_panel(
+    n_series: int,
+    length: int,
+    freq: str = "D",
+) -> pd.DataFrame:
+    """Generate a deterministic panel with `n_series` series of `length` rows.
+
+    `generate_series` is seeded by default, so the data is stable across runs.
+    """
+    df = generate_series(
+        n_series=n_series,
+        freq=freq,
+        min_length=length,
+        max_length=length,
+    )
+    df["unique_id"] = df["unique_id"].astype(str)
+    return df.reset_index(drop=True)
+
+
+@pytest.fixture(scope="session")
+def small_panel() -> pd.DataFrame:
+    """20 daily series of 200 observations."""
+    return make_panel(n_series=20, length=200)
+
+
+@pytest.fixture(scope="session")
+def large_panel() -> pd.DataFrame:
+    """200 daily series of 500 observations."""
+    return make_panel(n_series=200, length=500)
+
+
+@pytest.fixture(scope="session")
+def unsorted_panel(large_panel: pd.DataFrame) -> pd.DataFrame:
+    """A shuffled copy of `large_panel`, forcing the sorting code paths."""
+    rng = np.random.default_rng(42)
+    idxs = rng.permutation(len(large_panel))
+    return large_panel.iloc[idxs].reset_index(drop=True)
+
+
+@pytest.fixture(scope="session")
+def string_ds_panel(large_panel: pd.DataFrame) -> pd.DataFrame:
+    """`large_panel` with `ds` stored as ISO-8601 strings."""
+    df = large_panel.copy()
+    df["ds"] = df["ds"].dt.strftime("%Y-%m-%d")
+    return df

benchmarks/test_bench_core_utils.py

@@ -0,0 +1,78 @@
+"""Benchmarks for the pure-python helpers used on every forecasting call."""
+
+from __future__ import annotations
+
+import pandas as pd
+import pytest
+
+from foundationforecast.core.forecaster import (
+    Forecaster,
+    QuantileConverter,
+    get_seasonality,
+    maybe_convert_col_to_datetime,
+    maybe_infer_freq,
+)
+from foundationforecast.core.gluonts_forecaster import (
+    maybe_convert_col_to_float32,
+)
+
+
+def test_validate_input(benchmark, large_panel: pd.DataFrame):
+    benchmark(Forecaster.validate_input, large_panel, 12)
+
+
+def test_maybe_infer_freq(benchmark, large_panel: pd.DataFrame):
+    freq = benchmark(maybe_infer_freq, large_panel, None)
+    assert freq == "D"
+
+
+def test_maybe_convert_col_to_datetime_from_string(
+    benchmark,
+    string_ds_panel: pd.DataFrame,
+):
+    out = benchmark(maybe_convert_col_to_datetime, string_ds_panel, "ds")
+    assert pd.api.types.is_datetime64_any_dtype(out["ds"])
+
+
+def test_maybe_convert_col_to_float32(benchmark, large_panel: pd.DataFrame):
+    out = benchmark(maybe_convert_col_to_float32, large_panel, "y")
+    assert out["y"].dtype == "float32"
+
+
+@pytest.mark.parametrize("freq", ["D", "h", "MS"])
+def test_get_seasonality(benchmark, freq: str):
+    benchmark(get_seasonality, freq)
+
+
+def test_quantile_converter_from_level(benchmark):
+    qc = benchmark(QuantileConverter, [80, 90, 95], None)
+    assert qc.quantiles is not None
+
+
+def test_quantile_converter_level_to_quantiles(
+    benchmark,
+    large_panel: pd.DataFrame,
+):
+    """Levels -> quantile columns, as done when users pass `quantiles`."""
+    model = "Model"
+    df = large_panel.rename(columns={"y": model})
+    qc = QuantileConverter(quantiles=[0.1, 0.25, 0.5, 0.75, 0.9])
+    for lv in [50, 80]:
+        df[f"{model}-lo-{lv}"] = df[model] - lv / 100
+        df[f"{model}-hi-{lv}"] = df[model] + lv / 100
+    out = benchmark(qc.maybe_convert_level_to_quantiles, df, [model])
+    assert f"{model}-q-90" in out.columns
+
+
+def test_quantile_converter_quantiles_to_level(
+    benchmark,
+    large_panel: pd.DataFrame,
+):
+    """Quantile columns -> lo/hi levels, as done when users pass `level`."""
+    model = "Model"
+    df = large_panel.rename(columns={"y": model})
+    qc = QuantileConverter(level=[80, 90])
+    for q in qc.quantiles or []:
+        df[f"{model}-q-{int(q * 100)}"] = df[model] + q
+    out = benchmark(qc.maybe_convert_quantiles_to_level, df, [model])
+    assert f"{model}-lo-90" in out.columns

benchmarks/test_bench_dataset.py

@@ -0,0 +1,63 @@
+"""Benchmarks for the tensor-side data preparation used by every model."""
+
+from __future__ import annotations
+
+import pandas as pd
+import pytest
+import torch
+
+from foundationforecast.core.forecaster import _DataProcessor
+from foundationforecast.core.utils import TimeSeriesDataset
+
+
+@pytest.mark.parametrize("batch_size", [16, 128])
+def test_time_series_dataset_from_df(
+    benchmark,
+    large_panel: pd.DataFrame,
+    batch_size: int,
+):
+    dataset = benchmark(TimeSeriesDataset.from_df, large_panel, batch_size)
+    assert len(dataset) > 0
+
+
+def test_time_series_dataset_iteration(benchmark, large_panel: pd.DataFrame):
+    dataset = TimeSeriesDataset.from_df(large_panel, batch_size=16)
+
+    def consume() -> int:
+        return sum(len(batch) for batch in dataset)
+
+    assert benchmark(consume) == len(large_panel["unique_id"].unique())
+
+
+def test_time_series_dataset_make_future_dataframe(
+    benchmark,
+    large_panel: pd.DataFrame,
+):
+    dataset = TimeSeriesDataset.from_df(large_panel, batch_size=32)
+    future = benchmark(dataset.make_future_dataframe, 24, "D")
+    assert len(future) == 24 * len(large_panel["unique_id"].unique())
+
+
+def _variable_length_context(
+    n_series: int = 128,
+    max_len: int = 512,
+) -> list[torch.Tensor]:
+    generator = torch.Generator().manual_seed(0)
+    return [
+        torch.rand(max_len - (i % 64), generator=generator, dtype=torch.float32)
+        for i in range(n_series)
+    ]
+
+
+def test_data_processor_left_pad_and_stack(benchmark):
+    processor = _DataProcessor(dtype=torch.float32, device=torch.device("cpu"))
+    context = _variable_length_context()
+    out = benchmark(processor._prepare_and_validate_context, context)
+    assert out.shape[0] == len(context)
+
+
+def test_data_processor_impute_missing(benchmark):
+    processor = _DataProcessor(dtype=torch.float32, device=torch.device("cpu"))
+    batch = processor._prepare_and_validate_context(_variable_length_context())
+    out = benchmark(processor._maybe_impute_missing, batch)
+    assert not torch.isnan(out).any()