From 8a85fc4f26973710591bfd12b8c3475b649de3c5 Mon Sep 17 00:00:00 2001 From: AzulGarza Date: Fri, 11 Sep 2026 13:37:17 -0600 Subject: [PATCH 1/2] feat: add panel and multi-model CodSpeed benchmarks Add large-panel dataset fixture, parametrize TimeSeriesDataset.from_df benchmarks, and FoundationForecast multi-model orchestration benchmarks to measure preprocessing speedups from shared panel processing (PR #18). Co-authored-by: Cursor --- .github/workflows/codspeed.yml | 2 +- tests/benchmarks/conftest.py | 10 +++++++ tests/benchmarks/test_core.py | 11 ++++++-- tests/benchmarks/test_orchestration.py | 36 ++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 tests/benchmarks/test_orchestration.py diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 094872c..26b9d26 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -35,7 +35,7 @@ jobs: uses: CodSpeedHQ/action@v5 with: mode: simulation - run: uv run pytest tests/benchmarks/test_core.py --codspeed -m benchmark -o addopts= + run: uv run pytest tests/benchmarks/test_core.py tests/benchmarks/test_orchestration.py --codspeed -m benchmark -o addopts= benchmarks-models: name: CodSpeed models diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index 885396e..c37e756 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -12,6 +12,16 @@ def panel_df(): ) +@pytest.fixture(scope="session") +def large_panel_df(): + return generate_series( + n_series=100, + freq="D", + min_length=500, + max_length=500, + ) + + @pytest.fixture(scope="session") def chronos_bolt(): from foundationforecast.models.chronos import Chronos diff --git a/tests/benchmarks/test_core.py b/tests/benchmarks/test_core.py index 65e30fc..daf259c 100644 --- a/tests/benchmarks/test_core.py +++ b/tests/benchmarks/test_core.py @@ -12,9 +12,16 @@ def test_maybe_infer_freq(benchmark, panel_df): assert result == "D" -def test_timeseries_dataset_from_df(benchmark, panel_df): +@pytest.mark.parametrize( + "panel_df_fixture", + ["panel_df", "large_panel_df"], + ids=["small", "large"], +) +def test_timeseries_dataset_from_df(benchmark, panel_df_fixture, request): + df = request.getfixturevalue(panel_df_fixture) + def build_dataset(): - return TimeSeriesDataset.from_df(panel_df, batch_size=4) + return TimeSeriesDataset.from_df(df, batch_size=4) dataset = benchmark(build_dataset) assert len(dataset) > 0 diff --git a/tests/benchmarks/test_orchestration.py b/tests/benchmarks/test_orchestration.py new file mode 100644 index 0000000..0b59520 --- /dev/null +++ b/tests/benchmarks/test_orchestration.py @@ -0,0 +1,36 @@ +import pytest + +from foundationforecast import FoundationForecast +from foundationforecast.core.forecaster import Forecaster +from foundationforecast.core.utils import TimeSeriesDataset + +pytestmark = pytest.mark.benchmark + + +class DatasetTouchingModel(Forecaster): + """Lightweight model that exercises dataset construction like real forecasters.""" + + batch_size = 32 + + def __init__(self, alias: str = "DatasetTouch"): + self.alias = alias + + def forecast(self, df, h, freq=None, level=None, quantiles=None, **kwargs): + panel = kwargs.get("panel") + ds_kwargs = {"df": df, "batch_size": self.batch_size} + if panel is not None: + ds_kwargs["panel"] = panel + dataset = TimeSeriesDataset.from_df(**ds_kwargs) + fcst = dataset.make_future_dataframe(h=h, freq=freq or "D") + fcst[self.alias] = 1.0 + return fcst + + +@pytest.mark.parametrize("n_models", [3, 5], ids=["3-models", "5-models"]) +def test_foundation_forecast_multi_model(benchmark, large_panel_df, n_models): + models = [DatasetTouchingModel(alias=f"DatasetTouch{i}") for i in range(n_models)] + forecaster = FoundationForecast(models=models) + result = benchmark(forecaster.forecast, large_panel_df, h=12, freq="D") + assert len(result) == large_panel_df["unique_id"].nunique() * 12 + for model in models: + assert model.alias in result.columns From 8f5472bc1d715e1fe55101c27e003e729ba95a08 Mon Sep 17 00:00:00 2001 From: AzulGarza Date: Fri, 11 Sep 2026 14:50:47 -0600 Subject: [PATCH 2/2] fix: satisfy mypy list invariance in orchestration benchmark Annotate models as list[Forecaster] when constructing FoundationForecast. Co-authored-by: Cursor --- tests/benchmarks/test_orchestration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/benchmarks/test_orchestration.py b/tests/benchmarks/test_orchestration.py index 0b59520..a35766c 100644 --- a/tests/benchmarks/test_orchestration.py +++ b/tests/benchmarks/test_orchestration.py @@ -28,7 +28,9 @@ def forecast(self, df, h, freq=None, level=None, quantiles=None, **kwargs): @pytest.mark.parametrize("n_models", [3, 5], ids=["3-models", "5-models"]) def test_foundation_forecast_multi_model(benchmark, large_panel_df, n_models): - models = [DatasetTouchingModel(alias=f"DatasetTouch{i}") for i in range(n_models)] + models: list[Forecaster] = [ + DatasetTouchingModel(alias=f"DatasetTouch{i}") for i in range(n_models) + ] forecaster = FoundationForecast(models=models) result = benchmark(forecaster.forecast, large_panel_df, h=12, freq="D") assert len(result) == large_panel_df["unique_id"].nunique() * 12