From 4b9ac82b5284e2790ca817ffb73f9e598faaf21b Mon Sep 17 00:00:00 2001 From: AzulGarza Date: Fri, 11 Sep 2026 13:05:34 -0600 Subject: [PATCH] feat: add CodSpeed benchmarks for core pipeline and models Introduce pytest-codspeed benchmarks and a dedicated CI workflow to track performance regressions in core utilities and a representative model subset. Co-authored-by: Cursor --- .github/workflows/codspeed.yml | 72 +++++++++++++++++++++++++++++++++ pyproject.toml | 4 +- tests/benchmarks/conftest.py | 47 +++++++++++++++++++++ tests/benchmarks/test_core.py | 38 +++++++++++++++++ tests/benchmarks/test_models.py | 19 +++++++++ uv.lock | 39 ++++++++++++++++++ 6 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/codspeed.yml create mode 100644 tests/benchmarks/conftest.py create mode 100644 tests/benchmarks/test_core.py create mode 100644 tests/benchmarks/test_models.py diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 0000000..094872c --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,72 @@ +name: CodSpeed + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: codspeed-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmarks-core: + name: CodSpeed core + runs-on: ubuntu-latest + steps: + - name: Clone repo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Install dependencies + run: uv sync --frozen + + - name: Run core benchmarks + uses: CodSpeedHQ/action@v5 + with: + mode: simulation + run: uv run pytest tests/benchmarks/test_core.py --codspeed -m benchmark -o addopts= + + benchmarks-models: + name: CodSpeed models + runs-on: codspeed-macro + steps: + - name: Clone repo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Cache HuggingFace Models + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface/hub/ + key: ${{ runner.os }}-huggingface-codspeed-${{ hashFiles('tests/benchmarks/**') }} + + - name: Install dependencies + run: uv sync --frozen + + - name: Run model benchmarks + uses: CodSpeedHQ/action@v5 + with: + mode: walltime + run: uv run pytest tests/benchmarks/test_models.py --codspeed -m benchmark -o addopts= + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index 5e7b254..1017196 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ requires = ["hatchling"] dev = [ "mktestdocs>=0.2.5", "pre-commit", + "pytest-codspeed>=5.0.3", "pytest-cov>=6.0", "pytest-mock>=3.15.1", "pytest-rerunfailures>=15.1", @@ -94,8 +95,9 @@ omit = ["tests/*"] packages = ["foundationforecast"] [tool.pytest.ini_options] -addopts = "-m 'not docs'" +addopts = "-m 'not docs and not benchmark'" markers = [ + "benchmark: marks CodSpeed performance benchmarks", "docs: marks tests related to documentation", "models: marks tests that download model weights", ] diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py new file mode 100644 index 0000000..885396e --- /dev/null +++ b/tests/benchmarks/conftest.py @@ -0,0 +1,47 @@ +import pytest +from tests.helpers import generate_series + + +@pytest.fixture(scope="session") +def panel_df(): + return generate_series( + n_series=10, + freq="D", + min_length=100, + max_length=100, + ) + + +@pytest.fixture(scope="session") +def chronos_bolt(): + from foundationforecast.models.chronos import Chronos + + return Chronos(repo_id="amazon/chronos-bolt-tiny", alias="Chronos-Bolt") + + +@pytest.fixture(scope="session") +def timesfm(): + from foundationforecast.models.timesfm import TimesFM + + return TimesFM( + repo_id="google/timesfm-1.0-200m-pytorch", + context_length=256, + ) + + +@pytest.fixture(scope="session") +def toto(): + from foundationforecast.models.toto import Toto + + return Toto(context_length=256, batch_size=2) + + +@pytest.fixture(scope="session") +def moirai(): + from foundationforecast.models.moirai import Moirai + + return Moirai( + context_length=256, + batch_size=2, + repo_id="Salesforce/moirai-1.1-R-small", + ) diff --git a/tests/benchmarks/test_core.py b/tests/benchmarks/test_core.py new file mode 100644 index 0000000..65e30fc --- /dev/null +++ b/tests/benchmarks/test_core.py @@ -0,0 +1,38 @@ +import pytest +from tests.helpers import SeasonalNaiveModel + +from foundationforecast.core.forecaster import QuantileConverter, maybe_infer_freq +from foundationforecast.core.utils import TimeSeriesDataset + +pytestmark = pytest.mark.benchmark + + +def test_maybe_infer_freq(benchmark, panel_df): + result = benchmark(maybe_infer_freq, panel_df, None) + assert result == "D" + + +def test_timeseries_dataset_from_df(benchmark, panel_df): + def build_dataset(): + return TimeSeriesDataset.from_df(panel_df, batch_size=4) + + dataset = benchmark(build_dataset) + assert len(dataset) > 0 + assert len(next(iter(dataset))) <= 4 + + +def test_quantile_converter_level_to_quantiles(benchmark): + def convert(): + qc = QuantileConverter(level=[80, 95]) + return qc.quantiles + + quantiles = benchmark(convert) + assert quantiles is not None + assert len(quantiles) > 0 + + +def test_seasonal_naive_forecast(benchmark, panel_df): + model = SeasonalNaiveModel() + result = benchmark(model.forecast, panel_df, h=12, freq="D") + assert len(result) == panel_df["unique_id"].nunique() * 12 + assert "SeasonalNaive" in result.columns diff --git a/tests/benchmarks/test_models.py b/tests/benchmarks/test_models.py new file mode 100644 index 0000000..1a2a854 --- /dev/null +++ b/tests/benchmarks/test_models.py @@ -0,0 +1,19 @@ +import pytest + +pytestmark = [pytest.mark.benchmark, pytest.mark.models] + + +@pytest.mark.parametrize( + "model_fixture,expected_alias", + [ + pytest.param("chronos_bolt", "Chronos-Bolt", id="chronos-bolt"), + pytest.param("timesfm", "TimesFM", id="timesfm-1"), + pytest.param("toto", "Toto", id="toto"), + pytest.param("moirai", "Moirai", id="moirai-1.1"), + ], +) +def test_model_forecast(benchmark, model_fixture, expected_alias, panel_df, request): + model = request.getfixturevalue(model_fixture) + result = benchmark(model.forecast, panel_df, h=12, freq="D") + assert len(result) == panel_df["unique_id"].nunique() * 12 + assert expected_alias in result.columns diff --git a/uv.lock b/uv.lock index 52dc3d8..12b4223 100644 --- a/uv.lock +++ b/uv.lock @@ -1422,6 +1422,7 @@ dev = [ {name = "mktestdocs"}, {name = "pre-commit"}, {name = "pytest"}, + {name = "pytest-codspeed"}, {name = "pytest-cov"}, {name = "pytest-mock"}, {name = "pytest-rerunfailures"}, @@ -1472,6 +1473,7 @@ dev = [ {name = "mktestdocs", specifier = ">=0.2.5"}, {name = "pre-commit"}, {name = "pytest", specifier = ">=8.0"}, + {name = "pytest-codspeed", specifier = ">=5.0.3"}, {name = "pytest-cov", specifier = ">=6.0"}, {name = "pytest-mock", specifier = ">=3.15.1"}, {name = "pytest-rerunfailures", specifier = ">=15.1"}, @@ -4893,6 +4895,43 @@ wheels = [ {hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z", url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl"}, ] +[[package]] +dependencies = [ + {name = "pytest"}, + {name = "rich"}, +] +name = "pytest-codspeed" +sdist = {hash = "sha256:91afef90e6a96b013495e4702ef5d6358614a449e71008cdc194ef668778b92f", size = 324571, upload-time = "2026-05-22T16:20:49.231Z", url = "https://files.pythonhosted.org/packages/e1/b4/cf932fcd1960a2fd6d9b09eb403253a8709aeee975961afa6299239a830e/pytest_codspeed-5.0.3.tar.gz"} +source = {registry = "https://pypi.org/simple"} +version = "5.0.3" +wheels = [ + {hash = "sha256:005348ea52ace3ede2e2f595913912ad2564cca7b124211a88dc78a9cb1fca63", size = 366249, upload-time = "2026-05-22T16:20:39.985Z", url = "https://files.pythonhosted.org/packages/a7/f5/a8f70147216e4b84046ca406d03ecc8e83e3ea56ba1bdca0bb79cca79fee/pytest_codspeed-5.0.3-cp310-cp310-macosx_11_0_arm64.whl"}, + {hash = "sha256:0c383c9121deb58a69f174188e9e4488ffc0daced0ed276abf87747182511901", size = 932360, upload-time = "2026-05-22T16:20:30.589Z", url = "https://files.pythonhosted.org/packages/a9/7b/ae76fd8ac656b9695806a6aafd5f22ec32e6ce20e266a58f9112e01d3cd8/pytest_codspeed-5.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl"}, + {hash = "sha256:20eba63765be9d1b6cacbbfad84b87d49eb04b357a7045a0899880da181f81e3", size = 935522, upload-time = "2026-05-22T16:21:03.398Z", url = "https://files.pythonhosted.org/packages/d1/de/2213f868fa7694f743f96cccbc07e757f45c920c523cccc2da97bc8652df/pytest_codspeed-5.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl"}, + {hash = "sha256:25464363c7f9b9bd5022e969c0addba616fa40ac9b8f0fc9e030c4538863b32d", size = 366259, upload-time = "2026-05-22T16:21:06.039Z", url = "https://files.pythonhosted.org/packages/04/6a/fdcec19c7f267c195f147c51d3fd2245f6b8d09b80495ed0a90c008e0842/pytest_codspeed-5.0.3-cp314-cp314-macosx_11_0_arm64.whl"}, + {hash = "sha256:2eeb25fb1ac3f73c4de50e739e78fea396b89782bdb740bf2a7cd2df21f8d4ee", size = 366255, upload-time = "2026-05-22T16:20:56.214Z", url = "https://files.pythonhosted.org/packages/c2/22/456c48160b761d5028c8afa119f085a9fc42855a783a13d73918078969f0/pytest_codspeed-5.0.3-cp312-cp312-macosx_11_0_arm64.whl"}, + {hash = "sha256:4c682f6645d4eb472f3bd95dbda1805e3af4243610572cb7d6bf94a88e8a0b6c", size = 932465, upload-time = "2026-05-22T16:20:34.265Z", url = "https://files.pythonhosted.org/packages/2a/15/c66ef90a793c5d2c039e63a1726a5e55c678be2618b0f5f1660d0f79e25f/pytest_codspeed-5.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl"}, + {hash = "sha256:6524c57fec279a22ffef6112af404036afc71b4704758ae9f0abda429b8478d4", size = 366253, upload-time = "2026-05-22T16:20:46.192Z", url = "https://files.pythonhosted.org/packages/dc/8e/e032451e9e0a06b0c4bff53105f62b693d9a54595dd8c024693741ce3380/pytest_codspeed-5.0.3-cp313-cp313-macosx_11_0_arm64.whl"}, + {hash = "sha256:73c5c9d98a3372a42611989ccfa437cce3842431ac6d6b9ab42c4f0e59c070f7", size = 932325, upload-time = "2026-05-22T16:21:08.814Z", url = "https://files.pythonhosted.org/packages/74/33/ac7441fa937c9d9f158083a8c46920a5a5c81ed3c5f96240fc8d650db5c2/pytest_codspeed-5.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl"}, + {hash = "sha256:782f9985b6f6b45b8bc20152d206d3a52b56dd088ba81cb70a71f0b39841be9e", size = 934994, upload-time = "2026-05-22T16:20:28.809Z", url = "https://files.pythonhosted.org/packages/96/08/56ad8f1cc7d6962f8a680141b361e93467a2abc53d976cd9d5e1edd740e3/pytest_codspeed-5.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl"}, + {hash = "sha256:7ac4344f34bbcdd17f6f8c30dbac3da2f80d223dd112e568fd7f7c2cd4cbc693", size = 934647, upload-time = "2026-05-22T16:20:31.997Z", url = "https://files.pythonhosted.org/packages/b1/e1/414ea4c66559f24ec06aeb6db62bfc7079582dac1452e648affe1eb5cfb4/pytest_codspeed-5.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl"}, + {hash = "sha256:85505c96a3477c346ec2d2b7dced8478f4c651e2b1666ee102d53a832b511853", size = 933169, upload-time = "2026-05-22T16:20:43.178Z", url = "https://files.pythonhosted.org/packages/a7/3c/24c53f67a38ad48cb087105ac30a8aa0923223ee274ea9bf2dc705edaa59/pytest_codspeed-5.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl"}, + {hash = "sha256:8df77b3409f54f4a268f77f3ff74992fe1d995cdbaf2cecf8ad74d32db217ce7", size = 932537, upload-time = "2026-05-22T16:20:54.945Z", url = "https://files.pythonhosted.org/packages/3c/2b/af4d1b612f03b98a6cf3c7d5f62678917a60110a8bf380d49ab408b31137/pytest_codspeed-5.0.3-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl"}, + {hash = "sha256:9aa0815b90196f3c20d736ea8691381e97f12bbe8c7d87af10a351e434b452cb", size = 366311, upload-time = "2026-05-22T16:20:41.791Z", url = "https://files.pythonhosted.org/packages/0b/54/9096c4545f09da94b1b00f3be2fe4952949e86c9bcafca9a29b26aed1a75/pytest_codspeed-5.0.3-cp314-cp314t-macosx_11_0_arm64.whl"}, + {hash = "sha256:a2e0ab65df73e837666d12357280ca50ff6d6ac03ea5266703be518b68170edf", size = 934885, upload-time = "2026-05-22T16:21:01.444Z", url = "https://files.pythonhosted.org/packages/77/bc/8b994adcb9e9016e7d9a808056a3dd9cca21441e432ef456eae2b697d7fe/pytest_codspeed-5.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl"}, + {hash = "sha256:a4bcdb4b6522738152885ef067e0c8524d5699828d780fb6f464cdb3db44369c", size = 934928, upload-time = "2026-05-22T16:20:38.62Z", url = "https://files.pythonhosted.org/packages/a6/4a/dfd43d943fdb143be4fd62f34c2793ba349dc27aa188e521d19d629aa7ab/pytest_codspeed-5.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl"}, + {hash = "sha256:a5d8695a227ea1c3a41d25db5b3fe720bf1b4808bd38862be811a4efd902c792", size = 934153, upload-time = "2026-05-22T16:21:07.494Z", url = "https://files.pythonhosted.org/packages/f5/a2/c7ec45e36a61b418efb2a3cccaa67a0c2fcf1f21d5880f64c33114f0c249/pytest_codspeed-5.0.3-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl"}, + {hash = "sha256:abe793da40f87295d33988673d34f06ea569848b44490b847552cd416816258a", size = 933055, upload-time = "2026-05-22T16:20:44.861Z", url = "https://files.pythonhosted.org/packages/a8/37/fb27aeb40a81320e7349553b877a21333c897b27c8dfe215630452908f36/pytest_codspeed-5.0.3-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl"}, + {hash = "sha256:bf4cc4178cbace8f4d2bd240408276bc4da3850ac5fcb5fb5f8a74ab417615bb", size = 366339, upload-time = "2026-05-22T16:20:51.968Z", url = "https://files.pythonhosted.org/packages/cf/c7/d5bada9618a0af56a5c8065fc61280849cab8e7c1e24025807a51c3157ce/pytest_codspeed-5.0.3-cp315-cp315t-macosx_11_0_arm64.whl"}, + {hash = "sha256:c3a9ed38dfa776443b86f4b49a982e8443d0953db4974bd2673d63cc904ae1ad", size = 934481, upload-time = "2026-05-22T16:20:58.264Z", url = "https://files.pythonhosted.org/packages/f3/d9/6f2d69e96deaf0475a695fc9195af59e7a3b5fab50782855e65c63a7bc28/pytest_codspeed-5.0.3-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl"}, + {hash = "sha256:dbe6a4a00b449b6ba2771f644cbc38bdf55acf5c812e60e5659110e19dd9f510", size = 932229, upload-time = "2026-05-22T16:20:37.283Z", url = "https://files.pythonhosted.org/packages/f6/bd/7a4dbcf457fcc3ed788c55d402f3af2671e0e342b6098090fd590aa8712e/pytest_codspeed-5.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl"}, + {hash = "sha256:ec9fa6f0af0a9feb0e0bd517fb59ef28f806fbd50c0c6900ac26cbb4d080eba5", size = 366275, upload-time = "2026-05-22T16:20:59.463Z", url = "https://files.pythonhosted.org/packages/df/85/5dfea1c031d6cccc11653464828edf205c30f798caf5b2a85375aacd914a/pytest_codspeed-5.0.3-cp315-cp315-macosx_11_0_arm64.whl"}, + {hash = "sha256:efd43f82ea03ced8488a767ded9473f050791ab7783ea8654107e1e0ac66af40", size = 932395, upload-time = "2026-05-22T16:21:04.804Z", url = "https://files.pythonhosted.org/packages/6a/96/c6b03b81dcd21ae3d6b32cca0b3c10149fa378eb21b338d4b63c9eb8050b/pytest_codspeed-5.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl"}, + {hash = "sha256:f56d0339cd98d26f6e561987be25bdd2761a5d53d8f73493b1ebe02d0d451093", size = 366253, upload-time = "2026-05-22T16:21:10.013Z", url = "https://files.pythonhosted.org/packages/ac/ef/32ce60d42a4aa43e728d988e13eb6568fbc7b10a514517b459bafd3f2b94/pytest_codspeed-5.0.3-cp311-cp311-macosx_11_0_arm64.whl"}, + {hash = "sha256:f852bee785a7a124cb1720b1915670c6742af87747dc4d838f3ffdbd365ce9d9", size = 934925, upload-time = "2026-05-22T16:20:47.63Z", url = "https://files.pythonhosted.org/packages/1a/7b/d231279301967f05b7909160489e85ee3a1b9da76094ea25343faba1abc2/pytest_codspeed-5.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl"}, + {hash = "sha256:fe2ea83c924c2250675b75686c3ee456b8cf0208d83d552e182a195fdf467378", size = 74033, upload-time = "2026-05-22T16:20:26.814Z", url = "https://files.pythonhosted.org/packages/c5/b2/1d2a993c532146dce9eca5b5942d51898021c3579ce18b2454f932a915f8/pytest_codspeed-5.0.3-py3-none-any.whl"}, +] + [[package]] dependencies = [ {extra = ["toml"], name = "coverage"},