Benchmark against MiniSom and SOMPY, and track performance with asv

.github/workflows/benchmarks.yml

@@ -0,0 +1,78 @@
+name: benchmarks
+
+# Manual only, and that is the point rather than an omission.
+#
+# asv measures a project against its own git history, and its numbers mean something only as a
+# trend on one consistent machine. GitHub's shared runners are not that: an earlier version of
+# benchmarks/bench_vs_minisom.py swung a comparison from 2.56x to 1.01x between two runs on an idle
+# laptop, and a shared runner is considerably worse than an idle laptop. So nothing here runs on
+# push, nothing gates a merge, and the results are an artifact to read rather than a check to pass.
+#
+# What DOES run on every push is the smoke step in ci.yml, which executes each benchmark once and
+# discards the timings. That catches the failure this file cannot: benchmark code rotting unnoticed
+# because nobody runs it.
+#
+# For numbers worth quoting, run it locally on a quiet machine:
+#
+#   cd asv_benchmarks
+#   uv run --extra bench asv machine --yes
+#   uv run --extra bench asv continuous master HEAD
+on:
+  workflow_dispatch:
+    inputs:
+      base:
+        description: "Commit, tag or branch to compare against"
+        required: true
+        default: master
+      head:
+        description: "Commit, tag or branch to measure"
+        required: true
+        default: HEAD
+      factor:
+        description: "Ratio a change must exceed to be reported"
+        required: false
+        default: "1.1"
+
+# Actions are pinned by commit SHA rather than by tag: a tag can be moved to point at new code,
+# a SHA cannot. Dependabot keeps them current.
+permissions:
+  contents: read
+
+jobs:
+  asv:
+    name: asv continuous ${{ inputs.base }}..${{ inputs.head }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          # asv builds and installs each revision it is asked about, so it needs the history that
+          # a default shallow clone does not fetch.
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          enable-cache: true
+          python-version: "3.12"
+      - run: uv sync --extra bench
+
+      - name: register the runner
+        working-directory: asv_benchmarks
+        run: uv run --extra bench asv machine --yes
+
+      # `|| true` because `asv continuous` exits non-zero when it finds a regression, and a
+      # regression is the result this job exists to surface rather than a reason to fail it. The
+      # numbers are in the artifact either way; a human reads them and decides.
+      - name: compare the two revisions
+        working-directory: asv_benchmarks
+        run: |
+          uv run --extra bench asv continuous \
+            --factor "${{ inputs.factor }}" \
+            "${{ inputs.base }}" "${{ inputs.head }}" | tee ../asv-comparison.txt || true
+
+      # The comparison text only. `.asv/results/` also contains machine.json, which records the
+      # host's name, CPU model, core count, RAM and kernel version. That is a hardware fingerprint,
+      # it is useless to anyone reading the comparison, and a downloadable artifact is a poor place
+      # to keep it even when the host is an ephemeral runner.
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: asv-comparison
+          path: asv-comparison.txt

.github/workflows/ci.yml

@@ -60,6 +60,50 @@ jobs:
       - name: pytest
         run: uv run pytest --cov --cov-report=term-missing --cov-report=xml
 
+  benchmarks:
+    name: benchmarks still run
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          enable-cache: true
+          python-version: "3.12"
+      - run: uv sync --extra dev --extra bench
+
+      # Timings are deliberately NOT asserted on. A shared runner cannot measure anything: an
+      # earlier version of benchmarks/bench_vs_minisom.py swung a comparison from 2.56x to 1.01x
+      # between two runs on an *idle* machine. What this job protects is that the benchmark code
+      # still imports and executes, which is the thing that rots silently while nobody runs it.
+      #
+      # asv has no `check` subcommand, so the way to execute every benchmark once and record
+      # nothing is `run --quick --dry-run`. `--python=same` reuses this environment instead of
+      # building a wheel per revision. `asv machine --yes` is required first: without a machine
+      # profile asv exits 1 before running anything.
+      - name: every asv benchmark executes once
+        working-directory: asv_benchmarks
+        run: |
+          uv run --extra bench asv machine --yes
+          uv run --extra bench asv run --python=same --quick --dry-run
+
+      # The cross-library scripts are not run here. bench_vs_minisom.py trains real maps and takes
+      # minutes, and bench_vs_sompy.py needs an interpreter this environment cannot provide. Their
+      # correctness claims are covered by tests/test_minisom_agreement.py, which does run in CI.
+      # PYTHONPATH rather than sys.path.insert with a relative path, which silently resolves
+      # against whatever directory the step happens to start in.
+      - name: the comparison harnesses still import
+        env:
+          PYTHONPATH: benchmarks
+        run: |
+          uv run python -c "
+          import bench_update, bench_batch, bench_vs_minisom, bench_vs_sompy
+          print('benchmark scripts import OK')
+          "
+      - name: bench_vs_sompy degrades cleanly without its environment
+        run: |
+          test ! -d .venv-sompy
+          uv run python benchmarks/bench_vs_sompy.py | grep -q "uv venv .venv-sompy"
+
   build:
     name: build and verify the distribution
     runs-on: ubuntu-latest

.gitignore

@@ -109,6 +109,17 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+# Built by hand for benchmarks/bench_vs_sompy.py. SOMPY cannot be installed alongside this package,
+# because it uses np.Inf and so cannot be imported on NumPy 2, so it gets an interpreter of its own.
+.venv-sompy/
+
+# Everything airspeed velocity generates: environments, results and the rendered site.
+#
+# Two reasons, and the second is the stronger one. Results are per-machine, so a database from one
+# laptop tells nobody anything about another. And `asv machine` records the host's name, CPU model,
+# core count, RAM and kernel version into .asv/results/<machine>/machine.json, which is a hardware
+# fingerprint of whoever ran it and has no business in a public repository.
+.asv/
 
 # Spyder project settings
 .spyderproject

CHANGELOG.md

@@ -5,6 +5,44 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+Nothing in the shipped package changed. This is benchmarking and the tests behind it.
+
+### Added
+
+- **A published comparison against MiniSom and SOMPY**, at
+  [Comparison with MiniSom and SOMPY](https://andremsouza.github.io/python-som/explanation/comparison-with-som-libraries/).
+  Both peers implement Kohonen's Eq. (8) and cite the same sources this package works from, so the
+  comparison is between three implementations of one published algorithm rather than between three
+  different algorithms. Getting there needed eight controls, because the packages look far more
+  interchangeable than they are: MiniSom's `train_batch` is stepwise training rather than batch,
+  only the gaussian is the same function in all three, and SOMPY's
+  `calculate_quantization_error` returns an elementwise mean absolute error rather than the usual
+  mean Euclidean distance.
+
+  Measured on batch training, this package is 1.3x to 2.0x faster than SOMPY and 1.3x to 1.6x
+  **slower** than MiniSom, the latter growing with map size. `batch_update` walks every node in
+  Python, 108,000 `einsum` calls on a 60x60 map over 30 iterations, where MiniSom's node-side update
+  is a single vectorised divide. Stepwise training is within 10% of MiniSom's.
+
+- **`tests/test_minisom_agreement.py`**, which checks the neighborhood functions, the decay, the
+  best-matching-unit search and both training loops against MiniSom on every run. Trained models
+  agree to 2.8e-16 relative for stepwise and 1.3e-15 for batch, which is what makes timing the two
+  against each other meaningful. It also pins the two neighborhoods that deliberately *disagree*, so
+  neither is later "fixed" into the other. Same reasoning as
+  `tests/test_linalg_matches_sklearn.py`, which once found a real 5.8% defect in this package.
+
+- **An asv suite in `asv_benchmarks/`** tracking this package against its own git history, which is
+  what numpy, scipy, pandas and scikit-learn all use asv for. Sixteen benchmarks over the training
+  loops, the neighborhood kernel, the matching functions, the SVD and the artifact round trip. No
+  timing gates anything: CI executes each benchmark once and discards the numbers, because a shared
+  runner cannot measure anything.
+
+- **A `bench` extra** holding `asv` and `minisom`, kept out of `dev` because no gating job needs
+  them. SOMPY is deliberately absent and cannot be added: it uses `np.Inf`, removed in NumPy 2.0, at
+  class-definition time, so it gets an interpreter of its own that is built by hand.
+
 ## [0.6.1] - 2026-07-30
 
 ### Fixed
@@ -20,8 +58,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
   The upgrade notes were also short of two things worth knowing. 0.4.0 changes
   linear-initialization results for data far from the origin, where the previous PCA path was wrong
   by up to 5.8%, and it removed pandas and scikit-learn as runtime dependencies, which matters to
-  anyone who was importing either transitively. Both are now in the README, alongside a note that 0.5.0's
-  string deprecation was withdrawn so anyone who saw the warning knows to stop migrating.
+  anyone who was importing either transitively. Both are now in the README, alongside a note that
+  0.5.0's string deprecation was withdrawn so anyone who saw the warning knows to stop migrating.
 
 ## [0.6.0] - 2026-07-30
 

README.md

@@ -134,6 +134,21 @@ pre-commit install           # optional, run the gates on commit
 If you use the SonarQube for IDE (SonarLint) VS Code extension, it will also apply Sonar's Python
 rules locally; the ruff configuration is set up to cover most of the same ground.
 
+### Benchmarks
+
+Hand-run, never part of the test suite: a timing assertion on shared hardware measures noise.
+
+```bash
+uv run python benchmarks/bench_vs_minisom.py   # vs MiniSom, agreement verified before timing
+uv run python benchmarks/bench_batch.py        # the neighborhood kernel against evaluating per node
+cd asv_benchmarks && uv run --extra bench asv continuous master HEAD   # this package across commits
+```
+
+Results and the method behind them are in
+[Comparison with MiniSom and SOMPY](https://andremsouza.github.io/python-som/explanation/comparison-with-som-libraries/).
+The SOMPY comparison needs an interpreter of its own, since SOMPY cannot be imported on NumPy 2;
+`benchmarks/bench_vs_sompy.py` prints the setup command and installs nothing.
+
 ## References
 
 Based on:

asv_benchmarks/asv.conf.json

@@ -0,0 +1,41 @@
+{
+    // airspeed velocity configuration. This tracks python-som against its OWN git history; the
+    // cross-library comparisons live in benchmarks/ and are a different question with a different
+    // tool. numpy, scipy, pandas and scikit-learn all use asv for exactly this.
+    //
+    // Modelled on numpy's, which is the closer fit of the four since this is a pure-Python wheel.
+    // The directory is named after scikit-learn's so it cannot be confused with benchmarks/, which
+    // holds hand-run scripts rather than an asv suite.
+    //
+    //   uv run --extra bench asv run                  # benchmark the working tree
+    //   uv run --extra bench asv continuous master HEAD   # compare two commits
+    //   uv run --extra bench asv publish && asv preview   # render the results
+    //
+    // Results are per-machine and are NOT committed: a database from one laptop says nothing about
+    // anyone else's, and .asv/ is gitignored for that reason.
+    "version": 1,
+    "project": "python-som",
+    "project_url": "https://andremsouza.github.io/python-som/",
+    "repo": "..",
+    "branches": ["master"],
+    "dvcs": "git",
+    "environment_type": "virtualenv",
+    "show_commit_url": "https://github.com/andremsouza/python-som/commit/",
+    "build_command": [
+        "python -m build --wheel -o {build_cache_dir} {build_dir}"
+    ],
+    "install_command": ["python -mpip install {wheel_file}"],
+    "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"],
+    "benchmark_dir": "benchmarks",
+    // Everything asv generates lives under one gitignored directory rather than four.
+    "env_dir": "../.asv/env",
+    "results_dir": "../.asv/results",
+    "html_dir": "../.asv/html",
+    // NumPy is the only runtime dependency, so an empty list means "whatever resolves", which is
+    // what a library wanting to notice an upstream regression should track.
+    "matrix": {
+        "req": {
+            "numpy": []
+        }
+    }
+}

asv_benchmarks/benchmarks/__init__.py

@@ -0,0 +1,9 @@
+"""Benchmarks tracking python-som against its own git history.
+
+Run with ``uv run --extra bench asv run`` from ``asv_benchmarks/``. These measure this package
+alone; the cross-library comparisons are hand-run scripts in ``benchmarks/`` and answer a different
+question with a different tool.
+
+Nothing here gates anything. asv numbers are only meaningful as a trend on one consistent machine,
+so CI runs each benchmark once to prove it still executes and discards the timings.
+"""

asv_benchmarks/benchmarks/artifacts.py

@@ -0,0 +1,66 @@
+"""Saving and loading a trained map.
+
+Not a hot path, and tracked anyway: this is the boundary where a map becomes a file someone keeps,
+and the format deliberately refuses pickle. A change that made loading slow enough to discourage
+saving would quietly push people back toward ``pickle.dump``, which is arbitrary code execution on
+load. Cheap to measure, so it is measured.
+"""
+
+from __future__ import annotations
+
+import shutil
+import tempfile
+from pathlib import Path
+
+import python_som
+
+from .common import FEATURES, SHAPES, som
+
+
+class Artifacts:
+    """One save and one load per timed run, on a real temporary file."""
+
+    params = (SHAPES, FEATURES)
+    param_names = ("shape", "n_features")
+
+    som: python_som.SOM
+    directory: Path
+    path: Path
+
+    def setup(self, shape: tuple[int, int], n_features: int) -> None:
+        """Build a map and a temporary directory, and write one file to load from.
+
+        :param shape: Grid shape.
+        :param n_features: Number of features.
+        """
+        self.som = som(shape, n_features)
+        self.directory = Path(tempfile.mkdtemp(prefix="som-asv-"))
+        self.path = self.directory / "map.npz"
+        self.som.save_npz(self.path)
+
+    def teardown(self, shape: tuple[int, int], n_features: int) -> None:
+        """Remove the temporary directory.
+
+        :param shape: Unused.
+        :param n_features: Unused.
+        """
+        del shape, n_features
+        shutil.rmtree(self.directory, ignore_errors=True)
+
+    def time_save(self, shape: tuple[int, int], n_features: int) -> None:
+        """Time writing a map, models and metadata together.
+
+        :param shape: Unused.
+        :param n_features: Unused.
+        """
+        del shape, n_features
+        self.som.save_npz(self.directory / "written.npz")
+
+    def time_load(self, shape: tuple[int, int], n_features: int) -> None:
+        """Time reading one back, including the metadata validation.
+
+        :param shape: Unused.
+        :param n_features: Unused.
+        """
+        del shape, n_features
+        python_som.SOM.load_npz(self.path)