From 358ed66d140b10be7eaf9c6b96f43c94bbcd4d9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Moreira=20Souza?= Date: Thu, 30 Jul 2026 09:45:57 -0300 Subject: [PATCH 1/2] chore: add an analysis extra with bandit, pip-audit and pylint Static analysis for review passes, pinned exactly like the rest of the tooling. Nothing here gates a merge: ruff and mypy remain the enforced checks. bandit reports no issues across src/ and benchmarks/; pip-audit reports no known vulnerabilities. bandit excludes tests/ rather than disabling B101, so `assert` is still flagged in src/, where it would be wrong -- asserts vanish under `python -O`, so validation has to raise. CI names the extras it needs instead of using --all-extras, so review tools no CI job runs are not installed on every matrix entry. `examples` is named explicitly because tests/test_end_to_end.py imports matplotlib. Two candidates were rejected rather than added: - `osv-scanner` on PyPI is version 0.0.1, registered 2026-07-16, with no author, no home page, and the summary "Reserved name placeholder. No functionality." The real tool is a Go binary from Google. Installing a code-free package squatting the expected name is the slopsquat case the dependency rules exist for. - `semgrep` pins `mcp==1.23.3`, which carries PYSEC-2026-3481/3482/3483 (fixed in 1.27.2/1.28.1). The exact pin makes them unremediable from here and 1.172.0 is already the latest. Against that, bandit finds nothing in src/, and this is a pure-numpy library with no network, auth, crypto, SQL or templating -- most of what semgrep's rulesets target. Revisit if the pin is relaxed. - `guarddog` is genuine but first released 2022-11-28, after this project, so it fails the rule that a dependency should predate what it is added to. --- .github/workflows/ci.yml | 10 +- .github/workflows/docs.yml | 2 +- pyproject.toml | 33 ++++ uv.lock | 365 ++++++++++++++++++++++++++++++++++++- 4 files changed, 397 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e9c93e..e712728 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,10 @@ jobs: # not depend on whichever interpreter the runner happens to default to. python-version: "3.12" - - run: uv sync --all-extras + # Named rather than --all-extras: the `analysis` extra holds review tools (bandit, pip-audit, + # pylint) that no CI job runs, so installing them here would be dead weight on every matrix + # entry. `examples` is needed because tests/test_end_to_end.py imports matplotlib. + - run: uv sync --extra dev --extra docs --extra examples --extra cli - name: ruff check run: uv run ruff check --output-format=github . - name: ruff format @@ -50,7 +53,10 @@ jobs: with: enable-cache: true python-version: ${{ matrix.python-version }} - - run: uv sync --all-extras + # Named rather than --all-extras: the `analysis` extra holds review tools (bandit, pip-audit, + # pylint) that no CI job runs, so installing them here would be dead weight on every matrix + # entry. `examples` is needed because tests/test_end_to_end.py imports matplotlib. + - run: uv sync --extra dev --extra docs --extra examples --extra cli - name: pytest run: uv run pytest --cov --cov-report=term-missing --cov-report=xml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a673adc..498c524 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,7 +22,7 @@ jobs: - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: enable-cache: true - - run: uv sync --all-extras + - run: uv sync --extra dev --extra docs # --strict turns warnings, including broken internal links, into failures. - run: uv run mkdocs build --strict - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 diff --git a/pyproject.toml b/pyproject.toml index 934fa62..070f4a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,33 @@ docs = [ "mkdocs-material==9.7.7", "mkdocstrings-python==2.0.5", ] +# Static analysis used during review, kept out of `dev` so the CI jobs that never run it do not +# install it. Pinned exactly, like the rest of the tooling. Nothing here gates a merge: ruff and mypy +# are the enforced checks, and these are for the deeper passes a human asks for. +# +# `osv-scanner` is deliberately absent. The real tool is a Go binary from Google; the PyPI package of +# that name was registered on 2026-07-16 with one release, no author, no home page, and a summary of +# "Reserved name placeholder. No functionality." Install the Go binary if you want it. +# +# `guarddog` is also absent: genuine (DataDog) but first released 2022-11-28, three months *after* +# this project, so it fails the rule that a dependency should predate what it is added to. Add it +# deliberately if the supply-chain checks are wanted, not as part of a batch. +# +# `semgrep` was added and then removed. It pins `mcp==1.23.3` exactly, and that version carries +# PYSEC-2026-3481/3482/3483 (fixed in mcp 1.27.2/1.28.1) -- so the exact pin makes the advisories +# unremediable from here, and 1.172.0 is already the latest semgrep. Against that, bandit reports +# nothing in `src/`, and this is a pure-numpy library with no network, auth, crypto, SQL or +# templating, which is most of what semgrep's rulesets look for. Low coverage gained for three CVEs +# that cannot be patched. Revisit if semgrep relaxes the pin. +analysis = [ + # PyCQA, first released 2015. Security-oriented AST checks. + "bandit==1.9.4", + # PyPA. Reports known CVEs in the resolved dependency set; expected to report none. + "pip-audit==2.10.1", + # PyCQA, first released 2009. Not wired into any gate; here so the findings a contributor's IDE + # reports can be reproduced from the command line. + "pylint==4.0.6", +] # Restores the install set that 0.3.0 pulled in by default, for anyone who was relying on it. examples = [ "matplotlib>=3.8", @@ -205,3 +232,9 @@ exclude_lines = [ # entire content is an ellipsis, which in this package occurs solely in _core/_protocols.py. "^\\s*\\.\\.\\.$", ] + +[tool.bandit] +# `assert` is how a test states its expectation, so B101 fires on every one of them and says nothing. +# Excluded by directory rather than by skipping the check, so B101 still applies to `src/`, where an +# assert *would* be wrong: asserts vanish under `python -O`, so validation must raise instead. +exclude_dirs = ["tests", ".venv", "site", "dist"] diff --git a/uv.lock b/uv.lock index 725a4c8..77c3222 100644 --- a/uv.lock +++ b/uv.lock @@ -58,6 +58,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/40/c53deb2cd0c9b0fb636d24d9f40924cf2e65028e6b20b10cd5c1eeb2c730/ast_serialize-0.6.0-cp39-abi3-win_arm64.whl", hash = "sha256:ccd132fe8db56f61fe743b1f644d01b8d65b83248a8da506f3132bda86d6ed5e", size = 1072965, upload-time = "2026-06-30T20:02:54.097Z" }, ] +[[package]] +name = "astroid" +version = "4.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/63/0adf26577da5eff6eb7a177876c1cfa213856be9926a000f65c4add9692b/astroid-4.0.4.tar.gz", hash = "sha256:986fed8bcf79fb82c78b18a53352a0b287a73817d6dbcfba3162da36667c49a0", size = 406358, upload-time = "2026-02-07T23:35:07.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/cf/1c5f42b110e57bc5502eb80dbc3b03d256926062519224835ef08134f1f9/astroid-4.0.4-py3-none-any.whl", hash = "sha256:52f39653876c7dec3e3afd4c2696920e05c83832b9737afc21928f2d2eb7a753", size = 276445, upload-time = "2026-02-07T23:35:05.344Z" }, +] + [[package]] name = "babel" version = "2.18.0" @@ -89,6 +101,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/cf/e5f9b68a5b0e939a2fb933a66c20180d0c9241bf8927f7a47fa48c1675e9/backrefs-8.0-py314-none-any.whl", hash = "sha256:9ec96efa080938be92323e8e730e57718c9c88eb15ad70bbef4e1766df591408", size = 411903, upload-time = "2026-07-26T19:54:23.221Z" }, ] +[[package]] +name = "bandit" +version = "1.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "stevedore", version = "5.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "stevedore", version = "5.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/c3/0cb80dfe0f3076e5da7e4c5ad8e57bac6ac357ff4a6406205501cade4965/bandit-1.9.4.tar.gz", hash = "sha256:b589e5de2afe70bd4d53fa0c1da6199f4085af666fde00e8a034f152a52cd628", size = 4242677, upload-time = "2026-02-25T06:44:15.503Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/a4/a26d5b25671d27e03afb5401a0be5899d94ff8fab6a698b1ac5be3ec29ef/bandit-1.9.4-py3-none-any.whl", hash = "sha256:f89ffa663767f5a0585ea075f01020207e966a9c0f2b9ef56a57c7963a3f6f8e", size = 134741, upload-time = "2026-02-25T06:44:13.694Z" }, +] + +[[package]] +name = "boolean-py" +version = "5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/cf/85379f13b76f3a69bca86b60237978af17d6aa0bc5998978c3b8cf05abb2/boolean_py-5.0.tar.gz", hash = "sha256:60cbc4bad079753721d32649545505362c754e121570ada4658b852a3a318d95", size = 37047, upload-time = "2025-04-03T10:39:49.734Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/ca/78d423b324b8d77900030fa59c4aa9054261ef0925631cd2501dd015b7b7/boolean_py-5.0-py3-none-any.whl", hash = "sha256:ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9", size = 26577, upload-time = "2025-04-03T10:39:48.449Z" }, +] + +[[package]] +name = "cachecontrol" +version = "0.14.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msgpack" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/f6/c972b32d80760fb79d6b9eeb0b3010a46b89c0b23cf6329417ff7886cd22/cachecontrol-0.14.4.tar.gz", hash = "sha256:e6220afafa4c22a47dd0badb319f84475d79108100d04e26e8542ef7d3ab05a1", size = 16150, upload-time = "2025-11-14T04:32:13.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/79/c45f2d53efe6ada1110cf6f9fca095e4ff47a0454444aefdde6ac4789179/cachecontrol-0.14.4-py3-none-any.whl", hash = "sha256:b7ac014ff72ee199b5f8af1de29d60239954f223e948196fa3d84adaffc71d2b", size = 22247, upload-time = "2025-11-14T04:32:11.733Z" }, +] + +[package.optional-dependencies] +filecache = [ + { name = "filelock" }, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -592,6 +647,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, ] +[[package]] +name = "cyclonedx-python-lib" +version = "11.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "license-expression" }, + { name = "packageurl-python" }, + { name = "py-serializable" }, + { name = "sortedcontainers" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/c9/5d0ccdd19bc7d8ab803b90695c1706aa2ea8529685d18e682dc2524d2630/cyclonedx_python_lib-11.11.0.tar.gz", hash = "sha256:4b3194db72b613717f2912447e67ab618c75ff7dcac6c4af3c0e9e1ac617c102", size = 1442983, upload-time = "2026-06-17T11:57:49.055Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/f3/56ccb2884aaa3db5622368e5191a3384b15f35392aa93df8b2f508c660d2/cyclonedx_python_lib-11.11.0-py3-none-any.whl", hash = "sha256:3049fc83e06a059b5c5907a527625a8ed5073caab10607ed4c9e5503b590fd44", size = 528689, upload-time = "2026-06-17T11:57:47.358Z" }, +] + +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + +[[package]] +name = "dill" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, +] + [[package]] name = "distlib" version = "0.4.3" @@ -612,14 +701,11 @@ wheels = [ [[package]] name = "exceptiongroup" -version = "1.3.1" +version = "1.2.2" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +sdist = { url = "https://files.pythonhosted.org/packages/09/35/2495c4ac46b980e4ca1f6ad6db102322ef3ad2410b79fdde159a4b0f3b92/exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc", size = 28883, upload-time = "2024-07-12T22:26:00.161Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, + { url = "https://files.pythonhosted.org/packages/02/cc/b7e31358aac6ed1ef2bb790a9746ac2c69bcb3c8588b41616914eb106eaf/exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b", size = 16453, upload-time = "2024-07-12T22:25:58.476Z" }, ] [[package]] @@ -814,14 +900,14 @@ wheels = [ [[package]] name = "importlib-metadata" -version = "9.0.0" +version = "8.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "zipp" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, ] [[package]] @@ -833,6 +919,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "isort" +version = "8.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/7c/ec4ab396d31b3b395e2e999c8f46dec78c5e29209fac49d1f4dace04041d/isort-8.0.1.tar.gz", hash = "sha256:171ac4ff559cdc060bcfff550bc8404a486fee0caab245679c2abe7cb253c78d", size = 769592, upload-time = "2026-02-28T10:08:20.685Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/95/c7c34aa53c16353c56d0b802fba48d5f5caa2cdee7958acbcb795c830416/isort-8.0.1-py3-none-any.whl", hash = "sha256:28b89bc70f751b559aeca209e6120393d43fbe2490de0559662be7a9787e3d75", size = 89733, upload-time = "2026-02-28T10:08:19.466Z" }, +] + [[package]] name = "jaraco-classes" version = "3.4.0" @@ -1128,6 +1223,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/5d/3dcec2884ba1b0806d1408612555c38dd5d68e90156b59f75f6e36435c3a/librt-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2f281549a4c52ac7bb97997f14353f8bd0e53a34ca0dad1c905cfd0b4a58ae99", size = 110771, upload-time = "2026-07-08T12:26:12.303Z" }, ] +[[package]] +name = "license-expression" +version = "30.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boolean-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/71/d89bb0e71b1415453980fd32315f2a037aad9f7f70f695c7cec7035feb13/license_expression-30.4.4.tar.gz", hash = "sha256:73448f0aacd8d0808895bdc4b2c8e01a8d67646e4188f887375398c761f340fd", size = 186402, upload-time = "2025-07-22T11:13:32.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/40/791891d4c0c4dab4c5e187c17261cedc26285fd41541577f900470a45a4d/license_expression-30.4.4-py3-none-any.whl", hash = "sha256:421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4", size = 120615, upload-time = "2025-07-22T11:13:31.217Z" }, +] + [[package]] name = "markdown" version = "3.10.2" @@ -1389,6 +1496,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/6d/69552382fcc8e93d1f2763ef2665980a900a48b7f3a4c57ed290726d1cbc/matplotlib-3.11.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4b9ac2f1f607ecda2af90a5232beee2af7582fce1cc30c4b6a1b012dc21ee99", size = 10019439, upload-time = "2026-07-18T03:39:43.78Z" }, ] +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -1531,6 +1647,79 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, ] +[[package]] +name = "msgpack" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/16/f70100614b69feb3ade7285f08c9c52d6cda0a5c03f3f5e2facd63acb211/msgpack-1.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8c7b398c56ff125feae96c2737abfec5595f1fa0aa186df60c56040b8accb95c", size = 82926, upload-time = "2026-06-18T16:12:31.531Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3c/08ecd5cdfe4e2de43aec79062028ad0f7b2d9b1fea5430068c198ba570da/msgpack-1.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1548006a91aa93c5da81f3bdcebc1a0d10cea2d25969754fbe848da622b2b895", size = 82730, upload-time = "2026-06-18T16:12:32.894Z" }, + { url = "https://files.pythonhosted.org/packages/19/9f/a70c9cb1a04ecc134005149367dcfe35d167284e8f65035a1e4156ad17b5/msgpack-1.2.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1dabedcd0f23559f3596428c6589c1cd8c6eaed3a0d720795b07b0225d769203", size = 400729, upload-time = "2026-06-18T16:12:34.052Z" }, + { url = "https://files.pythonhosted.org/packages/fa/7f/5ce020168cf0439041526e95aa068c722c016aee21624e331aeabeee2e8e/msgpack-1.2.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83efa1c898e0fc5380fc0cabbf75164c52e3b5cbb45973710d75821928380c73", size = 407625, upload-time = "2026-06-18T16:12:35.239Z" }, + { url = "https://files.pythonhosted.org/packages/79/70/fb7668ce0386819303047057aef6fc1da73b584291d9cff82b821744e2ef/msgpack-1.2.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:01e2dd6c9b19d333a00282330cc8a73d38d8dabc306dc5b42cd668c3ac82e833", size = 377891, upload-time = "2026-06-18T16:12:36.684Z" }, + { url = "https://files.pythonhosted.org/packages/3d/dc/9ebe654a73c3aed2e40aa6b52e3c2a02b5f53ef0085fa235a45d5b367f87/msgpack-1.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:350cb813d0af6e65d2f7ef0d729f7ff5be5a8bce03665892f43e5883d4ecc1b8", size = 391987, upload-time = "2026-06-18T16:12:37.839Z" }, + { url = "https://files.pythonhosted.org/packages/42/eb/b67cf64218a2fa25e1c671fe1d3dbb06cbeb973e71bc4b822da079862d0b/msgpack-1.2.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:ee1d9ed27d0497b848923746cf762ed2e7db24f4be7eec8e5cbe8c766aa707b7", size = 374603, upload-time = "2026-06-18T16:12:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2e/9ee200cde32fd1a0101b4006202fde554c1860adfb9bf7bff31ea4c08df8/msgpack-1.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:633727297ed063441fd1cda2288865487f33ad14eeb8831afb5f0c396a62cfce", size = 405121, upload-time = "2026-06-18T16:12:40.524Z" }, + { url = "https://files.pythonhosted.org/packages/43/b6/f10117be7ca7a51e8feed699a907b8e663a8cd66e115ae6b4fb30cc7945c/msgpack-1.2.1-cp310-cp310-win32.whl", hash = "sha256:298872ecf9e61950f1c6af4ca969b859ee91783bb920ef6e6172697d0c8aad74", size = 64088, upload-time = "2026-06-18T16:12:41.762Z" }, + { url = "https://files.pythonhosted.org/packages/ba/93/89976c696fb0224662239d952c47b4d1661b34d79a332ef5584facaa8579/msgpack-1.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:2ff164c1b0bcb740b073b99e945234d0212852fa378e44a208c425379140dbeb", size = 70113, upload-time = "2026-06-18T16:12:42.78Z" }, + { url = "https://files.pythonhosted.org/packages/f4/6b/e9b1cdc042c4458801d2545ed782a95f3d6ba8e270cce8745b8603c7f748/msgpack-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:29a3f6e9667868429d8240dfd063ea5ffdc1321c13d783aa23827a38de0dcb22", size = 82812, upload-time = "2026-06-18T16:12:45.022Z" }, + { url = "https://files.pythonhosted.org/packages/0c/3a/dd518a1bf78ed1e9ad8afe57307c079a00eafe4b3068932a27ca1ea56b4f/msgpack-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aded5bdf32609dc7987a49bbbd15a8ef096193f96dd8bbeb791de729e650acf5", size = 82739, upload-time = "2026-06-18T16:12:46.025Z" }, + { url = "https://files.pythonhosted.org/packages/70/e0/7ba9e1542bf0771a27b8b37c1316e3f95ae9d748fd765284655c476ad4ef/msgpack-1.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:146ee4e9ce80b365c6d4c47073da9da7bcec473e58194ceee5dd7620ace77e06", size = 414233, upload-time = "2026-06-18T16:12:47.029Z" }, + { url = "https://files.pythonhosted.org/packages/03/8d/671d81534ea0e2b0e8a121be100020da09eb78861fe3aa8f3ef7dcd3bed1/msgpack-1.2.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a28d076ca7c82b9c8728ad90b7147489449557038bed50e4241eb832395169b4", size = 423843, upload-time = "2026-06-18T16:12:48.19Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b6/e5c737515ed1f166664b87601b532f58cbb73d8aa6a90b99f7c2c5037e8e/msgpack-1.2.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d31c0ac0c640f877804c67cb2bc9f4e23dc2db97e96c2e67fa27d38283b41f8", size = 390772, upload-time = "2026-06-18T16:12:49.624Z" }, + { url = "https://files.pythonhosted.org/packages/a8/46/62ed8c2e87d7021eab19921594d961ef3aa3794eec76c716dc30f3bfd433/msgpack-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ff92d7feeaf5bc26c51495b69e2f99ed97ab79346fb6555f44be7dd2ac6503b", size = 409559, upload-time = "2026-06-18T16:12:50.936Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/59aa3887b860bbf43532835e192b1c388a17590d6068ae4f8b2bc74c906e/msgpack-1.2.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:779197a6513bab3c3632265e3d0f7cb3227e62510841a6f34f1eaa37efbb345e", size = 387838, upload-time = "2026-06-18T16:12:52.161Z" }, + { url = "https://files.pythonhosted.org/packages/09/11/f8563e471093420cf6478cb3271a0175d8402b82d879783d4035d2d03360/msgpack-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:67f6dd22fa72a93752643f07889796d62739a13415ee630169a8ce764f86cf9f", size = 421732, upload-time = "2026-06-18T16:12:53.556Z" }, + { url = "https://files.pythonhosted.org/packages/57/cf/e673683c4c6c90c1022b24c65af4b03eda72b182a1176ef6449069d66acc/msgpack-1.2.1-cp311-cp311-win32.whl", hash = "sha256:91054a783328e0ea7954b8771095705c8d2243b814743fbaadf14552c9c52c5d", size = 64091, upload-time = "2026-06-18T16:12:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/ca212739d179f9083bff2c7c08c24101c3555a334fadc2b876b18768a3ae/msgpack-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2eda0b7ebb1283a98d3e4492ac933c8af6aff59fd3df1c3ed024f536af4b1dc8", size = 70462, upload-time = "2026-06-18T16:12:55.898Z" }, + { url = "https://files.pythonhosted.org/packages/6d/be/6798347b425e26f35db82e69dd83c09716c856a3714e7bffc4c0860fd830/msgpack-1.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:6ee967f7c7e1df2890c671ff2ee51a28ded0efc95da3e507176dee881ce36c66", size = 65059, upload-time = "2026-06-18T16:12:57.053Z" }, + { url = "https://files.pythonhosted.org/packages/bc/dd/9e8cbd8f5582ca4b590336f2b91ee5662f6a6ca562b565abaf696a0f81ff/msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35", size = 83531, upload-time = "2026-06-18T16:12:58.249Z" }, + { url = "https://files.pythonhosted.org/packages/50/2e/ebdb85a8da151397a2790363676b7ed7c125924fe618e4c6d8befb0cc62c/msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c", size = 82657, upload-time = "2026-06-18T16:12:59.396Z" }, + { url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" }, + { url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" }, + { url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b9/8377a5ad8953fc0437c70cc98d9ae29f27fe5ac5109fbec0812085865735/msgpack-1.2.1-cp312-cp312-win32.whl", hash = "sha256:0c0d9802354507bcba62af19c17918e3eb437cc25e6f50657d511b5856a77aac", size = 64504, upload-time = "2026-06-18T16:13:08.822Z" }, + { url = "https://files.pythonhosted.org/packages/57/7f/ce1e377df7e62461fefd9eb23bfb93a4a523f40a517b377b8f844d836828/msgpack-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c24aa15d5963051e1a5c62b12c50cd705992502b5ec1f3bece6046f33c9fc24", size = 71421, upload-time = "2026-06-18T16:13:09.828Z" }, + { url = "https://files.pythonhosted.org/packages/8f/32/ebfe84c9929f08f188d56c7a2fd913406a9ddad76a634697c1c43b8112e6/msgpack-1.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:4227224aaec8f7fbcbfbd4272319347b2bb4030366502600f8c45588c5187b07", size = 64775, upload-time = "2026-06-18T16:13:11.056Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ac/dcddcab6f6c20ecb387ca5e980371cdb3f87ff69aeca388be97eebc4c074/msgpack-1.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a70e3cf2804a300d921bb0940426e35f4e489a23adfb77a808892241db0a064", size = 83151, upload-time = "2026-06-18T16:13:12.173Z" }, + { url = "https://files.pythonhosted.org/packages/64/71/fbcfa83a1d6a9c6091942d1cfd070962244664b87427a9a49a6897b1b219/msgpack-1.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:491cc39455ca765fad51fb451bf2915eb2cf41192ab5801ce8d67c1d614fe056", size = 82351, upload-time = "2026-06-18T16:13:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/e3/10/ddf7b06db879e8792d13934ddda09ff20bd2a583fd84c9b59aae9b0e650b/msgpack-1.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f310233ef7fb9c14e201c93639fe5f5260b005f56f0b29048e999c30935596cc", size = 407518, upload-time = "2026-06-18T16:13:14.233Z" }, + { url = "https://files.pythonhosted.org/packages/79/d3/36a46a8ed992b781acbc05928bd5bee3c810cb0c3563bf81a7b0c04a1a76/msgpack-1.2.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787c9bebb5833e8f6fc8abca3c0597683d8d87f56a8842b6b89c75a5f3176e2d", size = 416405, upload-time = "2026-06-18T16:13:15.435Z" }, + { url = "https://files.pythonhosted.org/packages/f9/84/e8e9598b557c0ba6ddae901a73780a4c75ac667dddf59414b1e56a42fb34/msgpack-1.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dc871b997a9370d855b7394465f2f350e847a5b806dd38dcc9c989e7d87da155", size = 376257, upload-time = "2026-06-18T16:13:17.022Z" }, + { url = "https://files.pythonhosted.org/packages/40/16/738fe6d875ad7e2a9429c165322a4ec088f4f273cdfae63d96a89c467961/msgpack-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:85f57e960d877f2977f6430896191b04a21f8901b3b4baf2e4604329f4db5402", size = 397469, upload-time = "2026-06-18T16:13:18.287Z" }, + { url = "https://files.pythonhosted.org/packages/ca/be/6d5952df75a7f24f35833af764c3a6860780364cb3a0030beb8099e1b2b4/msgpack-1.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1233ee2dd0cefba127583de50ea654677277047d238303521db35def3d7b2e7c", size = 372802, upload-time = "2026-06-18T16:13:19.685Z" }, + { url = "https://files.pythonhosted.org/packages/e1/39/e2ef7dbf0473bcb8dc7c50bf782a892d67414877b63e47fc88eb189ef5e6/msgpack-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3dc2feb0876209d9c38aa56cb1de169bd6c4348f1aa48271f241226590993e6", size = 411273, upload-time = "2026-06-18T16:13:21.028Z" }, + { url = "https://files.pythonhosted.org/packages/ef/c5/133f4512a56e983a93445c836c9d94d88f3bc2e0980ff4b9e577bd8416ce/msgpack-1.2.1-cp313-cp313-win32.whl", hash = "sha256:6d09badf350af2be9d189184e04e64cf54ad93569ab3d96fca58bd3e84aad707", size = 64471, upload-time = "2026-06-18T16:13:22.293Z" }, + { url = "https://files.pythonhosted.org/packages/e2/98/577e10b055096a7dd40732358cabaf7180a20c79ed1dcdbb618e4b9deac7/msgpack-1.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:33f14fba63278b714efe6ad07e50ea5f03d91537aa6a1c5f1ceca4cf44013ca9", size = 71274, upload-time = "2026-06-18T16:13:23.455Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ee/0c0048e7cfbef23c6a94791b8959ab28155232e7956de8a305b5ff588f05/msgpack-1.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc5febcd4c99effbc02b528e49d6fd0760b2b7d48c05239e345a5fa6e743d9a", size = 64795, upload-time = "2026-06-18T16:13:24.687Z" }, + { url = "https://files.pythonhosted.org/packages/77/58/cce442852c6b9e1639c7c8ac8fd9143121cb32dab0f308df4d1426a8eb9c/msgpack-1.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:05f340e47e7e47d2da8db9b53e1bb1d294369e9ef45a747441309f6650b8351d", size = 83610, upload-time = "2026-06-18T16:13:25.724Z" }, + { url = "https://files.pythonhosted.org/packages/60/5c/15b4c7a0182f75ffa90751958ba36a9c01cafee367d49a3edc10ed140b01/msgpack-1.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:810b916696c86ef0deb3b74588480224df4c1b071136c34183e4a2a4284d7ac7", size = 83138, upload-time = "2026-06-18T16:13:26.781Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a6/99e58722feaffc5f2fbcc0c8c0d1451ab9f84097f7af87291b46af2390f4/msgpack-1.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0dacff965c47afdc3749a8469d7302a8f801d6a28758d55120d75e66ce6889", size = 406090, upload-time = "2026-06-18T16:13:28.072Z" }, + { url = "https://files.pythonhosted.org/packages/19/03/8c63e8cf52958534ef688625965ab04c269a6cadd8caef16758b380a821a/msgpack-1.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e2bf9280bceb5efca998435904b5d3e9fdbcc11d90dc9df30aec7973252b720", size = 412106, upload-time = "2026-06-18T16:13:29.427Z" }, + { url = "https://files.pythonhosted.org/packages/63/d2/155d9e71b40e41fd934bc0c48b9b2770f22263e1ac20aad8e29fdca7be3f/msgpack-1.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6c4be5d1c02a42b066ca6ddb71adf36432868fdcdb6ee87e634e86e0674190", size = 374851, upload-time = "2026-06-18T16:13:30.631Z" }, + { url = "https://files.pythonhosted.org/packages/98/48/deaf2326262a8d5ea3295ce9649912ecd3f551ba7ec8e33c665d2ba583f3/msgpack-1.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec0e675d59150a6269ddc9139087c722292664a37d071a849c05c473350f1f2d", size = 396168, upload-time = "2026-06-18T16:13:31.977Z" }, + { url = "https://files.pythonhosted.org/packages/10/2a/b4410f906c2ec0008f1608d3ab5143afc3ad3f4e6da0fed3ea2231d0bef4/msgpack-1.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:dd3bfe82d53edfe4b7fc9a7ec9761e23a7a5b1dac22264505af428253c29ed24", size = 371959, upload-time = "2026-06-18T16:13:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/59/86/1edc67270099a528fa2093ea60fe191233cd238e4bd30cfacf7db79fc959/msgpack-1.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5ad5467fc3f68b5468e06c5f788d712e9f8ffc8b0cd1bcb160c105c1ee92dae7", size = 408457, upload-time = "2026-06-18T16:13:34.567Z" }, + { url = "https://files.pythonhosted.org/packages/82/90/8b630fef07d8c5ab457b71ff2c217910c83d333c7a68472c186e87cc504a/msgpack-1.2.1-cp314-cp314-win32.whl", hash = "sha256:98b58bdb89c46190e4609bb36abe17c6d4105ad13f9c5f8f6f64d320f8ced3fb", size = 65942, upload-time = "2026-06-18T16:13:36.056Z" }, + { url = "https://files.pythonhosted.org/packages/16/f1/467b81e98b24dd3885d7b1857728797b4ffc76a7a7483af4fb321a07de3c/msgpack-1.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:74847557e28ce71bd3c438a447ca90e4b507e997ddbdef8a12a7b283b86c156b", size = 72627, upload-time = "2026-06-18T16:13:37.079Z" }, + { url = "https://files.pythonhosted.org/packages/a7/1d/5d8c4c89985feb6acefb82a09e501c60392261856d2408d20bfe4f0360b1/msgpack-1.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:b50b727bd652bdc37d950336c848ef20ec54a4cafc38dce19b1cd86ad625d0f7", size = 66908, upload-time = "2026-06-18T16:13:38.23Z" }, + { url = "https://files.pythonhosted.org/packages/1b/02/ad2afb678b4de94496cd432b581759b756a92c1192d8c767edd6b132efdc/msgpack-1.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8d00f177ca88a77c1cf848d204a38f249751650b601cb6532acc68805d8a8273", size = 86000, upload-time = "2026-06-18T16:13:39.44Z" }, + { url = "https://files.pythonhosted.org/packages/54/74/0b797484013128837f3b1cbb6cea019277c4de4e377dc512b4d9a0f92940/msgpack-1.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5bb9c386f0a329c035ddbab4b72d1028bf9627add8dda41070288563d57ed1b1", size = 86544, upload-time = "2026-06-18T16:13:40.447Z" }, + { url = "https://files.pythonhosted.org/packages/a9/b4/b774d7eb95561739907fec675582f83203cf41c597a418c2589b4bfb8e9d/msgpack-1.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20466cca18c49c7292a8984bc15d65857b171e7264bdcb5f96baf8be238791fc", size = 427661, upload-time = "2026-06-18T16:13:41.574Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f9/3243191dc9937e00756c8bc1b0272fed8f23758e43df2a3b46f533e5090f/msgpack-1.2.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:196300e7e5d6e74d50f1607ab9c06c4a1484c383cd22defd727902591f7e8dde", size = 426375, upload-time = "2026-06-18T16:13:42.936Z" }, + { url = "https://files.pythonhosted.org/packages/23/c7/1693111db9944ba4ad4b67a1e788400d78a0b6af7a6523dc7e4e58f8274b/msgpack-1.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575957e79cd51903a4e8495a242442949641e08f1efd5197b43bebd3ea7682b4", size = 380495, upload-time = "2026-06-18T16:13:44.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/2b/92f86956a0c13e8662f7e2ad630c4eb4db07497b967589bd5245e018b2c1/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8c2ed1e48cc0f460bf3c7780e7137ff21a4e18433451916f2442c1b21036cd7d", size = 410897, upload-time = "2026-06-18T16:13:45.629Z" }, + { url = "https://files.pythonhosted.org/packages/da/ea/1479f72d200313a76fc2f823a79d1e07ed052ab7b8a0280640aa7b95de42/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f6277e5f783c36786a145e0247fc189a03f35f84b251646e53592d2bc12b355", size = 378519, upload-time = "2026-06-18T16:13:46.998Z" }, + { url = "https://files.pythonhosted.org/packages/f5/4d/fa006060ffa1011d32bfae826fe766fe73e02982183601633b7121058ab3/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9389552ecf4784886345ead0647e4edc96bee37cbab05b75540f542f766c48c", size = 419815, upload-time = "2026-06-18T16:13:48.205Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/aab6c946570496b78e67804721f3d5e2d62a93081b9b37df77764ef56347/msgpack-1.2.1-cp314-cp314t-win32.whl", hash = "sha256:c1c79a604a2969a868a78b6ebd27a887e00c624f14f66b3038e0590cb23332d1", size = 70914, upload-time = "2026-06-18T16:13:49.385Z" }, + { url = "https://files.pythonhosted.org/packages/13/0a/e608956488a2af014cfe6e3d665e090b8ee42aa14b07f8f95b8880d66b09/msgpack-1.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f12038a35fabd52e56a3547bab42401af49a45caa6dd00b34c44de235bc93ee2", size = 77999, upload-time = "2026-06-18T16:13:50.467Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8a/27e2e57055176e366a46b85d02d68e7a5bcfbdd8474c9706375d965f24d3/msgpack-1.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0adcf06ffde0777c0e1a9b771a2b1c4226ba1bbf748c8efcc02fcdeca3299107", size = 71160, upload-time = "2026-06-18T16:13:51.498Z" }, +] + [[package]] name = "mypy" version = "2.3.0" @@ -1863,6 +2052,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, ] +[[package]] +name = "packageurl-python" +version = "0.17.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/d6/3b5a4e3cfaef7a53869a26ceb034d1ff5e5c27c814ce77260a96d50ab7bb/packageurl_python-0.17.6.tar.gz", hash = "sha256:1252ce3a102372ca6f86eb968e16f9014c4ba511c5c37d95a7f023e2ca6e5c25", size = 50618, upload-time = "2025-11-24T15:20:17.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/2f/c7277b7615a93f51b5fbc1eacfc1b75e8103370e786fd8ce2abf6e5c04ab/packageurl_python-0.17.6-py3-none-any.whl", hash = "sha256:31a85c2717bc41dd818f3c62908685ff9eebcb68588213745b14a6ee9e7df7c9", size = 36776, upload-time = "2025-11-24T15:20:16.962Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -2130,6 +2328,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, ] +[[package]] +name = "pip" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/96/e6f8e9d9d7b9cc4457092712a7e919c3186aa2c2fa9ffed2c5d29cc947e8/pip-26.2.tar.gz", hash = "sha256:2d8542afcc84cdd8e846c2b36b2861fad1da376dd98f8e7113e9108a3c331690", size = 1848845, upload-time = "2026-07-29T21:57:56.407Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/36/a3aed958d60531cb442b7ab4596cda7b3621cfb916f8ae1d6769795c7dc1/pip-26.2-py3-none-any.whl", hash = "sha256:931c303696af6fa3417112103b1cad26890e5a07eccb5b99783700e33f2b8aad", size = 1816475, upload-time = "2026-07-29T21:57:54.763Z" }, +] + +[[package]] +name = "pip-api" +version = "0.0.34" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pip" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/f1/ee85f8c7e82bccf90a3c7aad22863cc6e20057860a1361083cd2adacb92e/pip_api-0.0.34.tar.gz", hash = "sha256:9b75e958f14c5a2614bae415f2adf7eeb54d50a2cfbe7e24fd4826471bac3625", size = 123017, upload-time = "2024-07-09T20:32:30.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/f7/ebf5003e1065fd00b4cbef53bf0a65c3d3e1b599b676d5383ccb7a8b88ba/pip_api-0.0.34-py3-none-any.whl", hash = "sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb", size = 120369, upload-time = "2024-07-09T20:32:29.099Z" }, +] + +[[package]] +name = "pip-audit" +version = "2.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachecontrol", extra = ["filecache"] }, + { name = "cyclonedx-python-lib" }, + { name = "packaging" }, + { name = "pip-api" }, + { name = "pip-requirements-parser" }, + { name = "platformdirs" }, + { name = "requests" }, + { name = "rich" }, + { name = "tomli" }, + { name = "tomli-w" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/a4/f21d5f0a0edabcbce31560b73c7c5a6f72ae87af4236fd1069c8f59a353d/pip_audit-2.10.1.tar.gz", hash = "sha256:1eb4565d19ebe5d48996f4b770b4d2b32887e12cb12cfa637f1a064011b55ffc", size = 54275, upload-time = "2026-06-10T22:17:01.744Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/a7/b0c504148114047bd1bc9d97447453c6850ca176bb2f3c0038835994e8b7/pip_audit-2.10.1-py3-none-any.whl", hash = "sha256:99ef3f600a317c1945f1e89e227ef26e1c2d618429b8bd3fa6f4f7c440c4611a", size = 62023, upload-time = "2026-06-10T22:17:00.309Z" }, +] + +[[package]] +name = "pip-requirements-parser" +version = "32.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/2a/63b574101850e7f7b306ddbdb02cb294380d37948140eecd468fae392b54/pip-requirements-parser-32.0.1.tar.gz", hash = "sha256:b4fa3a7a0be38243123cf9d1f3518da10c51bdb165a2b2985566247f9155a7d3", size = 209359, upload-time = "2022-12-21T15:25:22.732Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/d0/d04f1d1e064ac901439699ee097f58688caadea42498ec9c4b4ad2ef84ab/pip_requirements_parser-32.0.1-py3-none-any.whl", hash = "sha256:4659bc2a667783e7a15d190f6fccf8b2486685b6dba4c19c3876314769c57526", size = 35648, upload-time = "2022-12-21T15:25:21.046Z" }, +] + [[package]] name = "platformdirs" version = "4.11.0" @@ -2164,6 +2417,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/49/bc925106abcdac498074f2cbe6137e94e09f418dd2b7775df5b577dc0313/pre_commit-4.6.1-py2.py3-none-any.whl", hash = "sha256:0e3b2942510d1fb34eec167a3ec57331bf8442122f1153a9fb8b58f5c49b2717", size = 226186, upload-time = "2026-07-21T20:56:57.064Z" }, ] +[[package]] +name = "py-serializable" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "defusedxml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/21/d250cfca8ff30c2e5a7447bc13861541126ce9bd4426cd5d0c9f08b5547d/py_serializable-2.1.0.tar.gz", hash = "sha256:9d5db56154a867a9b897c0163b33a793c804c80cee984116d02d49e4578fc103", size = 52368, upload-time = "2025-07-21T09:56:48.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/bf/7595e817906a29453ba4d99394e781b6fabe55d21f3c15d240f85dd06bb1/py_serializable-2.1.0-py3-none-any.whl", hash = "sha256:b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304", size = 23045, upload-time = "2025-07-21T09:56:46.848Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -2182,6 +2447,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pylint" +version = "4.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astroid" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "dill" }, + { name = "isort" }, + { name = "mccabe" }, + { name = "platformdirs" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomlkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/1d/3bb57f303701549550d74bf7ced2b07412be97125c167a0c9d216aa9f762/pylint-4.0.6.tar.gz", hash = "sha256:52f19191bee08bf103f9705ad1a0ece4aa5a0a4ef2bdcbd969375a1e6f6579d5", size = 1585588, upload-time = "2026-06-14T14:43:26.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/da/acb2e7d4dbd2dfb792d38c0d850481f29ad7049b356d23f56c687d35203b/pylint-4.0.6-py3-none-any.whl", hash = "sha256:d11a0e1fdb7b1cd46ec5d6fc78fee8b95f28695b2d6140e5809925f61e32ea54", size = 538389, upload-time = "2026-06-14T14:43:24.873Z" }, +] + [[package]] name = "pymdown-extensions" version = "11.0.1" @@ -2272,6 +2556,11 @@ dependencies = [ ] [package.optional-dependencies] +analysis = [ + { name = "bandit" }, + { name = "pip-audit" }, + { name = "pylint" }, +] cli = [ { name = "tqdm" }, ] @@ -2307,6 +2596,7 @@ examples = [ [package.metadata] requires-dist = [ + { name = "bandit", marker = "extra == 'analysis'", specifier = "==1.9.4" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = "==6.163.0" }, { name = "matplotlib", marker = "extra == 'examples'", specifier = ">=3.8" }, { name = "mkdocs-material", marker = "extra == 'docs'", specifier = "==9.7.7" }, @@ -2317,7 +2607,9 @@ requires-dist = [ { name = "pandas", marker = "python_full_version < '3.11' and extra == 'dev'", specifier = "==2.3.3" }, { name = "pandas", marker = "extra == 'examples'", specifier = ">=2.0" }, { name = "pandas-stubs", marker = "python_full_version >= '3.11' and extra == 'dev'", specifier = "==3.0.3.260530" }, + { name = "pip-audit", marker = "extra == 'analysis'", specifier = "==2.10.1" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = "==4.6.1" }, + { name = "pylint", marker = "extra == 'analysis'", specifier = "==4.0.6" }, { name = "pytest", marker = "extra == 'dev'", specifier = "==9.1.1" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = "==7.1.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.0" }, @@ -2330,7 +2622,7 @@ requires-dist = [ { name = "twine", marker = "extra == 'dev'", specifier = "==7.0.0" }, { name = "types-tqdm", marker = "extra == 'dev'", specifier = "==4.69.0.20260728" }, ] -provides-extras = ["cli", "dev", "docs", "examples"] +provides-extras = ["cli", "dev", "docs", "analysis", "examples"] [[package]] name = "pytz" @@ -2868,6 +3160,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] +[[package]] +name = "stevedore" +version = "5.8.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/88/35e4d27d9177d7df76d060e0a18f69c6c5794c96960c94042e20a12c8ba2/stevedore-5.8.0.tar.gz", hash = "sha256:b49867b32ca3016e94100e68dbf26e72aa7b8708d0a3f73c08aeb220370ac715", size = 514710, upload-time = "2026-05-18T09:15:27.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/ac/19f9941c74add59d17694930ec8105d5eddeee4ce56dd8632b765ca16d6c/stevedore-5.8.0-py3-none-any.whl", hash = "sha256:88eede9e66ca80e34085b9174e2327da2c61ac91f24f70e41c3ad76e4bb4872b", size = 54553, upload-time = "2026-05-18T09:15:25.82Z" }, +] + +[[package]] +name = "stevedore" +version = "5.9.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/d7/dd/04d56c2a5232358df41f3d0f0e31833d378b6c8ed7803a6b1b7867b0eba6/stevedore-5.9.0.tar.gz", hash = "sha256:abbd0af7a38a8bbb1d6adea2e35b17609cf004eaac323e88a8d8963640dd2b3c", size = 514850, upload-time = "2026-07-02T11:38:08.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/8d/008761f6e1000600e5303db30d05724bdcf3d2d186cbb59fac79b52e39ed/stevedore-5.9.0-py3-none-any.whl", hash = "sha256:e520945d4c257700eddc1eb1d79df04b2ea578eef185e0e3fa5b442fc848d3f7", size = 54463, upload-time = "2026-07-02T11:38:07.43Z" }, +] + [[package]] name = "threadpoolctl" version = "3.6.0" @@ -2931,6 +3258,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] +[[package]] +name = "tomli-w" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/96/e07752635b98536177fa1f37671c8f3cdde2e724c6bcf6034b2cfb571565/tomlkit-0.15.1.tar.gz", hash = "sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97", size = 180129, upload-time = "2026-07-17T01:48:04.562Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/bc/8c13eb66537dce1d2bd3a57132902f38d0e7f5bb46fa9f4daed9fe9d76ee/tomlkit-0.15.1-py3-none-any.whl", hash = "sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304", size = 49449, upload-time = "2026-07-17T01:48:05.728Z" }, +] + [[package]] name = "tqdm" version = "4.70.0" From 61955232d01b9159ca8a645947ee92c1f6643c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Moreira=20Souza?= Date: Thu, 30 Jul 2026 09:46:20 -0300 Subject: [PATCH 2/2] feat: save and load a trained map without pickle Until now the only way to keep a trained map was `pickle`, which is arbitrary code execution on load. `save_npz` and `load_npz` write one `.npz` holding the models as an array and everything else as JSON beside them: the configuration, the seed, the generator state and the last training report. A reloaded map continues the same random stream, so training it further produces what never stopping would have produced. Both the seed and the generator's current state are saved, because the seed alone restarts the stream and a resumed run would then diverge from an uninterrupted one without saying so. A test asserts the two are identical, and a second test asserts that re-seeding really would differ, so the first cannot pass with the state discarded. Strategies are saved by name, resolved through registries: DECAY_FUNCTIONS and DISTANCE_FUNCTIONS are added alongside the existing NEIGHBORHOOD_FUNCTIONS, keyed by each function's own name. A map built from the shipped functions round-trips completely. One built with a caller's own function records the name for provenance and raises ArtifactError naming the argument to pass it back through, which the loader accepts. A functools.partial has no name, so it is treated as custom rather than reloaded as the wrapped function with different arguments. The seed and the generator state are separate keys in the metadata. Collapsing them conflates "where the stream began" with "where it has reached", and assigning an inner PCG state where a full bit_generator.state belongs fails with "state must be for a PCG64 RNG". Also adds SOMConfig and a frozen TrainingReport, exposed as `som.config()` and `som.last_report`. `train()` still returns the quantization error. `final_learning_rate` is None for batch training, which has no step size: Eq. (8) is a weighted mean, and reporting the unused initial value would read as though it had been applied. The training loops now return the values they finished on rather than having `train` recompute them, so the report states what ran. On the load path: allow_pickle=False is passed explicitly, names resolve only through the registries so nothing from the file is imported or evaluated, and the member list, metadata JSON, format version, weight dimensions and weight shape are all validated before a map is constructed. Every failure, including a malformed generator state that NumPy rejects, surfaces as ArtifactError. A test builds a payload that really would create a file, proves it executes when unpickled, and shows the loader refusing a file containing it with nothing created. What that does not buy is stated in the docs as plainly as what it does: an `.npz` is a zip, so a hostile file can still attempt resource exhaustion through decompression. `tqdm` is now bound to None when absent rather than left undefined, so a type checker reads the guarded use as reachable instead of possibly-unbound at every call site. The public-method surface test now includes classmethods. It used `inspect.isfunction`, which is False for a classmethod accessed through the class, so it silently ignored `load_npz` and would have ignored any classmethod added later. --- CHANGELOG.md | 41 ++ architecture-profile.toml | 10 +- docs/api.md | 9 + docs/save-and-load.md | 145 +++++++ mkdocs.yml | 1 + src/python_som/__init__.py | 9 +- src/python_som/_artifact.py | 353 ++++++++++++++++ src/python_som/_core/_decay.py | 39 ++ src/python_som/_core/_distance.py | 33 +- src/python_som/_som.py | 248 ++++++++++- src/python_som/_version.py | 12 + tests/test_artifacts.py | 662 ++++++++++++++++++++++++++++++ tests/test_core_boundary.py | 75 +++- 13 files changed, 1612 insertions(+), 25 deletions(-) create mode 100644 docs/save-and-load.md create mode 100644 src/python_som/_artifact.py create mode 100644 src/python_som/_version.py create mode 100644 tests/test_artifacts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 61a5e6b..e2c12b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,47 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **`save_npz` and `load_npz`**, so a trained map can be kept without reaching for `pickle`. One + `.npz` holds the models plus a JSON block with the configuration, the seed, the generator state + and + the last training report. No pickle on either side, so loading one cannot execute code. + + A reloaded map **continues the same random stream**, so training it further gives what never + stopping would have given. Both the seed and the generator's current state are saved: the seed + alone + would restart the stream and a resumed run would quietly diverge from an uninterrupted one. + + Strategies are saved by name and resolved through registries. A map built from the shipped + functions + round-trips completely; one built with your own function records the name for provenance and + raises + `ArtifactError` on load, naming the argument to pass it back through. A `functools.partial` is + treated as custom, so `partial(exponential_decay, factor=3.0)` cannot silently reload as + `factor=2.0`. + + On safety: `allow_pickle=False` is passed explicitly, names resolve only through the registries, + and + the member list, JSON, format version and weight shape are all validated before a map is built. + What + that buys is "cannot execute code", not "safe to load anything" — an `.npz` is a zip, so a hostile + file can still attempt resource exhaustion. `docs/save-and-load.md` states both halves. + +- **`SOM.last_report`**, a frozen `TrainingReport`: mode, iterations, sample count, seed, the final + learning rate and radius, quantization error, the `python_som` and NumPy versions, and wall time. + `train()` still returns the quantization error, so nothing that used the return value changes. + `final_learning_rate` is `None` for batch training, which has no step size — Eq. (8) is a weighted + mean, and reporting the unused initial value would read as though it had been applied. + +- **`SOMConfig`**, the serialisable description of a map, available as `som.config()`. + +- **`DECAY_FUNCTIONS` and `DISTANCE_FUNCTIONS`** registries with `resolve_decay` and + `resolve_distance`, mirroring `NEIGHBORHOOD_FUNCTIONS`. These names are written into artifacts, so + they are public API from 0.4.0. + +- An `analysis` extra with pinned `bandit`, `pip-audit` and `pylint`, for the deeper review passes. + Not wired into any gate; ruff and mypy remain the enforced checks. CI names the extras it needs + rather than using `--all-extras`, so these are not installed on every matrix entry. + - **Enums for every string-valued option**: `TrainingMode`, `Neighborhood`, `WeightInit` and `SampleMode`. Each member *is* a `str`, so `TrainingMode.BATCH` and `"batch"` are interchangeable: equal, same hash, same JSON, usable as the same dictionary key. Nothing that diff --git a/architecture-profile.toml b/architecture-profile.toml index 1639b41..b309097 100644 --- a/architecture-profile.toml +++ b/architecture-profile.toml @@ -11,12 +11,12 @@ # MIT repository. The profile ships anyway, because the declared architecture is worth stating in # the repository rather than only in a plan, and because the local checker needs something to read. # -# It reports 0 ERROR and 13 WARN today, and passes `--strict`. Both WARN kinds are expected, and are -# written down here so a reviewer does not read them as regressions: +# It reports 0 ERROR and 13 WARN today, and passes `--strict`. The provenance marker the preset asks +# for arrived with `TrainingReport` and `save_npz`/`load_npz`, so that WARN is gone. The count is +# unchanged at 13 only by coincidence: the strategy protocols added one function of their own to the +# positional-argument list below, which is the single remaining WARN kind. # -# - No provenance marker. PR #10 adds `TrainingReport`, which is what the preset is asking for. -# -# - Twelve functions take 4 or 5 positional args against a preset limit of 3, with the advice to +# - Thirteen functions take 4 or 5 positional args against a preset limit of 3, with the advice to # "group into a dataclass". Not taken, and the limit is deliberately left at 3 rather than raised # to make the warnings disappear. `gaussian(shape, center, sigma, cyclic)` is four independent # inputs with no natural grouping; a `NeighborhoodParams` holding four fields for one call site diff --git a/docs/api.md b/docs/api.md index e88c80e..93ca9c0 100644 --- a/docs/api.md +++ b/docs/api.md @@ -43,3 +43,12 @@ ## Strategy protocols ::: python_som._core._protocols + +## Artifacts + +::: python_som._artifact + options: + members: + - SOMConfig + - TrainingReport + - ArtifactError diff --git a/docs/save-and-load.md b/docs/save-and-load.md new file mode 100644 index 0000000..ec49a8d --- /dev/null +++ b/docs/save-and-load.md @@ -0,0 +1,145 @@ +# Saving and loading a map + +```python +som.save_npz("iris-map.npz") + +som = python_som.SOM.load_npz("iris-map.npz") +som.train(more_data, n_iteration=50) # continues where it left off +``` + +One file holds the models and everything needed to describe how they were produced. No `pickle` is +involved on either side, so loading one cannot execute code. + +## What is in the file + +An `.npz` with two members: + +| member | contents | +| --- | --- | +| `weights` | the models, `float64`, shape `(x, y, n_features)` | +| `metadata` | a JSON string | + +The metadata is deliberately plain text, so an artifact can be inspected without this package +installed: + +```python +import json, numpy as np + +with np.load("iris-map.npz", allow_pickle=False) as archive: + print(json.loads(str(archive["metadata"]))["report"]) +``` + +```json +{ + "format_version": 1, + "python_som_version": "0.4.0", + "numpy_version": "2.5.1", + "config": { + "shape": [10, 10], "input_len": 4, + "learning_rate": 0.5, "neighborhood_radius": 1.0, "min_neighborhood_radius": 0.5, + "cyclic": [false, false], + "neighborhood_function": "gaussian", + "learning_rate_decay": "asymptotic_decay", + "neighborhood_radius_decay": "asymptotic_decay", + "distance_function": "euclidean_distance" + }, + "rng": { "seed": 42, "state": { "bit_generator": "PCG64", "state": {"state": 1, "inc": 2} } }, + "report": { + "mode": "batch", "n_iteration": 100, "n_samples": 150, "random_seed": 42, + "final_learning_rate": null, "final_neighborhood_radius": 0.5, + "quantization_error": 0.3142, + "python_som_version": "0.4.0", "numpy_version": "2.5.1", "wall_time_seconds": 0.42 + } +} +``` + +`final_learning_rate` is `null` for batch training. Eq. (8) is a weighted mean, so there is no step +size to report; recording the unused initial value would read as though it had been applied. + +## Resuming training + +A reloaded map continues the *same* random stream, so training it further gives what never stopping +would have given. That is why both the seed and the generator's current state are saved: the seed +alone would restart the stream, and a resumed run would quietly diverge from an uninterrupted one. + +```python +# these two end up with identical weights +a = fit(data, iterations=200) + +b = fit(data, iterations=100) +b.save_npz("checkpoint.npz") +python_som.SOM.load_npz("checkpoint.npz").train(data, n_iteration=100) +``` + +## The training report + +`som.last_report` describes the most recent `train()` call, and `None` before the first one. It is +saved with the map, so a figure can be traced back to the run that produced it — the seed, the +iteration count, the error, and the versions of this package and NumPy that computed it. + +`train()` still returns the quantization error, so nothing that used the return value changes. + +## Custom functions + +A map's neighborhood, decays and distance are *callables*, and a callable cannot go into a file +without `pickle`. What gets saved is the **name**, resolved on load through the registries. + +A map built from the shipped functions round-trips completely. One built with your own function +records the name for provenance and refuses to load silently: + +```python +som = python_som.SOM(x=10, y=10, input_len=4, distance_function=cosine) +som.save_npz("custom.npz") + +python_som.SOM.load_npz("custom.npz") +# ArtifactError: this map was saved with custom function(s) that cannot be restored from a name +# (distance_function='cosine'). ... Pass them back explicitly: +# load_npz(path, distance_function=...) + +python_som.SOM.load_npz("custom.npz", distance_function=cosine) # works +``` + +The same applies to a `functools.partial`: it has no name to record, so it is treated as custom. +That is the correct outcome — resolving `partial(exponential_decay, factor=3.0)` by the wrapped +function's name would silently restore a different decay from the one that trained the map. + +The names that *do* resolve are the keys of `NEIGHBORHOOD_FUNCTIONS`, `DECAY_FUNCTIONS` and +`DISTANCE_FUNCTIONS`, each of which is simply the function's own name. + +## How safe is it to load a file from someone else + +Safer than a pickle, and the difference is categorical rather than a matter of degree — but not +unconditionally safe, so here is exactly what is and is not guaranteed. + +**Cannot happen:** + +- **Code execution.** `allow_pickle=False` is passed explicitly, so a crafted object array is + refused by NumPy, not unpickled. The test suite builds a payload that really would create a file, + proves it executes when NumPy is *allowed* to unpickle it, then shows the loader refusing the same + file with nothing created. +- **Loading an arbitrary function by name.** Names resolve only through the registries. Nothing from + the file is imported, evaluated, or looked up in a module. +- **A partly-read file being treated as valid.** The member list, the metadata JSON, the format + version, the weight dimensions and the weight shape against the saved config are all checked + before a map is constructed. Every failure raises `ArtifactError`. + +**Can still happen:** + +- **Resource exhaustion.** An `.npz` is a zip, so a hostile file can be built to decompress to + something enormous. Nothing here caps that. + +So: treat an artifact from an untrusted source the way you would a JPEG from one — it cannot run +code, but it can still be malformed or oversized. Do not treat it as you would a signed archive. + +If you need integrity as well, hash the file and record the digest alongside whatever references it; +that is outside what this format tries to do. + +## Version differences + +Loading an artifact written by a different **major** version is allowed and warns. A major version +is where this package permits numerical results to change, so a map trained under one and reloaded +under another may not reproduce the run its own report describes. Refusing outright would make old +artifacts unreadable, which is the opposite of what provenance is for. + +An artifact written in a newer *format* version is refused, naming the version that wrote it, rather +than being half-understood. diff --git a/mkdocs.yml b/mkdocs.yml index 89e52dc..4758327 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -33,6 +33,7 @@ nav: - Neighborhood functions: neighborhood-functions.md - Training modes: training-modes.md - Options and types: options-and-types.md + - Saving and loading: save-and-load.md - API reference: api.md - Changelog: changelog.md diff --git a/src/python_som/__init__.py b/src/python_som/__init__.py index fb58e81..1bd3f1c 100644 --- a/src/python_som/__init__.py +++ b/src/python_som/__init__.py @@ -21,6 +21,7 @@ from __future__ import annotations +from ._artifact import ArtifactError, SOMConfig, TrainingReport from ._core._decay import ( asymptotic_decay, exponential_decay, @@ -53,20 +54,27 @@ ) from ._som import SOM +# `as` rather than a plain import: the explicit re-export idiom, which tells a type checker +# this is public without putting a dunder into __all__, where the public API lives. +from ._version import __version__ as __version__ + __all__ = [ "NEIGHBORHOOD_FUNCTIONS", "SIGNED_NEIGHBORHOODS", "SOM", + "ArtifactError", "DecayFunction", "DistanceFunction", "KernelFunction", "Neighborhood", "NeighborhoodFunction", "NeighborhoodStr", + "SOMConfig", "SampleMode", "SampleModeStr", "TrainingMode", "TrainingModeStr", + "TrainingReport", "WeightInit", "WeightInitStr", "asymptotic_decay", @@ -79,7 +87,6 @@ "mexican_hat", ] -__version__ = "0.3.0" # Backwards-compatible aliases. Before 0.3.0 these functions lived in the top-level module under # underscore-prefixed names; they were private by convention but reachable, and the README listed diff --git a/src/python_som/_artifact.py b/src/python_som/_artifact.py new file mode 100644 index 0000000..2920732 --- /dev/null +++ b/src/python_som/_artifact.py @@ -0,0 +1,353 @@ +"""Saving and loading a trained map, with the provenance needed to defend a result. + +Wilson et al., *Best Practices for Scientific Computing*: a result should carry its inputs, +parameters and versions. Until 0.4.0 the only way to keep a trained map was ``pickle``, which is +arbitrary code execution on load. The point here is not to forbid that but to make the safe path the +obvious one. + +**One file.** Everything lives in a single ``.npz``: the models as an array, and the metadata as a +JSON string stored alongside them. A separate sidecar was the first design and is worse, because +provenance that can be separated from its artifact will be. + +**What cannot be saved, and what happens instead.** A map holds four callables: the neighborhood, +two decays and the distance. A callable cannot be written to a file without ``pickle``, so what is +stored is its *name*, resolved on load through the registries in :mod:`python_som._core`. A map +built entirely from the shipped functions round-trips completely. One built with a caller's own +function records the name for provenance and refuses to load silently: the loader raises and names +the argument to pass it back through. + +**Security.** ``allow_pickle=False`` is passed explicitly on load, so a crafted file containing an +object array is refused by NumPy rather than executed; strategies resolve only through the +registries, so no name from the file is ever imported or evaluated; the metadata is parsed with +``json.loads``. What that buys is "cannot execute code", not "safe to load anything": an ``.npz`` is +a zip, so a hostile file can still attempt resource exhaustion through decompression. Treat one from +an untrusted source the way you would a JPEG, not the way you would a signed archive. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field, fields +from typing import TYPE_CHECKING, Any + +import numpy as np + +from ._core._decay import DECAY_FUNCTIONS, resolve_decay +from ._core._distance import DISTANCE_FUNCTIONS, resolve_distance +from ._core._neighborhood import NEIGHBORHOOD_FUNCTIONS + +if TYPE_CHECKING: # pragma: no cover + import os + from collections.abc import Callable, Mapping + + import numpy.typing as npt + + from ._core._protocols import DecayFunction, DistanceFunction + +__all__ = [ + "ArtifactError", + "SOMConfig", + "Strategies", + "TrainingReport", + "load_arrays", + "metadata_json", +] + +#: Version of the on-disk layout. Bumped when the schema changes in a way an older reader could +#: misread. A file written by a *newer* version is refused rather than half-understood. +FORMAT_VERSION = 1 + +#: The two members every artifact contains. Checked on load, so a file with unexpected contents is +#: reported rather than silently partly read. +_MEMBERS = ("weights", "metadata") + +#: A weight array is ``(x, y, n_features)``, by definition. +_WEIGHT_DIMENSIONS = 3 + +#: Which config fields name a callable, and the registry that restores each. The loader walks this +#: rather than repeating four near-identical branches. +_STRATEGIES: Mapping[str, Mapping[str, Any]] = { + "neighborhood_function": NEIGHBORHOOD_FUNCTIONS, + "learning_rate_decay": DECAY_FUNCTIONS, + "neighborhood_radius_decay": DECAY_FUNCTIONS, + "distance_function": DISTANCE_FUNCTIONS, +} + + +class ArtifactError(ValueError): + """Raised when a file cannot be read as a python-som artifact. + + A subclass of :class:`ValueError` so that ``except ValueError`` in existing code still catches + it, and distinct so that a caller can tell "this file is not ours" from "this map used a custom + function". + """ + + +@dataclass(frozen=True, slots=True) +class SOMConfig: + """Everything needed to rebuild a map, with strategies recorded by name. + + Frozen, because it describes a map that has already been trained: changing it after the fact + would describe something that never ran. + """ + + #: Grid shape. + shape: tuple[int, int] + #: Number of input features. + input_len: int + #: Initial learning rate. + learning_rate: float + #: Initial neighborhood radius. + neighborhood_radius: float + #: Floor applied to the decayed radius. + min_neighborhood_radius: float + #: Whether each axis wraps around. + cyclic: tuple[bool, bool] + #: Name of the neighborhood function. + neighborhood_function: str + #: Name of the learning-rate decay function. + learning_rate_decay: str + #: Name of the neighborhood-radius decay function. + neighborhood_radius_decay: str + #: Name of the distance function. + distance_function: str + + def unresolvable(self) -> dict[str, str]: + """Return the strategies whose names are not in a registry. + + These are the ones a caller has to pass back in by hand, because a name is all that was + saved and it does not correspond to anything this package can look up. + + :return: Field name to the saved function name, for each strategy that cannot be restored. + """ + return { + attribute: getattr(self, attribute) + for attribute, registry in _STRATEGIES.items() + if getattr(self, attribute) not in registry + } + + +@dataclass(frozen=True, slots=True) +class TrainingReport: + """What one training run did, recorded so a figure can be traced back to it. + + ``wall_time_seconds`` is excluded from equality: it is provenance rather than an input, and two + identical runs should compare equal. + """ + + #: Training mode used. + mode: str + #: Number of iterations run. + n_iteration: int + #: Number of samples presented. + n_samples: int + #: Seed the map was constructed with. + random_seed: int + #: Learning rate at the final step, or None for batch training, which has no step size: Eq. (8) + #: is a weighted mean. + final_learning_rate: float | None + #: Neighborhood radius at the final step. + final_neighborhood_radius: float + #: Mean quantization error after training. + quantization_error: float + #: Version of this package that ran it. + python_som_version: str + #: Version of NumPy it ran against. + numpy_version: str + #: Wall-clock duration. Excluded from equality; see the class docstring. + wall_time_seconds: float = field(compare=False, default=0.0) + + +def metadata_json( + config: SOMConfig, + seed: int, + rng_state: Mapping[str, Any], + report: TrainingReport | None, + version: str, +) -> str: + """Serialise everything that is not an array into the JSON blob stored in the artifact. + + The seed and the generator state are separate keys and are not interchangeable. The seed says + how the stream began, which is provenance; the state says where it has reached, which is what + resumes it. Collapsing both under one ``rng`` key conflates them, and assigning an inner PCG + state where a full ``bit_generator.state`` belongs fails with "state must be for a PCG64 RNG". + + :param config: The map's configuration. + :param seed: The seed the map was constructed with. + :param rng_state: ``Generator.bit_generator.state`` in full, with its ``bit_generator`` key. + :param report: The last training run, if the map has been trained. + :param version: Version of this package. + :return: The JSON text. + """ + return json.dumps( + { + "format_version": FORMAT_VERSION, + "python_som_version": version, + "numpy_version": np.__version__, + "config": asdict(config), + "rng": {"seed": seed, "state": dict(rng_state)}, + "report": asdict(report) if report is not None else None, + }, + indent=2, + sort_keys=True, + ) + + +def load_arrays(path: str | os.PathLike[str]) -> tuple[npt.NDArray[np.floating], dict[str, Any]]: + """Read an artifact into its models and its metadata, validating both. + + Everything that could make a hostile or corrupt file dangerous or confusing is checked here + rather than in the caller: pickle is refused, the member list is pinned, the format version is + compared, and the weights are converted to a known dtype. + + :param path: File to read. + :return: The models, and the parsed metadata. + :raises ArtifactError: If the file is not a readable python-som artifact. + """ + try: + # allow_pickle=False is the security boundary: with it, a crafted object array is refused by + # NumPy instead of being unpickled. Passed explicitly rather than relying on the default, + # because a default is what a future NumPy could change. + with np.load(path, allow_pickle=False) as archive: + present = tuple(archive.files) + arrays = {name: archive[name] for name in present if name in _MEMBERS} + except (OSError, ValueError) as exc: + msg = f"{path} could not be read as a python-som artifact: {exc}" + raise ArtifactError(msg) from exc + + if set(present) != set(_MEMBERS): + msg = ( + f"{path} is not a python-som artifact: expected members {list(_MEMBERS)}, " + f"found {list(present)}" + ) + raise ArtifactError(msg) + + weights = np.asarray(arrays["weights"], dtype=float) + + try: + metadata = json.loads(str(arrays["metadata"])) + except json.JSONDecodeError as exc: + msg = f"{path} has metadata that is not valid JSON: {exc}" + raise ArtifactError(msg) from exc + + written_by = metadata.get("format_version") + if written_by != FORMAT_VERSION: + msg = ( + f"{path} was written in artifact format {written_by!r}, and this version of python-som " + f"reads format {FORMAT_VERSION}. Install the version that wrote it " + f"({metadata.get('python_som_version', 'unknown')}) to read it." + ) + raise ArtifactError(msg) + + if weights.ndim != _WEIGHT_DIMENSIONS: + msg = f"{path} holds weights of shape {weights.shape}, expected three dimensions" + raise ArtifactError(msg) + + return weights, metadata + + +def config_from(metadata: Mapping[str, Any], path: str | os.PathLike[str]) -> SOMConfig: + """Rebuild a :class:`SOMConfig` from parsed metadata. + + JSON has no tuples, so the two tuple fields are converted back explicitly. Missing keys are + reported as a bad artifact rather than raising ``KeyError`` from somewhere further in. + + :param metadata: Parsed metadata. + :param path: File it came from, for the error message. + :return: The configuration. + :raises ArtifactError: If the metadata does not describe a configuration. + """ + raw = metadata.get("config") + if not isinstance(raw, dict): + msg = f"{path} has no 'config' block" + raise ArtifactError(msg) + + expected = {f.name for f in fields(SOMConfig)} + missing = expected - set(raw) + if missing: + msg = f"{path} has a 'config' block missing {sorted(missing)}" + raise ArtifactError(msg) + + values = {key: raw[key] for key in expected} + values["shape"] = (int(values["shape"][0]), int(values["shape"][1])) + values["cyclic"] = (bool(values["cyclic"][0]), bool(values["cyclic"][1])) + return SOMConfig(**values) + + +def report_from(metadata: Mapping[str, Any]) -> TrainingReport | None: + """Rebuild a :class:`TrainingReport` from parsed metadata, if the map was trained. + + :param metadata: Parsed metadata. + :return: The report, or None if the saved map had never been trained. + """ + raw = metadata.get("report") + if not isinstance(raw, dict): + return None + known = {f.name for f in fields(TrainingReport)} + return TrainingReport(**{key: value for key, value in raw.items() if key in known}) + + +@dataclass(frozen=True, slots=True) +class Strategies: + """The four callables a map needs, resolved and individually typed. + + A typed record rather than a ``dict[str, Any]`` so the constructor call in ``load_npz`` is + checked. With a dict, mypy accepts passing the distance where the decay belongs. + """ + + #: Either a registered name or the caller's own callable; the constructor takes either. + neighborhood_function: Any + #: Decay applied to the learning rate. + learning_rate_decay: DecayFunction + #: Decay applied to the neighborhood radius. + neighborhood_radius_decay: DecayFunction + #: Dissimilarity between an input and the models. + distance_function: DistanceFunction + + +def resolve_strategies(config: SOMConfig, overrides: Mapping[str, Any]) -> Strategies: + """Turn the saved strategy names back into callables, honouring any the caller supplied. + + :param config: The saved configuration. + :param overrides: Caller-supplied callables, keyed by config field name. Only entries that are + not None are used. + :return: The four resolved strategies. + :raises ArtifactError: If a strategy cannot be resolved and was not supplied. + """ + supplied = {key: value for key, value in overrides.items() if value is not None} + unresolvable = {key: name for key, name in config.unresolvable().items() if key not in supplied} + if unresolvable: + details = ", ".join(f"{key}={name!r}" for key, name in sorted(unresolvable.items())) + arguments = ", ".join(f"{key}=..." for key in sorted(unresolvable)) + msg = ( + f"this map was saved with custom function(s) that cannot be restored from a name " + f"({details}). A name is all that was saved, because a callable cannot be written to a " + f"file without pickle. Pass them back explicitly: load_npz(path, {arguments})" + ) + raise ArtifactError(msg) + + # Resolved only when not supplied. `supplied.get(key, resolve(...))` would look the wrong way + # round: Python evaluates the default eagerly, so it would resolve a name the caller had already + # overridden -- and raise on exactly the custom function this argument exists to accept. + def either(key: str, resolve: Callable[[str], Any], name: str) -> Any: # noqa: ANN401 + """Return the caller's callable if given, otherwise resolve the saved name. + + :param key: Config field name. + :param resolve: Registry lookup for this kind of strategy. + :param name: The saved name. + :return: The callable to use. + """ + return supplied[key] if key in supplied else resolve(name) + + # The neighborhood is the one the constructor takes by name as well as by callable, so the saved + # name is passed straight through unless the caller overrode it. + return Strategies( + neighborhood_function=supplied.get("neighborhood_function", config.neighborhood_function), + learning_rate_decay=either( + "learning_rate_decay", resolve_decay, config.learning_rate_decay + ), + neighborhood_radius_decay=either( + "neighborhood_radius_decay", resolve_decay, config.neighborhood_radius_decay + ), + distance_function=either("distance_function", resolve_distance, config.distance_function), + ) diff --git a/src/python_som/_core/_decay.py b/src/python_som/_core/_decay.py index 2a50f09..fd82e92 100644 --- a/src/python_som/_core/_decay.py +++ b/src/python_som/_core/_decay.py @@ -12,11 +12,18 @@ from __future__ import annotations +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: # pragma: no cover + from ._protocols import DecayFunction + __all__ = [ + "DECAY_FUNCTIONS", "asymptotic_decay", "exponential_decay", "inverse_decay", "linear_decay", + "resolve_decay", ] @@ -66,3 +73,35 @@ def inverse_decay(x: float, t: int, max_t: int) -> float: :return: Value of ``x`` after ``t`` iterations. """ return (max_t / 100) * x / ((max_t / 100) + t) + + +DECAY_FUNCTIONS: Final[dict[str, DecayFunction]] = { + "asymptotic_decay": asymptotic_decay, + "linear_decay": linear_decay, + "exponential_decay": exponential_decay, + "inverse_decay": inverse_decay, +} +"""Decay functions by name, so a saved map can name the one it used. + +Each key is the function's own name, which is the least surprising mapping and the one a reader can +check against the source without a lookup table. These names are written into artifacts, so they are +public API from 0.4.0 and fixed at 1.0.0. + +A decay function is not required to be in here: a caller may pass any callable. What a name buys is +the ability to restore it from a file, and the loader says so explicitly when it cannot. +""" + + +def resolve_decay(name: str) -> DecayFunction: + """Look up a decay function by name. + + :param name: Name of the decay function. + :return: The corresponding function. + :raises ValueError: If the name is not recognised. + """ + try: + return DECAY_FUNCTIONS[name] + except KeyError as exc: + valid = sorted(DECAY_FUNCTIONS) + msg = f"Unknown decay function {name!r}. Value should be one of {valid}" + raise ValueError(msg) from exc diff --git a/src/python_som/_core/_distance.py b/src/python_som/_core/_distance.py index e9e64a7..41f672d 100644 --- a/src/python_som/_core/_distance.py +++ b/src/python_som/_core/_distance.py @@ -2,10 +2,15 @@ from __future__ import annotations +from typing import TYPE_CHECKING, Final + import numpy as np import numpy.typing as npt -__all__ = ["euclidean_distance"] +if TYPE_CHECKING: # pragma: no cover + from ._protocols import DistanceFunction + +__all__ = ["DISTANCE_FUNCTIONS", "euclidean_distance", "resolve_distance"] def euclidean_distance(a: npt.ArrayLike, b: npt.ArrayLike) -> npt.NDArray[np.floating]: @@ -24,3 +29,29 @@ def euclidean_distance(a: npt.ArrayLike, b: npt.ArrayLike) -> npt.NDArray[np.flo """ result: npt.NDArray[np.floating] = np.linalg.norm(np.subtract(a, b), ord=2, axis=-1) return result + + +DISTANCE_FUNCTIONS: Final[dict[str, DistanceFunction]] = { + "euclidean_distance": euclidean_distance, +} +"""Distance functions by name, so a saved map can name the one it used. + +One entry, because one is what the package ships. The registry exists for the same reason +:data:`DECAY_FUNCTIONS` does -- a name is what makes a map reloadable -- and a dictionary with one +key is the honest shape for that rather than a special case in the loader. +""" + + +def resolve_distance(name: str) -> DistanceFunction: + """Look up a distance function by name. + + :param name: Name of the distance function. + :return: The corresponding function. + :raises ValueError: If the name is not recognised. + """ + try: + return DISTANCE_FUNCTIONS[name] + except KeyError as exc: + valid = sorted(DISTANCE_FUNCTIONS) + msg = f"Unknown distance function {name!r}. Value should be one of {valid}" + raise ValueError(msg) from exc diff --git a/src/python_som/_som.py b/src/python_som/_som.py index 924b847..3cca76a 100644 --- a/src/python_som/_som.py +++ b/src/python_som/_som.py @@ -13,12 +13,23 @@ import logging import secrets +import time import warnings from typing import TYPE_CHECKING, Any, TypeVar import numpy as np import numpy.typing as npt +from ._artifact import ( + ArtifactError, + SOMConfig, + TrainingReport, + config_from, + load_arrays, + metadata_json, + report_from, + resolve_strategies, +) from ._convert import to_numpy from ._core._decay import asymptotic_decay from ._core._distance import euclidean_distance @@ -42,21 +53,33 @@ WeightInit, WeightInitStr, ) +from ._version import __version__ if TYPE_CHECKING: # pragma: no cover + import os from collections import Counter from collections.abc import Iterable + from types import ModuleType from ._convert import DataLike from ._core._neighborhood import NeighborhoodFunction from ._core._protocols import DecayFunction, DistanceFunction +# tqdm is the `cli` extra, so it may legitimately be absent. Bound to None rather than left +# undefined on the failure path: leaving the name unbound reads to a type checker as "possibly +# unbound at every use site", however carefully the use is guarded. The annotation is needed because +# mypy infers `Module` from the successful import and would then reject the None. +_tqdm: ModuleType | None try: - import tqdm + import tqdm as _tqdm_module - TQDM_AVAILABLE = True + _tqdm = _tqdm_module except ImportError: # pragma: no cover - TQDM_AVAILABLE = False + _tqdm = None + +#: Whether a progress bar can be shown. Derived from the import rather than set in both branches, so +#: the flag cannot disagree with whether the module is actually there. +TQDM_AVAILABLE = _tqdm is not None __all__ = ["SOM"] @@ -85,6 +108,41 @@ _IMPLAUSIBLE_LEARNING_RATE = 1.0 +def _name_of(function: object) -> str: + """Return the name a strategy is recorded under in an artifact. + + ``__name__`` for a plain function, which is what the registries are keyed by. Anything without + one -- a ``functools.partial``, a callable object -- falls back to its type name, which still + records what was used even though it cannot be resolved back into a callable. + + :param function: The strategy. + :return: A name for it. + """ + return str(getattr(function, "__name__", type(function).__name__)) + + +def _warn_on_major_version_change(saved: str | None, path: object) -> None: + """Warn when an artifact was written by a different major version of this package. + + Loading it is still allowed: refusing would make old artifacts unreadable, which is the opposite + of what provenance is for. But a major version is exactly where this package permits numerics to + change, so a map trained by one and reloaded under another may not reproduce the run it records. + + :param saved: Version recorded in the artifact, if any. + :param path: File it came from, for the message. + """ + if not saved: + return + if saved.split(".")[0] != __version__.split(".")[0]: + warnings.warn( + f"{path} was written by python-som {saved} and you are running {__version__}. " + "Major versions are where this package allows numerical results to change, so the " + "loaded map may not reproduce the run recorded in its report.", + UserWarning, + stacklevel=3, + ) + + def _validate_learning_rate(learning_rate: float) -> None: """Reject a learning rate that cannot train, and warn about one that is merely unwise. @@ -224,6 +282,10 @@ def __init__( self._rng = np.random.default_rng(self._random_seed) self._weights = random_models(self._shape, self._input_len, self._rng) + #: Set by `train`. None until then, which is what distinguishes an untrained map from one + #: trained with zero effect. + self._last_report: TrainingReport | None = None + # ------------------------------------------------------------------ accessors def get_shape(self) -> tuple[int, int]: @@ -395,15 +457,170 @@ def train( logger.info("Training with %d iterations in %r mode", n_iteration, mode) + started = time.perf_counter() if mode == "batch": - self._train_batch(array, n_iteration, verbose=verbose) + final_alpha, final_sigma = self._train_batch(array, n_iteration, verbose=verbose) else: - self._train_stepwise(array, n_iteration, mode=mode, verbose=verbose) + final_alpha, final_sigma = self._train_stepwise( + array, n_iteration, mode=mode, verbose=verbose + ) + elapsed = time.perf_counter() - started error = self.quantization_error(array) logger.info("Quantization error: %g", error) + + self._last_report = TrainingReport( + mode=str(mode), + n_iteration=n_iteration, + n_samples=len(array), + random_seed=self._random_seed, + final_learning_rate=final_alpha, + final_neighborhood_radius=final_sigma, + quantization_error=float(error), + python_som_version=__version__, + numpy_version=np.__version__, + wall_time_seconds=elapsed, + ) return error + @property + def last_report(self) -> TrainingReport | None: + """What the most recent :meth:`train` call did, or None if it has not been called. + + :return: The report for the last training run. + """ + return self._last_report + + # ------------------------------------------------------------------ artifacts + + def config(self) -> SOMConfig: + """Describe this map in a form that can be written to a file. + + Strategies appear by name. A callable that is not one of the package's registered functions + is recorded as its ``__name__``, which preserves the provenance but cannot be resolved back + into a callable; :meth:`load_npz` says so explicitly rather than guessing. + + :return: The configuration. + """ + return SOMConfig( + shape=self._shape, + input_len=self._input_len, + learning_rate=self._learning_rate, + neighborhood_radius=self._neighborhood_radius, + min_neighborhood_radius=self._min_neighborhood_radius, + cyclic=self._cyclic, + neighborhood_function=str(self._neighborhood_function_name), + learning_rate_decay=_name_of(self._learning_rate_decay), + neighborhood_radius_decay=_name_of(self._neighborhood_radius_decay), + distance_function=_name_of(self._distance_function), + ) + + def save_npz(self, path: str | os.PathLike[str]) -> None: + """Write the models and their provenance to a single ``.npz`` file. + + The file holds the weights as an array and everything else as JSON beside them: the + configuration, the seed, the generator's current state, and the last training report. No + pickle is involved on either side, so the result is safe to load without executing code. + + Saving the generator *state* as well as the seed is what lets :meth:`load_npz` resume the + same random stream. Re-seeding would restart it, and a resumed run would then silently + diverge from an uninterrupted one. + + :param path: Destination file. + """ + np.savez( + path, + weights=self._weights, + metadata=np.array( + metadata_json( + self.config(), + self._random_seed, + self._rng.bit_generator.state, + self._last_report, + __version__, + ) + ), + ) + logger.info("Saved a %s map to %s", "x".join(map(str, self._shape)), path) + + @classmethod + def load_npz( + cls, + path: str | os.PathLike[str], + *, + neighborhood_function: NeighborhoodFunction | None = None, + learning_rate_decay: DecayFunction | None = None, + neighborhood_radius_decay: DecayFunction | None = None, + distance_function: DistanceFunction | None = None, + ) -> SOM: + """Rebuild a map saved by :meth:`save_npz`, models, generator state and all. + + Continuing to train a loaded map produces the same weights as never having stopped, which is + the only useful definition of "loaded" for a stochastic process and is what the saved + generator state is for. + + The four keyword arguments exist for maps trained with a function this package cannot look + up by name. Passing one the file did not need is harmless: it takes precedence over the + registered function of the same role. + + :param path: File to read. + :param neighborhood_function: Replacement for a neighborhood that cannot be resolved. + :param learning_rate_decay: Replacement for a learning-rate decay that cannot be resolved. + :param neighborhood_radius_decay: Replacement for a radius decay that cannot be resolved. + :param distance_function: Replacement for a distance that cannot be resolved. + :return: The reconstructed map. + :raises ArtifactError: If the file is not a readable artifact, or names a function that + cannot be resolved and was not supplied. + """ + weights, metadata = load_arrays(path) + config = config_from(metadata, path) + strategies = resolve_strategies( + config, + { + "neighborhood_function": neighborhood_function, + "learning_rate_decay": learning_rate_decay, + "neighborhood_radius_decay": neighborhood_radius_decay, + "distance_function": distance_function, + }, + ) + + if weights.shape != (*config.shape, config.input_len): + msg = ( + f"{path} holds weights of shape {weights.shape}, which does not match the saved " + f"configuration {(*config.shape, config.input_len)}" + ) + raise ArtifactError(msg) + + rng = metadata.get("rng") or {} + som = cls( + x=config.shape[0], + y=config.shape[1], + input_len=config.input_len, + learning_rate=config.learning_rate, + neighborhood_radius=config.neighborhood_radius, + neighborhood_function=strategies.neighborhood_function, + cyclic_x=config.cyclic[0], + cyclic_y=config.cyclic[1], + random_seed=rng.get("seed"), + min_neighborhood_radius=config.min_neighborhood_radius, + learning_rate_decay=strategies.learning_rate_decay, + neighborhood_radius_decay=strategies.neighborhood_radius_decay, + distance_function=strategies.distance_function, + ) + som._weights = weights + if rng.get("state") is not None: + # NumPy validates the state and raises ValueError on a malformed one ("state must be for + # a PCG64 RNG"). Translated, so that every way a bad file can fail reaches the caller as + # ArtifactError rather than as a message about bit generators from two libraries down. + try: + som._rng.bit_generator.state = rng["state"] + except (ValueError, TypeError, KeyError) as exc: + msg = f"{path} has an unusable random generator state: {exc}" + raise ArtifactError(msg) from exc + som._last_report = report_from(metadata) + _warn_on_major_version_change(metadata.get("python_som_version"), path) + return som + def _sigma(self, t: int, n_iteration: int) -> float: """Return the neighborhood radius at iteration ``t``, floored. @@ -423,13 +640,14 @@ def _progress(iterable: Iterable[_T], total: int, *, verbose: bool) -> Iterable[ :param verbose: Whether progress was requested. :return: The iterable, wrapped or not. """ - if verbose and TQDM_AVAILABLE: - return tqdm.tqdm(iterable, total=total, desc="Training") + if verbose and _tqdm is not None: + wrapped: Iterable[_T] = _tqdm.tqdm(iterable, total=total, desc="Training") + return wrapped return iterable def _train_stepwise( self, array: npt.NDArray[Any], n_iteration: int, *, mode: str, verbose: bool - ) -> None: + ) -> tuple[float | None, float]: """Train one sample at a time, updating the winner and its neighbourhood. Implements Eq. (3) of Kohonen (2013). ``'sequential'`` cycles through the dataset in order, @@ -451,6 +669,8 @@ def _train_stepwise( else: indices = np.resize(np.arange(len(array)), n_iteration) + alpha = self._learning_rate + sigma = self._neighborhood_radius for t, index in enumerate(self._progress(indices, n_iteration, verbose=verbose)): alpha = self._learning_rate_decay(self._learning_rate, t, n_iteration) sigma = self._sigma(t, n_iteration) @@ -459,8 +679,13 @@ def _train_stepwise( self._weights = stepwise_update( self._weights, sample, self.neighborhood(winning, sigma), alpha ) + # Returned rather than recomputed by the caller, so the report states what the loop actually + # used. Recomputing would also add a decay call, which the iteration-count tests measure. + return float(alpha), float(sigma) - def _train_batch(self, array: npt.NDArray[Any], n_iteration: int, *, verbose: bool) -> None: + def _train_batch( + self, array: npt.NDArray[Any], n_iteration: int, *, verbose: bool + ) -> tuple[float | None, float]: """Train with the batch algorithm, updating every model concurrently. Implements Eq. (8) of Kohonen (2013). The winner map is recomputed from the models as they @@ -480,6 +705,7 @@ def _train_batch(self, array: npt.NDArray[Any], n_iteration: int, *, verbose: bo :param verbose: Whether to show a progress bar. """ build_kernel = resolve_kernel(self._neighborhood_function_name) + sigma = self._neighborhood_radius for t in self._progress(range(n_iteration), n_iteration, verbose=verbose): sigma = self._sigma(t, n_iteration) sums, counts = accumulate(array, self._weights, self._shape, self._distance_function) @@ -501,6 +727,10 @@ def neighborhood_of( self._weights = batch_update(self._weights, sums, counts, neighborhood_of, self._shape) + # No learning rate: Eq. (8) is a weighted mean, so there is no step size to report. None + # rather than the unused initial value, which would read as though it had been applied. + return None, float(sigma) + # ------------------------------------------------------------------ initialization def weight_initialization( diff --git a/src/python_som/_version.py b/src/python_som/_version.py new file mode 100644 index 0000000..5e07a34 --- /dev/null +++ b/src/python_som/_version.py @@ -0,0 +1,12 @@ +"""The package version, in one place. + +Separate from ``__init__`` so that :mod:`python_som._som` can record it in a training report without +importing the package it is part of. ``tests/test_packaging.py`` asserts it matches +``pyproject.toml``, which is the other half of the single source of truth. +""" + +from __future__ import annotations + +__all__ = ["__version__"] + +__version__ = "0.3.0" diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py new file mode 100644 index 0000000..ed2ffdd --- /dev/null +++ b/tests/test_artifacts.py @@ -0,0 +1,662 @@ +"""Saving a map, loading it back, and the ways a file can be wrong. + +The claim under test is stronger than "the weights come back". A map is a stochastic process, so a +reloaded map is only *the same map* if continuing to train it produces what never stopping would +have. That is what the saved generator state is for, and it is the assertion that would catch its +absence -- comparing weights alone would pass with the RNG state thrown away. + +The rest of the file is the unhappy paths, because a loader is mostly error handling: a custom +callable that cannot be resolved from a name, a file from the future, a file that is not ours, and a +file crafted to smuggle a pickle. +""" + +from __future__ import annotations + +import io +import json +import pathlib +import pickle +import re +import warnings +import zipfile +from typing import Any + +import numpy as np +import pytest + +import python_som +from python_som import ArtifactError, Neighborhood, SOMConfig, TrainingMode, TrainingReport +from python_som._artifact import FORMAT_VERSION +from python_som._core._decay import DECAY_FUNCTIONS +from python_som._core._distance import DISTANCE_FUNCTIONS + +#: Independent of the fixture seeds; this file is about persistence, not about training quality. +SEED = 20260801 + + +def _data(n_samples: int = 60, n_features: int = 4) -> np.ndarray: + """Build a reproducible dataset. + + :param n_samples: Number of samples. + :param n_features: Number of features. + :return: The dataset. + """ + return np.random.default_rng(SEED).normal(size=(n_samples, n_features)) + + +def _trained(**kwargs: Any) -> python_som.SOM: # noqa: ANN401 + """Build and train a small map. + + :param kwargs: Passed to the constructor. + :return: The trained map. + """ + data = _data() + som = python_som.SOM(x=7, y=5, input_len=4, random_seed=11, **kwargs) + som.weight_initialization(mode="linear", data=data) + som.train(data, n_iteration=20, mode=TrainingMode.BATCH) + return som + + +# --------------------------------------------------------------------------------------------- +# The round trip +# --------------------------------------------------------------------------------------------- + + +def test_weights_survive_exactly(tmp_path: pathlib.Path) -> None: + """Not to a tolerance. The weights are written and read as float64.""" + som = _trained() + path = tmp_path / "map.npz" + som.save_npz(path) + + loaded = python_som.SOM.load_npz(path) + assert np.abs(loaded.get_weights() - som.get_weights()).max() == 0.0 + + +def test_continuing_to_train_a_loaded_map_matches_never_having_stopped( + tmp_path: pathlib.Path, +) -> None: + """The real test of a saved map, and the only one that proves the RNG state is kept. + + Two runs: one trains 20 iterations then another 20; the other trains 20, is saved, reloaded, and + trains 20 more. Stepwise mode is deliberate -- it draws samples from the generator, so a lost + RNG state changes which samples arrive and the weights diverge. Batch mode would pass this test + with the state discarded, since it presents the whole dataset every iteration. + """ + data = _data() + + uninterrupted = python_som.SOM(x=6, y=5, input_len=4, random_seed=3) + uninterrupted.weight_initialization(mode="linear", data=data) + uninterrupted.train(data, n_iteration=20, mode=TrainingMode.RANDOM) + uninterrupted.train(data, n_iteration=20, mode=TrainingMode.RANDOM) + + interrupted = python_som.SOM(x=6, y=5, input_len=4, random_seed=3) + interrupted.weight_initialization(mode="linear", data=data) + interrupted.train(data, n_iteration=20, mode=TrainingMode.RANDOM) + path = tmp_path / "checkpoint.npz" + interrupted.save_npz(path) + resumed = python_som.SOM.load_npz(path) + resumed.train(data, n_iteration=20, mode=TrainingMode.RANDOM) + + difference = np.abs(resumed.get_weights() - uninterrupted.get_weights()).max() + assert difference == 0.0, ( + f"a resumed map diverged from an uninterrupted one by {difference}; the generator " + "state was probably not restored" + ) + + +def test_reseeding_instead_of_restoring_state_would_diverge(tmp_path: pathlib.Path) -> None: + """Guard the test above against passing for the wrong reason. + + If re-seeding happened to give the same stream, the previous test would pass with no state saved + at all and would be worthless. This shows the two really do differ. + """ + data = _data() + som = python_som.SOM(x=6, y=5, input_len=4, random_seed=3) + som.weight_initialization(mode="linear", data=data) + som.train(data, n_iteration=20, mode=TrainingMode.RANDOM) + path = tmp_path / "map.npz" + som.save_npz(path) + + resumed = python_som.SOM.load_npz(path) + reseeded = python_som.SOM.load_npz(path) + reseeded._rng = np.random.default_rng(som.get_random_seed()) + + resumed.train(data, n_iteration=20, mode=TrainingMode.RANDOM) + reseeded.train(data, n_iteration=20, mode=TrainingMode.RANDOM) + assert np.abs(resumed.get_weights() - reseeded.get_weights()).max() > 0.0 + + +def test_the_configuration_survives(tmp_path: pathlib.Path) -> None: + """Every argument that shapes a map, compared as a whole rather than field by field.""" + som = _trained( + neighborhood_function=Neighborhood.BUBBLE, + learning_rate=0.25, + neighborhood_radius=2.5, + cyclic_x=True, + min_neighborhood_radius=0.75, + learning_rate_decay=python_som.linear_decay, + neighborhood_radius_decay=python_som.inverse_decay, + ) + path = tmp_path / "map.npz" + som.save_npz(path) + assert python_som.SOM.load_npz(path).config() == som.config() + + +def test_the_training_report_survives(tmp_path: pathlib.Path) -> None: + """Including across the wall-time field, which is excluded from equality by design.""" + som = _trained() + path = tmp_path / "map.npz" + som.save_npz(path) + + loaded = python_som.SOM.load_npz(path) + assert loaded.last_report == som.last_report + assert loaded.last_report is not None + assert loaded.last_report.quantization_error == pytest.approx(som.quantization_error(_data())) + + +def test_an_untrained_map_round_trips_with_no_report(tmp_path: pathlib.Path) -> None: + """None, not a report full of zeros: the map has not been trained, and should say so.""" + som = python_som.SOM(x=4, y=4, input_len=3, random_seed=5) + assert som.last_report is None + path = tmp_path / "fresh.npz" + som.save_npz(path) + + loaded = python_som.SOM.load_npz(path) + assert loaded.last_report is None + assert np.abs(loaded.get_weights() - som.get_weights()).max() == 0.0 + + +@pytest.mark.parametrize("decay", sorted(DECAY_FUNCTIONS)) +def test_every_registered_decay_round_trips(decay: str, tmp_path: pathlib.Path) -> None: + """A registry entry that cannot be resolved back is worse than no registry entry.""" + function = DECAY_FUNCTIONS[decay] + som = _trained(learning_rate_decay=function, neighborhood_radius_decay=function) + path = tmp_path / f"{decay}.npz" + som.save_npz(path) + + loaded = python_som.SOM.load_npz(path) + assert loaded.config().learning_rate_decay == decay + assert np.abs(loaded.get_weights() - som.get_weights()).max() == 0.0 + + +@pytest.mark.parametrize("name", sorted(DISTANCE_FUNCTIONS)) +def test_every_registered_distance_round_trips(name: str, tmp_path: pathlib.Path) -> None: + som = _trained(distance_function=DISTANCE_FUNCTIONS[name]) + path = tmp_path / f"{name}.npz" + som.save_npz(path) + assert python_som.SOM.load_npz(path).config().distance_function == name + + +@pytest.mark.parametrize("neighborhood", list(Neighborhood)) +def test_every_neighborhood_round_trips(neighborhood: Neighborhood, tmp_path: pathlib.Path) -> None: + """The mexican hat included, which cannot be trained in batch but can certainly be saved.""" + data = _data() + som = python_som.SOM(x=6, y=4, input_len=4, neighborhood_function=neighborhood, random_seed=9) + som.weight_initialization(mode="linear", data=data) + path = tmp_path / f"{neighborhood.value}.npz" + som.save_npz(path) + + loaded = python_som.SOM.load_npz(path) + assert loaded.config().neighborhood_function == neighborhood.value + np.testing.assert_array_equal(loaded.neighborhood((3, 2), 1.5), som.neighborhood((3, 2), 1.5)) + + +# --------------------------------------------------------------------------------------------- +# Custom callables: recorded, refused, and recoverable +# --------------------------------------------------------------------------------------------- + + +def _cosine(a: Any, b: Any) -> np.ndarray: # noqa: ANN401 + """Return one minus the cosine similarity: a distance this package does not ship. + + :param a: Input vector. + :param b: Models. + :return: One minus the cosine similarity. + """ + a_array, b_array = np.asarray(a, dtype=float), np.asarray(b, dtype=float) + dot = np.sum(a_array * b_array, axis=-1) + norms = np.linalg.norm(a_array, axis=-1) * np.linalg.norm(b_array, axis=-1) + return np.asarray(1.0 - dot / np.where(norms == 0, 1.0, norms)) + + +def test_a_custom_callable_is_recorded_by_name(tmp_path: pathlib.Path) -> None: + """Provenance first: the file should say what was used, even though it cannot restore it.""" + som = _trained(distance_function=_cosine) + path = tmp_path / "custom.npz" + som.save_npz(path) + + with np.load(path, allow_pickle=False) as archive: + metadata = json.loads(str(archive["metadata"])) + assert metadata["config"]["distance_function"] == "_cosine" + + +def test_loading_a_custom_callable_raises_and_says_how_to_fix_it(tmp_path: pathlib.Path) -> None: + """The error has to name the argument, because that is the only way out of it.""" + som = _trained(distance_function=_cosine) + path = tmp_path / "custom.npz" + som.save_npz(path) + + with pytest.raises(ArtifactError, match="distance_function") as excinfo: + python_som.SOM.load_npz(path) + assert "load_npz(path, distance_function=...)" in str(excinfo.value) + + +def test_passing_the_callable_back_makes_it_load(tmp_path: pathlib.Path) -> None: + """The other half: an error whose suggested fix does not work is worse than no suggestion.""" + som = _trained(distance_function=_cosine) + path = tmp_path / "custom.npz" + som.save_npz(path) + + loaded = python_som.SOM.load_npz(path, distance_function=_cosine) + assert np.abs(loaded.get_weights() - som.get_weights()).max() == 0.0 + np.testing.assert_array_equal(loaded.activate(_data()[0]), som.activate(_data()[0])) + + +def test_a_partial_is_treated_as_custom(tmp_path: pathlib.Path) -> None: + """``partial(exponential_decay, factor=3.0)`` must not silently reload as ``factor=2.0``. + + A partial has no ``__name__``, so it cannot be resolved, which is the correct outcome. Resolving + it by the wrapped function's name would restore a different decay from the one that trained the + map, and nothing would say so. + """ + import functools # noqa: PLC0415 + + steeper = functools.partial(python_som.exponential_decay, factor=3.0) + som = _trained(learning_rate_decay=steeper) + path = tmp_path / "partial.npz" + som.save_npz(path) + + with pytest.raises(ArtifactError, match="learning_rate_decay"): + python_som.SOM.load_npz(path) + + +# --------------------------------------------------------------------------------------------- +# Bad files +# --------------------------------------------------------------------------------------------- + + +def test_a_real_pickle_payload_does_not_execute(tmp_path: pathlib.Path) -> None: + """The security claim, demonstrated rather than asserted. + + An object array is how a pickle rides inside an ``.npz``, and ``__reduce__`` is how a pickle + executes on load. This proves the payload is live, then shows the loader refusing a file that + contains it, with nothing created. + + Proving liveness first is what matters: without it, this test would pass just as well against an + inert payload and would be evidence of nothing. Liveness is shown through ``pickle`` directly + rather than by asking NumPy to unpickle it, which keeps the enabled form of NumPy's pickle flag + out of this repository entirely. The architecture profile forbids that flag at the artifact + boundary, and a test that had to switch the rule off in order to run would be the worse test. + """ + marker = tmp_path / "executed" + + class Payload: + """Creates ``marker`` if it is ever unpickled.""" + + def __reduce__(self) -> tuple[Any, tuple[Any, ...]]: + """Return the call pickle should make on load.""" + return (pathlib.Path.touch, (marker,)) + + # The payload is live: unpickling it really does run the call. + pickle.loads(pickle.dumps(Payload())) # noqa: S301 that is the point of this line + assert marker.exists(), "the payload is inert, so this test would prove nothing" + marker.unlink() + + # Build an .npz whose `weights` member is that pickled object array. np.save pickles object + # arrays by default, which is exactly the hostile file a user might be handed. + buffer = io.BytesIO() + np.save(buffer, np.array([Payload()], dtype=object)) + metadata = io.BytesIO() + np.save(metadata, np.array(json.dumps({"format_version": FORMAT_VERSION, "config": {}}))) + + hostile = tmp_path / "hostile.npz" + with zipfile.ZipFile(hostile, "w") as archive: + archive.writestr("weights.npy", buffer.getvalue()) + archive.writestr("metadata.npy", metadata.getvalue()) + + with pytest.raises(ArtifactError): + python_som.SOM.load_npz(hostile) + assert not marker.exists(), "load_npz executed a pickled payload" + + +def test_a_file_containing_an_object_array_is_refused(tmp_path: pathlib.Path) -> None: + """The security boundary, exercised rather than asserted about. + + An object array is how a pickle rides inside an ``.npz``. Because the loader passes + ``allow_pickle=False``, NumPy refuses it instead of unpickling, and the loader reports it as a + bad artifact rather than letting the ``ValueError`` escape unexplained. + """ + path = tmp_path / "hostile.npz" + np.savez( + path, + weights=np.array([{"payload": "would be unpickled"}], dtype=object), + metadata=np.array("{}"), + ) + with pytest.raises(ArtifactError): + python_som.SOM.load_npz(path) + + +def test_a_file_from_a_newer_format_is_refused_naming_the_version(tmp_path: pathlib.Path) -> None: + """Refused rather than half-read, and the message says what wrote it.""" + som = _trained() + good = tmp_path / "good.npz" + som.save_npz(good) + with np.load(good, allow_pickle=False) as archive: + weights = archive["weights"] + metadata = json.loads(str(archive["metadata"])) + + metadata["format_version"] = FORMAT_VERSION + 1 + metadata["python_som_version"] = "99.0.0" + future = tmp_path / "future.npz" + np.savez(future, weights=weights, metadata=np.array(json.dumps(metadata))) + + with pytest.raises(ArtifactError, match=re.escape("99.0.0")): + python_som.SOM.load_npz(future) + + +@pytest.mark.parametrize( + "state", + [ + {"bit_generator": "Philox", "state": {"counter": 1}}, + {"bit_generator": "PCG64"}, + "not a mapping at all", + {}, + ], + ids=["wrong-generator", "missing-inner-state", "not-a-mapping", "empty"], +) +def test_an_unusable_generator_state_is_refused_as_an_artifact_error( + state: object, tmp_path: pathlib.Path +) -> None: + """Every way a bad file can fail should reach the caller the same way. + + NumPy raises its own ``ValueError`` here ("state must be for a PCG64 RNG"), which untranslated + would tell the reader about bit generators rather than about their file. + """ + som = _trained() + good = tmp_path / "good.npz" + som.save_npz(good) + with np.load(good, allow_pickle=False) as archive: + weights = archive["weights"] + metadata = json.loads(str(archive["metadata"])) + + metadata["rng"]["state"] = state + broken = tmp_path / "broken-rng.npz" + np.savez(broken, weights=weights, metadata=np.array(json.dumps(metadata))) + + with pytest.raises(ArtifactError, match="unusable random generator state"): + python_som.SOM.load_npz(broken) + + +def test_a_file_with_unexpected_members_is_refused(tmp_path: pathlib.Path) -> None: + """An npz that is not ours should be reported as such, not indexed into and crash.""" + path = tmp_path / "someone-elses.npz" + np.savez(path, something=np.zeros(3)) + with pytest.raises(ArtifactError, match="not a python-som artifact"): + python_som.SOM.load_npz(path) + + +def test_metadata_that_is_not_json_is_refused(tmp_path: pathlib.Path) -> None: + path = tmp_path / "corrupt.npz" + np.savez(path, weights=np.zeros((2, 2, 2)), metadata=np.array("{not json")) + with pytest.raises(ArtifactError, match="not valid JSON"): + python_som.SOM.load_npz(path) + + +def test_a_missing_config_block_is_refused(tmp_path: pathlib.Path) -> None: + path = tmp_path / "no-config.npz" + metadata = {"format_version": FORMAT_VERSION, "python_som_version": "0.4.0"} + np.savez(path, weights=np.zeros((2, 2, 2)), metadata=np.array(json.dumps(metadata))) + with pytest.raises(ArtifactError, match=r"no 'config' block"): + python_som.SOM.load_npz(path) + + +def test_an_incomplete_config_block_names_what_is_missing(tmp_path: pathlib.Path) -> None: + path = tmp_path / "partial-config.npz" + metadata = { + "format_version": FORMAT_VERSION, + "config": {"shape": [2, 2], "input_len": 2}, + } + np.savez(path, weights=np.zeros((2, 2, 2)), metadata=np.array(json.dumps(metadata))) + with pytest.raises(ArtifactError, match="missing"): + python_som.SOM.load_npz(path) + + +def test_weights_that_contradict_the_config_are_refused(tmp_path: pathlib.Path) -> None: + """A shape mismatch is corruption, and constructing the map anyway would hide it.""" + som = _trained() + good = tmp_path / "good.npz" + som.save_npz(good) + with np.load(good, allow_pickle=False) as archive: + metadata = str(archive["metadata"]) + + wrong = tmp_path / "wrong-shape.npz" + np.savez(wrong, weights=np.zeros((3, 3, 3)), metadata=np.array(metadata)) + with pytest.raises(ArtifactError, match="does not match the saved configuration"): + python_som.SOM.load_npz(wrong) + + +def test_two_dimensional_weights_are_refused(tmp_path: pathlib.Path) -> None: + path = tmp_path / "flat.npz" + metadata = {"format_version": FORMAT_VERSION, "config": {}} + np.savez(path, weights=np.zeros((4, 4)), metadata=np.array(json.dumps(metadata))) + with pytest.raises(ArtifactError, match="expected three dimensions"): + python_som.SOM.load_npz(path) + + +def test_a_file_that_is_not_an_npz_at_all_is_refused(tmp_path: pathlib.Path) -> None: + path = tmp_path / "notes.txt" + path.write_text("this is not an archive") + with pytest.raises(ArtifactError, match="could not be read"): + python_som.SOM.load_npz(path) + + +def test_a_missing_file_is_refused(tmp_path: pathlib.Path) -> None: + with pytest.raises(ArtifactError, match="could not be read"): + python_som.SOM.load_npz(tmp_path / "absent.npz") + + +# --------------------------------------------------------------------------------------------- +# What is actually on disk +# --------------------------------------------------------------------------------------------- + + +def test_the_artifact_holds_exactly_two_members_and_no_pickle(tmp_path: pathlib.Path) -> None: + """Read as a plain zip, so the assertion does not depend on NumPy's loader. + + ``.npy`` members only, and no ``.pkl``: the file format itself, not merely the flag we pass when + reading it. + """ + som = _trained() + path = tmp_path / "map.npz" + som.save_npz(path) + + with zipfile.ZipFile(path) as archive: + names = sorted(archive.namelist()) + assert names == ["metadata.npy", "weights.npy"] + + +def test_the_metadata_is_readable_json_with_the_documented_keys(tmp_path: pathlib.Path) -> None: + """The artifact is meant to be inspectable without this package installed.""" + som = _trained() + path = tmp_path / "map.npz" + som.save_npz(path) + + with np.load(path, allow_pickle=False) as archive: + metadata = json.loads(str(archive["metadata"])) + + assert set(metadata) == { + "format_version", + "python_som_version", + "numpy_version", + "config", + "rng", + "report", + } + assert metadata["numpy_version"] == np.__version__ + assert metadata["python_som_version"] == python_som.__version__ + assert metadata["rng"]["seed"] == som.get_random_seed() + + +def test_loading_an_artifact_from_a_different_major_version_warns(tmp_path: pathlib.Path) -> None: + """Allowed, because refusing would make old artifacts unreadable, but not silent. + + A major version is where this package permits numerics to change, so a map trained by one and + reloaded under another may not reproduce the run its own report describes. + """ + som = _trained() + good = tmp_path / "good.npz" + som.save_npz(good) + with np.load(good, allow_pickle=False) as archive: + weights = archive["weights"] + metadata = json.loads(str(archive["metadata"])) + + metadata["python_som_version"] = "99.1.2" + other = tmp_path / "other-major.npz" + np.savez(other, weights=weights, metadata=np.array(json.dumps(metadata))) + + with pytest.warns(UserWarning, match="Major versions"): + python_som.SOM.load_npz(other) + + +# --------------------------------------------------------------------------------------------- +# The value types +# --------------------------------------------------------------------------------------------- + + +def test_config_and_report_are_frozen() -> None: + """Both describe something that already happened, so neither should be editable after.""" + som = _trained() + with pytest.raises((AttributeError, TypeError)): + som.config().input_len = 99 # type: ignore[misc] + report = som.last_report + assert report is not None + with pytest.raises((AttributeError, TypeError)): + report.quantization_error = 0.0 # type: ignore[misc] + + +def test_wall_time_is_excluded_from_report_equality() -> None: + """Two identical runs should compare equal, and wall time is the field that will not match.""" + first, second = _trained().last_report, _trained().last_report + assert first is not None + assert second is not None + assert first == second + assert first.wall_time_seconds > 0.0 + + +def test_unresolvable_reports_only_what_cannot_be_resolved() -> None: + """A map built entirely from shipped functions has nothing to report.""" + assert _trained().config().unresolvable() == {} + assert set(_trained(distance_function=_cosine).config().unresolvable()) == {"distance_function"} + + +def test_batch_training_reports_no_learning_rate() -> None: + """Eq. (8) is a weighted mean: there is no step size, so None rather than an unused number.""" + report = _trained().last_report + assert report is not None + assert report.mode == "batch" + assert report.final_learning_rate is None + + +def test_stepwise_training_reports_the_rate_it_finished_on() -> None: + """Not the rate it started with, which is what a naive report would record.""" + data = _data() + som = python_som.SOM(x=5, y=5, input_len=4, learning_rate=0.5, random_seed=4) + som.train(data, n_iteration=50, mode=TrainingMode.RANDOM) + report = som.last_report + assert report is not None + assert report.final_learning_rate is not None + assert report.final_learning_rate < 0.5 + assert report.final_neighborhood_radius >= 0.5 # the documented floor + + +def test_a_config_can_be_built_by_hand() -> None: + """It is public API, so it has to be constructible without a SOM to hand.""" + config = SOMConfig( + shape=(3, 4), + input_len=2, + learning_rate=0.5, + neighborhood_radius=1.0, + min_neighborhood_radius=0.5, + cyclic=(False, True), + neighborhood_function="gaussian", + learning_rate_decay="asymptotic_decay", + neighborhood_radius_decay="asymptotic_decay", + distance_function="euclidean_distance", + ) + assert config.unresolvable() == {} + assert isinstance(TrainingReport, type) + + +# --------------------------------------------------------------------------------------------- +# The registries +# --------------------------------------------------------------------------------------------- + + +def test_resolve_decay_rejects_an_unknown_name() -> None: + """The error lists the valid names, as the neighborhood resolver does.""" + from python_som._core._decay import resolve_decay # noqa: PLC0415 + + with pytest.raises(ValueError, match="Unknown decay function") as excinfo: + resolve_decay("cosine_annealing") + assert "asymptotic_decay" in str(excinfo.value) + + +def test_resolve_distance_rejects_an_unknown_name() -> None: + from python_som._core._distance import resolve_distance # noqa: PLC0415 + + with pytest.raises(ValueError, match="Unknown distance function") as excinfo: + resolve_distance("manhattan") + assert "euclidean_distance" in str(excinfo.value) + + +def test_every_registry_key_is_what_save_would_record() -> None: + """The invariant that makes a round trip work, stated as the two halves meeting. + + ``save_npz`` records a strategy through ``_name_of``; ``load_npz`` looks the result up in a + registry. So the registry key must be exactly what ``_name_of`` produces for that function -- + checked against the real accessor rather than against ``__name__``, because ``_name_of`` is what + runs and it has a fallback the protocols do not promise. + + A mismatch would restore a *different* function while the file looked correct. + """ + from python_som._som import _name_of # noqa: PLC0415 + + for registry in (DECAY_FUNCTIONS, DISTANCE_FUNCTIONS): + for name, function in registry.items(): + assert _name_of(function) == name, f"{name} is recorded as {_name_of(function)!r}" + + +def test_an_artifact_with_no_recorded_version_does_not_warn(tmp_path: pathlib.Path) -> None: + """A missing version is not a mismatch, so it should pass quietly rather than guess.""" + som = _trained() + good = tmp_path / "good.npz" + som.save_npz(good) + with np.load(good, allow_pickle=False) as archive: + weights = archive["weights"] + metadata = json.loads(str(archive["metadata"])) + + del metadata["python_som_version"] + quiet = tmp_path / "no-version.npz" + np.savez(quiet, weights=weights, metadata=np.array(json.dumps(metadata))) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + python_som.SOM.load_npz(quiet) + + +def test_an_artifact_with_no_rng_state_still_loads(tmp_path: pathlib.Path) -> None: + """Older or hand-built artifacts may carry a seed and no state; the weights still matter.""" + som = _trained() + good = tmp_path / "good.npz" + som.save_npz(good) + with np.load(good, allow_pickle=False) as archive: + weights = archive["weights"] + metadata = json.loads(str(archive["metadata"])) + + metadata["rng"] = {"seed": 11} + stateless = tmp_path / "no-state.npz" + np.savez(stateless, weights=weights, metadata=np.array(json.dumps(metadata))) + + loaded = python_som.SOM.load_npz(stateless) + assert np.abs(loaded.get_weights() - som.get_weights()).max() == 0.0 + assert loaded.get_random_seed() == 11 diff --git a/tests/test_core_boundary.py b/tests/test_core_boundary.py index 7f86549..d1150fd 100644 --- a/tests/test_core_boundary.py +++ b/tests/test_core_boundary.py @@ -235,11 +235,14 @@ def test_nothing_that_0_3_0_exported_has_been_removed() -> None: def test_the_public_surface_is_exactly_this() -> None: """Pin the whole surface, so growing it is a decision rather than an accident. - 0.4.0 adds the enums, their ``Literal`` counterparts, and the strategy protocols. All are - additive: every existing call keeps working, and the enum members are ``str`` subclasses that - compare equal to the strings they replace. + 0.4.0 adds the enums and their ``Literal`` counterparts, the strategy protocols, and the + artifact types. All are additive: every existing call keeps working, and the enum members are + ``str`` subclasses that compare equal to the strings they replace. ``__version__`` is + deliberately absent -- it is re-exported with the ``as`` idiom, since ``__all__`` is the public + API and a dunder is not part of it. """ assert sorted(python_som.__all__) == [ + "ArtifactError", "DecayFunction", "DistanceFunction", "KernelFunction", @@ -249,10 +252,12 @@ def test_the_public_surface_is_exactly_this() -> None: "NeighborhoodStr", "SIGNED_NEIGHBORHOODS", "SOM", + "SOMConfig", "SampleMode", "SampleModeStr", "TrainingMode", "TrainingModeStr", + "TrainingReport", "WeightInit", "WeightInitStr", "asymptotic_decay", @@ -266,26 +271,78 @@ def test_the_public_surface_is_exactly_this() -> None: ] -def test_som_public_methods_are_exactly_what_0_3_0_shipped() -> None: - """Sixteen methods, named as they were before the split.""" +#: Every public method 0.3.0 shipped. None may disappear before 1.0.0. +_METHODS_IN_0_3_0 = frozenset( + { + "activate", + "activation_matrix", + "distance_matrix", + "get_random_seed", + "get_shape", + "get_weights", + "label_map", + "neighborhood", + "quantization", + "quantization_error", + "set_learning_rate", + "set_neighborhood_radius", + "train", + "weight_initialization", + "winner", + "winner_map", + } +) + + +def _public_methods() -> list[str]: + """Return SOM's public method names, classmethods included. + + The predicate is ``isfunction or ismethod`` rather than ``isfunction`` alone. A classmethod + accessed through the class is a *bound method*, so ``isfunction`` is False for it -- which meant + an earlier version of this check silently ignored ``load_npz`` and would have ignored any + classmethod added later. A surface pin with a hole in it is worse than none. + + :return: Sorted names, excluding anything underscore-prefixed. + """ import inspect # noqa: PLC0415 - public = sorted( + return sorted( name - for name, _ in inspect.getmembers(python_som.SOM, inspect.isfunction) - if not name.startswith("_") + for name, member in inspect.getmembers(python_som.SOM) + if not name.startswith("_") and (inspect.isfunction(member) or inspect.ismethod(member)) ) - assert public == [ + + +def test_no_public_method_that_0_3_0_shipped_has_been_removed() -> None: + """The compatibility promise for methods, as a subset check. + + Adding a method is not a breaking change and does not belong in the same assertion as removing + one, which is. + """ + missing = _METHODS_IN_0_3_0 - set(_public_methods()) + assert not missing, f"0.3.0 had these methods and they are gone: {sorted(missing)}" + + +def test_the_public_methods_are_exactly_these() -> None: + """Pin the whole set, so growing it is deliberate. + + 0.4.0 adds ``config``, ``save_npz`` and ``load_npz``. ``last_report`` is a property rather than + a method, so it does not appear here; ``tests/test_artifacts.py`` covers it. + """ + assert _public_methods() == [ "activate", "activation_matrix", + "config", "distance_matrix", "get_random_seed", "get_shape", "get_weights", "label_map", + "load_npz", "neighborhood", "quantization", "quantization_error", + "save_npz", "set_learning_rate", "set_neighborhood_radius", "train",