From 93b0e207417e36257300f9e2c593439618db1dfa Mon Sep 17 00:00:00 2001 From: Johannes Wagner Date: Fri, 4 Sep 2026 10:01:26 +0200 Subject: [PATCH 1/4] Add compression argument to publish() audeer.create_archive() used to hardcode deflate at zlib's default level 6, which is single threaded at around 10 MB/s and thereby accounted for nearly the whole publication time of a model, e.g. 758 s of 776 s for a 7.4 GiB model. audmodel.publish() now exposes the compression level added in audeer 2.6.0, and defaults it to 1. On model weights level 1 compresses as well as level 6, 0.8 pp apart, at 2.4x the speed, and 0 stores the files and makes publication 44x faster than before at the price of a 24% larger archive. Co-Authored-By: Claude Opus 5 (1M context) --- audmodel/core/api.py | 26 ++++++++++++++++ audmodel/core/backend.py | 7 +++++ pyproject.toml | 2 +- tests/test_publish.py | 65 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) diff --git a/audmodel/core/api.py b/audmodel/core/api.py index 5451b76..511305a 100644 --- a/audmodel/core/api.py +++ b/audmodel/core/api.py @@ -434,6 +434,7 @@ def publish( repository: Repository, alias: str | None = None, author: str | None = None, + compression: int = 1, date: datetime.date | None = None, meta: dict[str, object] | None = None, subgroup: str | None = None, @@ -509,6 +510,26 @@ def publish( If provided, the model can be accessed using this alias in addition to its UID author: author name(s), defaults to user name + compression: compression level + of the model archive. + ``0`` stores the model files + without compression, + ``1``-``9`` selects a deflate level. + Deflate is single threaded, + and dominates the publication time + of large models, + while it shrinks + a typical model checkpoint + by around 20% only. + Higher levels than ``1`` + hardly compress better + on model weights, + but take at least twice as long. + Select ``0``, + if you want to publish + as fast as possible, + and neither storage + nor download time matters date: date, defaults to current timestamp meta: dictionary with meta information subgroup: subgroup under which @@ -540,6 +561,7 @@ def publish( FileNotFoundError: if ``root`` folder cannot be found ValueError: if ``alias`` can be confused with an UID, or it does contain chars other than ``[A-Za-z0-9._-]+`` + ValueError: if ``compression`` is not between 0 and 9 Examples: >>> # Assuming your model files are stored under `model_root` @@ -616,6 +638,9 @@ def publish( "and are not allowed to be confused with a model ID." ) + if not 0 <= compression <= 9: + raise ValueError(f"'compression' has to be between 0 and 9, not {compression}.") + if not os.path.isdir(root): raise FileNotFoundError( errno.ENOENT, @@ -663,6 +688,7 @@ def publish( root, backend_interface, verbose, + compression=compression, tmp_root=tmp_root, ) if alias: diff --git a/audmodel/core/backend.py b/audmodel/core/backend.py index 6023229..43de458 100644 --- a/audmodel/core/backend.py +++ b/audmodel/core/backend.py @@ -390,6 +390,7 @@ def put_archive( root: str, backend_interface: audbackend.interface.Maven, verbose: bool, + compression: int = 1, tmp_root: str | None = None, ) -> str: r"""Put archive to backend. @@ -403,6 +404,11 @@ def put_archive( backend_interface: backend interface instance verbose: if ``True`` show message when uploading file + compression: compression level + of the model archive. + ``0`` stores the model files + without compression, + ``1``-``9`` selects a deflate level tmp_root: folder under which the temporary archive is created. If ``None``, @@ -433,6 +439,7 @@ def put_archive( root, files, src_path, + compression=compression, verbose=verbose, ) with backend_interface.backend: diff --git a/pyproject.toml b/pyproject.toml index 500f4d2..3c65f11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ classifiers = [ requires-python = '>=3.10' dependencies = [ 'audbackend[all] >=3.0.0', + 'audeer >=2.6.0', 'filelock >=3.10', 'oyaml', ] @@ -46,7 +47,6 @@ documentation = 'http://tools.pp.audeering.com/audmodel/' # ===== Dependency groups ================================================= [dependency-groups] dev = [ - 'audeer >=1.3.0', 'parse', 'pytest', 'pytest-cov', diff --git a/tests/test_publish.py b/tests/test_publish.py index b3ea9a6..8f49b64 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -1,4 +1,5 @@ import os +import zipfile import pytest @@ -297,3 +298,67 @@ def test_publish_missing_repository_raises(): "1.0.0", repository=None, ) + + +@pytest.mark.parametrize( + "compression, expected_compress_type", + [ + # deflate level 1 by default + (None, zipfile.ZIP_DEFLATED), + (0, zipfile.ZIP_STORED), + (1, zipfile.ZIP_DEFLATED), + (9, zipfile.ZIP_DEFLATED), + ], +) +def test_publish_compression(tmp_path, compression, expected_compress_type): + r"""Test compression of the published model archive. + + Args: + tmp_path: tmp_path fixture + compression: compression level, + ``None`` publishes without the argument + expected_compress_type: expected compression method + of the entries in the model archive + + """ + root = audeer.mkdir(tmp_path, "model") + with open(os.path.join(root, "model.bin"), "w") as file: + file.write("a" * 10000) + + kwargs = {} if compression is None else {"compression": compression} + uid = audmodel.publish( + root, + pytest.NAME, + {"compression": compression}, + "1.0.0", + author=pytest.AUTHOR, + date=pytest.DATE, + subgroup=f"{SUBGROUP}.compression", + repository=pytest.REPOSITORIES[0], + **kwargs, + ) + + with zipfile.ZipFile(audmodel.url(uid)) as archive: + infos = archive.infolist() + assert [info.filename for info in infos] == ["model.bin"] + assert infos[0].compress_type == expected_compress_type + + +@pytest.mark.parametrize("compression", [-1, 10]) +def test_publish_compression_error(compression): + r"""Test error for a compression level outside 0-9. + + Args: + compression: invalid compression level + + """ + error_msg = f"'compression' has to be between 0 and 9, not {compression}." + with pytest.raises(ValueError, match=error_msg): + audmodel.publish( + pytest.MODEL_ROOT, + pytest.NAME, + {}, + "1.0.0", + compression=compression, + repository=pytest.REPOSITORIES[0], + ) From 6061964390c87d90cd88d2b272c065b072412bb3 Mon Sep 17 00:00:00 2001 From: Johannes Wagner Date: Fri, 4 Sep 2026 11:40:41 +0200 Subject: [PATCH 2/4] shorten docstring --- audmodel/core/api.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/audmodel/core/api.py b/audmodel/core/api.py index 511305a..c8651ae 100644 --- a/audmodel/core/api.py +++ b/audmodel/core/api.py @@ -517,19 +517,14 @@ def publish( ``1``-``9`` selects a deflate level. Deflate is single threaded, and dominates the publication time - of large models, - while it shrinks - a typical model checkpoint - by around 20% only. + of large models. Higher levels than ``1`` hardly compress better on model weights, but take at least twice as long. Select ``0``, if you want to publish - as fast as possible, - and neither storage - nor download time matters + as fast as possible date: date, defaults to current timestamp meta: dictionary with meta information subgroup: subgroup under which From 3aede7025a53dc82cd835bee13d4897f0ac62620 Mon Sep 17 00:00:00 2001 From: Johannes Wagner Date: Fri, 4 Sep 2026 11:42:05 +0200 Subject: [PATCH 3/4] Require compression in put_archive() put_archive() is internal, and publish() is its only caller, so the argument does not need a default. This also keeps tmp_root the last argument, and the only one with a default. Co-Authored-By: Claude Opus 5 (1M context) --- audmodel/core/api.py | 2 +- audmodel/core/backend.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/audmodel/core/api.py b/audmodel/core/api.py index c8651ae..06d1300 100644 --- a/audmodel/core/api.py +++ b/audmodel/core/api.py @@ -682,8 +682,8 @@ def publish( subgroup, root, backend_interface, + compression, verbose, - compression=compression, tmp_root=tmp_root, ) if alias: diff --git a/audmodel/core/backend.py b/audmodel/core/backend.py index 43de458..568dfba 100644 --- a/audmodel/core/backend.py +++ b/audmodel/core/backend.py @@ -389,8 +389,8 @@ def put_archive( subgroup: str, root: str, backend_interface: audbackend.interface.Maven, + compression: int, verbose: bool, - compression: int = 1, tmp_root: str | None = None, ) -> str: r"""Put archive to backend. @@ -402,13 +402,13 @@ def put_archive( subgroup: model subgroup root: path to model root folder backend_interface: backend interface instance - verbose: if ``True`` show message - when uploading file compression: compression level of the model archive. ``0`` stores the model files without compression, ``1``-``9`` selects a deflate level + verbose: if ``True`` show message + when uploading file tmp_root: folder under which the temporary archive is created. If ``None``, From 9240f05ad1bda87dabbcaf072954ee7e5c766c8d Mon Sep 17 00:00:00 2001 From: Johannes Wagner Date: Fri, 4 Sep 2026 12:11:47 +0200 Subject: [PATCH 4/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/test_publish.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_publish.py b/tests/test_publish.py index 8f49b64..331b8e8 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -326,10 +326,11 @@ def test_publish_compression(tmp_path, compression, expected_compress_type): file.write("a" * 10000) kwargs = {} if compression is None else {"compression": compression} + effective_compression = 1 if compression is None else compression uid = audmodel.publish( root, pytest.NAME, - {"compression": compression}, + {"compression": effective_compression, "compression_arg": compression}, "1.0.0", author=pytest.AUTHOR, date=pytest.DATE,