diff --git a/audmodel/core/api.py b/audmodel/core/api.py index 5451b76..06d1300 100644 --- a/audmodel/core/api.py +++ b/audmodel/core/api.py @@ -434,6 +434,7 @@ def publish( repository: Repository, alias: str | None = None, author: str | None = None, + compression: int = 1, date: datetime.date | None = None, meta: dict[str, object] | None = None, subgroup: str | None = None, @@ -509,6 +510,21 @@ def publish( If provided, the model can be accessed using this alias in addition to its UID author: author name(s), defaults to user name + compression: compression level + of the model archive. + ``0`` stores the model files + without compression, + ``1``-``9`` selects a deflate level. + Deflate is single threaded, + and dominates the publication time + of large models. + Higher levels than ``1`` + hardly compress better + on model weights, + but take at least twice as long. + Select ``0``, + if you want to publish + as fast as possible date: date, defaults to current timestamp meta: dictionary with meta information subgroup: subgroup under which @@ -540,6 +556,7 @@ def publish( FileNotFoundError: if ``root`` folder cannot be found ValueError: if ``alias`` can be confused with an UID, or it does contain chars other than ``[A-Za-z0-9._-]+`` + ValueError: if ``compression`` is not between 0 and 9 Examples: >>> # Assuming your model files are stored under `model_root` @@ -616,6 +633,9 @@ def publish( "and are not allowed to be confused with a model ID." ) + if not 0 <= compression <= 9: + raise ValueError(f"'compression' has to be between 0 and 9, not {compression}.") + if not os.path.isdir(root): raise FileNotFoundError( errno.ENOENT, @@ -662,6 +682,7 @@ def publish( subgroup, root, backend_interface, + compression, verbose, tmp_root=tmp_root, ) diff --git a/audmodel/core/backend.py b/audmodel/core/backend.py index 6023229..568dfba 100644 --- a/audmodel/core/backend.py +++ b/audmodel/core/backend.py @@ -389,6 +389,7 @@ def put_archive( subgroup: str, root: str, backend_interface: audbackend.interface.Maven, + compression: int, verbose: bool, tmp_root: str | None = None, ) -> str: @@ -401,6 +402,11 @@ def put_archive( subgroup: model subgroup root: path to model root folder backend_interface: backend interface instance + compression: compression level + of the model archive. + ``0`` stores the model files + without compression, + ``1``-``9`` selects a deflate level verbose: if ``True`` show message when uploading file tmp_root: folder under which the temporary archive @@ -433,6 +439,7 @@ def put_archive( root, files, src_path, + compression=compression, verbose=verbose, ) with backend_interface.backend: diff --git a/pyproject.toml b/pyproject.toml index 500f4d2..3c65f11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ classifiers = [ requires-python = '>=3.10' dependencies = [ 'audbackend[all] >=3.0.0', + 'audeer >=2.6.0', 'filelock >=3.10', 'oyaml', ] @@ -46,7 +47,6 @@ documentation = 'http://tools.pp.audeering.com/audmodel/' # ===== Dependency groups ================================================= [dependency-groups] dev = [ - 'audeer >=1.3.0', 'parse', 'pytest', 'pytest-cov', diff --git a/tests/test_publish.py b/tests/test_publish.py index b3ea9a6..331b8e8 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -1,4 +1,5 @@ import os +import zipfile import pytest @@ -297,3 +298,68 @@ def test_publish_missing_repository_raises(): "1.0.0", repository=None, ) + + +@pytest.mark.parametrize( + "compression, expected_compress_type", + [ + # deflate level 1 by default + (None, zipfile.ZIP_DEFLATED), + (0, zipfile.ZIP_STORED), + (1, zipfile.ZIP_DEFLATED), + (9, zipfile.ZIP_DEFLATED), + ], +) +def test_publish_compression(tmp_path, compression, expected_compress_type): + r"""Test compression of the published model archive. + + Args: + tmp_path: tmp_path fixture + compression: compression level, + ``None`` publishes without the argument + expected_compress_type: expected compression method + of the entries in the model archive + + """ + root = audeer.mkdir(tmp_path, "model") + with open(os.path.join(root, "model.bin"), "w") as file: + file.write("a" * 10000) + + kwargs = {} if compression is None else {"compression": compression} + effective_compression = 1 if compression is None else compression + uid = audmodel.publish( + root, + pytest.NAME, + {"compression": effective_compression, "compression_arg": compression}, + "1.0.0", + author=pytest.AUTHOR, + date=pytest.DATE, + subgroup=f"{SUBGROUP}.compression", + repository=pytest.REPOSITORIES[0], + **kwargs, + ) + + with zipfile.ZipFile(audmodel.url(uid)) as archive: + infos = archive.infolist() + assert [info.filename for info in infos] == ["model.bin"] + assert infos[0].compress_type == expected_compress_type + + +@pytest.mark.parametrize("compression", [-1, 10]) +def test_publish_compression_error(compression): + r"""Test error for a compression level outside 0-9. + + Args: + compression: invalid compression level + + """ + error_msg = f"'compression' has to be between 0 and 9, not {compression}." + with pytest.raises(ValueError, match=error_msg): + audmodel.publish( + pytest.MODEL_ROOT, + pytest.NAME, + {}, + "1.0.0", + compression=compression, + repository=pytest.REPOSITORIES[0], + )