From 7c0b06aa1d45f0101f6ebff82987a63357c6d5a4 Mon Sep 17 00:00:00 2001 From: Aaron Meyer Date: Wed, 14 Oct 2026 07:17:53 -0700 Subject: [PATCH] Always write X/raw_X as IVCSC/IVCSR; remove the format option from write_h5ad/write_zarr --- README.md | 7 ++++--- docs/usage.md | 5 ++--- src/vsparse/_anndata_class.py | 30 ++++++++++-------------------- src/vsparse/_rapid_load.py | 4 ++-- tests/test_anndata_class.py | 10 ++-------- tests/test_index_dtypes.py | 9 ++++----- tests/test_metadata_encoding.py | 8 +++----- tests/test_rapid_load.py | 12 ++++++------ 8 files changed, 33 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index d0dc2ae..323866d 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ va.X # VCSCArray va.raw_X # VCSCArray (separate from anndata's .raw) # Persist to HDF5 (.h5ad) or Zarr with default Blosc2+LZ4 compression +# (stored as IVCSC/IVCSR, see below) va.write_h5ad("compressed.h5ad") # Read back with VCSCAnnData.read_h5ad va2 = vsparse.VCSCAnnData.read_h5ad("compressed.h5ad") @@ -121,13 +122,13 @@ plain = va.to_anndata() ### Byte-Packed On-Disk Format (IVCSC / IVCSR) -For smaller files, pass `format="ivcsc"` (or `"ivcsr"`) on write. This byte-packs the minor-axis indices with delta + varint encoding (inspired by [IVSparse's IVCSC](https://github.com/Seth-Wolfgang/IVSparse)). +`write_h5ad` / `write_zarr` always store `X`/`raw_X` as IVCSC (from VCSC) or IVCSR (from VCSR). This byte-packs the minor-axis indices with delta + varint encoding (inspired by [IVSparse's IVCSC](https://github.com/Seth-Wolfgang/IVSparse)). It is purely an archival storage format: `read_h5ad` / `read_zarr` decompress the indices on load and return a standard `VCSCArray`/`VCSRArray`. ```python -# Write delta+varint byte-packed indices -va.write_h5ad("archived.h5ad", format="ivcsc") +# Writes delta+varint byte-packed indices +va.write_h5ad("archived.h5ad") # Reads back directly as an ordinary VCSCAnnData va_loaded = vsparse.VCSCAnnData.read_h5ad("archived.h5ad") diff --git a/docs/usage.md b/docs/usage.md index b26039d..4fdfb88 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -89,8 +89,7 @@ allowlist of types, so a plain `AnnData` cannot hold a `VCSCArray` in `X`. zarr). Pass `dataset_kwargs={}` to write uncompressed, or your own `dataset_kwargs` to use a different codec. -`X`/`raw_X` are stored in the VCSC/VCSR layout by default (`format="vcsc"`). -Passing `format="ivcsc"` instead stores them as **IVCSC/IVCSR** -- the same +`X`/`raw_X` are always stored as **IVCSC/IVCSR** -- the VCSC/VCSR layout, but with the minor-axis `indices` array delta+varint byte-packed (inspired by [IVSparse's IVCSC](https://github.com/Seth-Wolfgang/IVSparse)). This is a *file-storage-only* format: it trades extra CPU on write/read for a @@ -99,7 +98,7 @@ smaller file, and there is no in-memory IVCSC array type to compute with -- decompressed from IVCSC/IVCSR immediately on load. ```python -va.write_h5ad("archived.h5ad", format="ivcsc") +va.write_h5ad("archived.h5ad") va4 = vsparse.VCSCAnnData.read_h5ad("archived.h5ad") # va4.X is a plain VCSCArray ``` `VCSCAnnData` works around this by overriding the `X` property; as a consequence, diff --git a/src/vsparse/_anndata_class.py b/src/vsparse/_anndata_class.py index 4519b31..98b6137 100644 --- a/src/vsparse/_anndata_class.py +++ b/src/vsparse/_anndata_class.py @@ -27,7 +27,6 @@ _DF_KEYS = ("obs", "var") _MAPPING_KEYS = ("obsm", "varm", "obsp", "varp", "layers", "uns") _FIELD_KEYS = (*_DF_KEYS, *_MAPPING_KEYS) -_STORE_FORMATS = ("vcsc", "ivcsc") # Names statistics are recorded under in .obs/.varm/.uns -- see .normalized(). _VSPARSE_UNS_KEY = "vsparse" @@ -419,21 +418,17 @@ def _write_group( self, g: Any, *, - format: str = "vcsc", convert_strings_to_categoricals: bool = True, dataset_kwargs: Mapping[str, Any] = MappingProxyType({}), ) -> None: - if format not in _STORE_FORMATS: - raise ValueError(f"format must be one of {_STORE_FORMATS}, got {format!r}") if convert_strings_to_categoricals: # Writing field-by-field skips what anndata's own writers do here, # leaving low-cardinality columns as one string per row. self.strings_to_categoricals() - write_array = ad.io.write_elem if format == "vcsc" else _io.write_ivcs_elem if self._vcs_X is not None: - write_array(g, "X", self._vcs_X, dataset_kwargs=dataset_kwargs) + _io.write_ivcs_elem(g, "X", self._vcs_X, dataset_kwargs=dataset_kwargs) if self._vcs_raw_X is not None: - write_array(g, "raw_X", self._vcs_raw_X, dataset_kwargs=dataset_kwargs) + _io.write_ivcs_elem(g, "raw_X", self._vcs_raw_X, dataset_kwargs=dataset_kwargs) for key in _DF_KEYS: ad.io.write_elem(g, key, getattr(self, key), dataset_kwargs=dataset_kwargs) for key in _MAPPING_KEYS: @@ -455,22 +450,20 @@ def write_h5ad( # ty: ignore[invalid-method-override] self, filename: str | PathLike[str], *, - format: str = "vcsc", convert_strings_to_categoricals: bool = True, dataset_kwargs: Mapping[str, Any] | None = None, **_kwargs: Any, ) -> None: """Write to ``.h5ad``. Read back with :meth:`read_h5ad`. + ``X``/``raw_X`` are always stored as IVCSC (from VCSC) or IVCSR (from + VCSR): the minor-axis indices are byte-packed (delta + varint + encoding), at the cost of extra work on write/read. They come back from + :meth:`read_h5ad` as ordinary VCSCArray/VCSRArray objects -- IVCSC/IVCSR + is purely an on-disk storage format. + Parameters ---------- - format - ``"vcsc"`` (default) stores ``X``/``raw_X`` with plain int arrays - for the minor-axis indices. ``"ivcsc"`` (IVCSC/IVCSR) instead - byte-packs them (delta + varint encoding) for a smaller file, at - the cost of extra work on write/read. Either way, ``X``/``raw_X`` - come back from :meth:`read_h5ad` as ordinary VCSCArray/VCSRArray - objects -- ``"ivcsc"`` is purely an on-disk storage format. convert_strings_to_categoricals Convert ``obs``/``var`` string columns to categorical in place before writing, as ``anndata``'s own writers do. Only columns @@ -491,7 +484,6 @@ def write_h5ad( # ty: ignore[invalid-method-override] ): self._write_group( f, - format=format, convert_strings_to_categoricals=convert_strings_to_categoricals, dataset_kwargs=dataset_kwargs, ) @@ -508,15 +500,14 @@ def write_zarr( self, store: Any, *, - format: str = "vcsc", convert_strings_to_categoricals: bool = True, dataset_kwargs: Mapping[str, Any] | None = None, **_kwargs: Any, ) -> None: """Write to a zarr store. Read back with :meth:`read_zarr`. - See :meth:`write_h5ad` for ``format``/``convert_strings_to_categoricals``/ - ``dataset_kwargs`` (including the numeric-only compression behavior); + See :meth:`write_h5ad` for the IVCSC/IVCSR storage format and + ``convert_strings_to_categoricals``/``dataset_kwargs`` (including the numeric-only compression behavior); the default compression here is Blosc+LZ4 via ``numcodecs``. """ import zarr @@ -527,7 +518,6 @@ def write_zarr( f = zarr.open_group(store, mode="w") self._write_group( f, - format=format, convert_strings_to_categoricals=convert_strings_to_categoricals, dataset_kwargs=dataset_kwargs, ) diff --git a/src/vsparse/_rapid_load.py b/src/vsparse/_rapid_load.py index 19d8a3e..aa0455d 100644 --- a/src/vsparse/_rapid_load.py +++ b/src/vsparse/_rapid_load.py @@ -487,7 +487,7 @@ def load_and_normalize( ---------- path Path to an ``.h5ad`` file whose ``X`` (or ``layers[x_key]``) was - written with ``format="ivcsc"``/``"ivcsr"`` (see + stored as IVCSC/IVCSR (see :meth:`~vsparse.VCSCAnnData.write_h5ad`). min_cell_counts Cells with total raw counts <= this are dropped. @@ -594,7 +594,7 @@ def load_packed(path: str | PathLike[str], *, x_key: str = "X") -> VCSCAnnData: ---------- path Path to an ``.h5ad`` file whose ``X`` (or a top-level group named - ``x_key``) was written with ``format="ivcsc"``/``"ivcsr"`` (see + ``x_key``) was stored as IVCSC/IVCSR (see :meth:`~vsparse.VCSCAnnData.write_h5ad`). x_key Top-level h5ad group holding the IVCSR/IVCSC array (``"X"`` by default). diff --git a/tests/test_anndata_class.py b/tests/test_anndata_class.py index 141ef17..85b7bb3 100644 --- a/tests/test_anndata_class.py +++ b/tests/test_anndata_class.py @@ -111,7 +111,7 @@ def test_zarr_roundtrip(base_adata, dense, tmp_path): def test_h5ad_ivcsc_roundtrip(base_adata, dense, tmp_path): va = VCSCAnnData.from_anndata(base_adata) path = tmp_path / "test_ivcsc.h5ad" - va.write_h5ad(path, format="ivcsc") + va.write_h5ad(path) import h5py @@ -130,19 +130,13 @@ def test_h5ad_ivcsc_roundtrip(base_adata, dense, tmp_path): def test_zarr_ivcsc_roundtrip(base_adata, dense, tmp_path): va = VCSCAnnData.from_anndata(base_adata, format="csr") path = tmp_path / "test_ivcsr.zarr" - va.write_zarr(path, format="ivcsc") + va.write_zarr(path) read_back = VCSCAnnData.read_zarr(path) assert isinstance(read_back.X, VCSRArray) np.testing.assert_allclose(read_back.X.to_scipy().toarray(), dense) -def test_write_h5ad_rejects_bad_format(base_adata, tmp_path): - va = VCSCAnnData.from_anndata(base_adata) - with pytest.raises(ValueError): - va.write_h5ad(tmp_path / "bad.h5ad", format="bogus") - - def test_write_h5ad_default_compression(base_adata, tmp_path): va = VCSCAnnData.from_anndata(base_adata) path = tmp_path / "compressed.h5ad" diff --git a/tests/test_index_dtypes.py b/tests/test_index_dtypes.py index 8291244..8b89d53 100644 --- a/tests/test_index_dtypes.py +++ b/tests/test_index_dtypes.py @@ -98,13 +98,12 @@ def test_derived_arrays_stay_narrow(vcls, dense, op): assert op(v, dense).indices.dtype == np.int32 -@pytest.mark.parametrize("fmt", ["vcsc", "ivcsc"]) -def test_roundtrip_preserves_values_and_stores_narrow_indices(tmp_path, dense, fmt): +def test_roundtrip_preserves_values_and_stores_narrow_indices(tmp_path, dense): import anndata as ad va = VCSCAnnData.from_anndata(ad.AnnData(X=sp.csr_array(dense)), format="csr") - path = tmp_path / f"data.{fmt}.h5ad" - va.write_h5ad(path, format=fmt) + path = tmp_path / "data.h5ad" + va.write_h5ad(path) back = VCSCAnnData.read_h5ad(path) assert isinstance(back.X, VCSRArray) @@ -121,7 +120,7 @@ def test_packed_write_narrows_a_wide_in_memory_array(tmp_path, dense): va.X.indices = va.X.indices.astype(np.int64) path = tmp_path / "wide.h5ad" - va.write_h5ad(path, format="ivcsc") + va.write_h5ad(path) back = VCSCAnnData.read_h5ad(path) assert isinstance(back.X, VCSRArray) diff --git a/tests/test_metadata_encoding.py b/tests/test_metadata_encoding.py index db17da9..bc315de 100644 --- a/tests/test_metadata_encoding.py +++ b/tests/test_metadata_encoding.py @@ -3,7 +3,6 @@ import anndata as ad import numpy as np import pandas as pd -import pytest import scipy.sparse as sp from vsparse import VCSCAnnData, VCSRArray @@ -31,13 +30,12 @@ def _adata(n_cells: int = 400, n_genes: int = 20) -> ad.AnnData: return ad.AnnData(X=X, obs=obs, var=var) -@pytest.mark.parametrize("fmt", ["vcsc", "ivcsc"]) -def test_string_columns_are_written_as_categoricals(tmp_path, fmt): +def test_string_columns_are_written_as_categoricals(tmp_path): va = VCSCAnnData.from_anndata(_adata(), format="csr") assert not isinstance(va.obs["cell_type"].dtype, pd.CategoricalDtype) - path = tmp_path / f"data.{fmt}.h5ad" - va.write_h5ad(path, format=fmt) + path = tmp_path / "data.h5ad" + va.write_h5ad(path) back = VCSCAnnData.read_h5ad(path) assert isinstance(back.obs["cell_type"].dtype, pd.CategoricalDtype) diff --git a/tests/test_rapid_load.py b/tests/test_rapid_load.py index 3f5ee22..989aa65 100644 --- a/tests/test_rapid_load.py +++ b/tests/test_rapid_load.py @@ -42,7 +42,7 @@ def _write_ivcsr(tmp_path, dense: np.ndarray, name="data.ivcsr.h5ad"): adata = ad.AnnData(X=sp.csr_array(dense)) vad = VCSCAnnData.from_anndata(adata, format="csr") path = tmp_path / name - vad.write_h5ad(path, format="ivcsc") + vad.write_h5ad(path) return path @@ -126,7 +126,7 @@ def test_min_cells_uses_obs_filtered_cohort(tmp_path): ) adata = ad.AnnData(X=sp.csr_array(dense), obs=obs) path = tmp_path / "min_cells_obs_filter.ivcsr.h5ad" - VCSCAnnData.from_anndata(adata, format="csr").write_h5ad(path, format="ivcsc") + VCSCAnnData.from_anndata(adata, format="csr").write_h5ad(path) obs_mask = np.asarray(obs["condition"] == "control") result = load_and_normalize( @@ -180,7 +180,7 @@ def test_obs_filter_matches_subset_reference(tmp_path): ) adata = ad.AnnData(X=sp.csr_array(dense), obs=obs) path = tmp_path / "obs_filter.ivcsr.h5ad" - VCSCAnnData.from_anndata(adata, format="csr").write_h5ad(path, format="ivcsc") + VCSCAnnData.from_anndata(adata, format="csr").write_h5ad(path) obs_mask = np.asarray(obs["condition"] == "control") result = load_and_normalize( @@ -217,7 +217,7 @@ def test_obs_filter_applied_before_cell_filtering_and_slices_metadata(tmp_path, obsp = {"distances": rng.random((n_cells, n_cells))} adata = ad.AnnData(X=sp.csr_array(dense), obs=obs, var=var, obsm=obsm, obsp=obsp) path = tmp_path / "obs_filter_metadata.ivcsr.h5ad" - VCSCAnnData.from_anndata(adata, format="csr").write_h5ad(path, format="ivcsc") + VCSCAnnData.from_anndata(adata, format="csr").write_h5ad(path) obs_mask = np.asarray(obs["condition"] == "control") selected_rows = np.nonzero(obs_mask)[0] @@ -317,7 +317,7 @@ def test_metadata_sliced_correctly(tmp_path, rng): path = tmp_path / "metadata_test.ivcsr.h5ad" vad = VCSCAnnData.from_anndata(adata, format="csr") - vad.write_h5ad(path, format="ivcsc") + vad.write_h5ad(path) min_cell_counts = 10.0 gene_threshold = 0.05 @@ -423,7 +423,7 @@ def test_load_packed_metadata_preserved(tmp_path, rng): adata.var["gene"] = [f"g{i}" for i in range(6)] vad = VCSCAnnData.from_anndata(adata, format="csr") path = tmp_path / "meta.ivcsr.h5ad" - vad.write_h5ad(path, format="ivcsc") + vad.write_h5ad(path) result = load_packed(path) pd.testing.assert_index_equal(result.obs.index, adata.obs.index)