diff --git a/src/parcels/_core/_windowed_array.py b/src/parcels/_core/_windowed_array.py index f330268af..3bb5b065b 100644 --- a/src/parcels/_core/_windowed_array.py +++ b/src/parcels/_core/_windowed_array.py @@ -10,6 +10,7 @@ (``dt > 0``) or backward (``dt < 0``). Eviction keeps only the levels each ``isel`` actually requests, which is symmetric in time -- so direction never enters the logic and no integration-direction flag is needed. + * Time coordinate of U/V data/files is monotonic (ie, grabbing slice(i,i+10) is 10 sequential times) """ from __future__ import annotations @@ -29,12 +30,13 @@ class WindowedArray: """Wrap a lazy DataArray so ``isel`` loads/caches/evicts time levels as NumPy.""" - def __init__(self, data: xr.DataArray, time_dim: str = "time", max_levels: int | None = None): + def __init__(self, data: xr.DataArray, time_dim: str = "time", max_levels: int | None = None, time_chunk: int = 1): if data.dims[0] != time_dim: raise ValueError(f"WindowedArray expects {time_dim!r} as the leading dimension, got {data.dims}") self._data = data self._tdim = time_dim self._max = max_levels + self._tchunk = time_chunk # Let us load a bunch of stuff in at once (e.g. 1 day, or 1 month) self._cache: xr.DataArray = xr.DataArray( np.empty((0, *data.shape[1:]), dtype=data.dtype), dims=data.dims, @@ -43,7 +45,9 @@ def __init__(self, data: xr.DataArray, time_dim: str = "time", max_levels: int | # diagnostics self.loads = 0 self.bytes_read = 0 - self._slab_bytes = int(np.prod(data.isel({time_dim: 0}).shape)) * data.dtype.itemsize + self._slab_bytes = ( + int(np.prod(data.isel({time_dim: 0}).shape)) * data.dtype.itemsize * self._tchunk + ) # If I understand this variable correctly # -- transparency: forward everything we don't override ------------------- def __getattr__(self, name): @@ -57,9 +61,9 @@ def __repr__(self): ) # -- window management ---------------------------------------------------- - def _read_level(self, lvl: int) -> np.ndarray: - """Bulk, sequential read of one time level into NumPy (the dask->NumPy step).""" - return np.asarray(self._data.isel({self._tdim: int(lvl)}).values) + def _read_level(self, min_lvl: int, max_lvl: int) -> np.ndarray: + """Bulk, sequential read of self._tchunk time levels into NumPy (the dask->NumPy step).""" + return np.asarray(self._data.isel({self._tdim: slice(min_lvl, max_lvl)}).values) def _ensure(self, levels: np.ndarray) -> None: if self._max is not None and levels.size > self._max: @@ -79,8 +83,11 @@ def _ensure(self, levels: np.ndarray) -> None: ) lo, hi = int(np.min(levels)), int(np.max(levels)) + # Expand these out for the full number of chunks we're keeping + lo = lo // self._tchunk * self._tchunk + hi = hi // self._tchunk * self._tchunk + self._tchunk # Suggest using [ ) rather than [ ] sets with a -1? coord = self._cache[self._tdim].values - keep = (coord >= lo) & (coord <= hi) + keep = (coord >= lo) & (coord < hi) if self._max is not None: cached_in_span = np.flatnonzero(keep) @@ -100,11 +107,17 @@ def _ensure(self, levels: np.ndarray) -> None: if lvl in self._cache[self._tdim].values: continue - slab = self._read_level(lvl) + # What are the bounds of the chunk that this lvl goes into? + min_lvl = lvl // self._tchunk * self._tchunk + max_lvl = min( + min_lvl + self._tchunk, len(self._data[self._tdim]) + ) # pretty sure this is fine but would appreciate a second pair of eyes for off-by-one errors and not grabbing the end of the array + slab = self._read_level(min_lvl, max_lvl) self.loads += 1 self.bytes_read += self._slab_bytes - slab_as_xr = xr.DataArray(slab[None], dims=self._data.dims, coords={self._tdim: [lvl]}) - + slab_as_xr = xr.DataArray( + slab, dims=self._data.dims, coords={self._tdim: np.arange(min_lvl, max_lvl, dtype=int)} + ) # Have replaced slab[None] with slab because read_level now keeps dimensions with the slice # The cache is ordered based on time index coord = self._cache[self._tdim].values pos = int(np.searchsorted(coord, lvl)) @@ -112,7 +125,7 @@ def _ensure(self, levels: np.ndarray) -> None: self._cache = xr.concat( [ self._cache.isel({self._tdim: slice(0, pos)}), - slab_as_xr, + slab_as_xr, # With modified _read_level to give multiple timesteps at once this is multiple timesteps, but .concat should still be fine self._cache.isel({self._tdim: slice(pos, None)}), ], dim=self._tdim, @@ -146,7 +159,7 @@ def isel(self, indexers: dict | None = None, **kwargs): return self._cache.isel(sel) # return the requested isel directly from the cached DataArray -def maybe_windowed(data: xr.DataArray, max_levels: int | None = None): +def maybe_windowed(data: xr.DataArray, max_levels: int | None = None, time_chunk: int = 1): """Wrap dask-backed, field data in a ``WindowedArray``; else pass through. NumPy-backed fields (already resident) and fields without a leading ``time`` @@ -156,7 +169,7 @@ def maybe_windowed(data: xr.DataArray, max_levels: int | None = None): if isinstance(data, WindowedArray): return data if data.dims and data.dims[0] == "time" and is_dask_collection(data.data): - return WindowedArray(data, max_levels=max_levels) + return WindowedArray(data, max_levels=max_levels, time_chunk=time_chunk) elif data.dims and data.dims[0] == "mockT" and is_dask_collection(data.data): return data.compute() return data diff --git a/src/parcels/_core/fieldset.py b/src/parcels/_core/fieldset.py index c87575ae5..a0d325c0f 100644 --- a/src/parcels/_core/fieldset.py +++ b/src/parcels/_core/fieldset.py @@ -161,7 +161,7 @@ def time_interval(self): return overlap - def to_windowed_arrays(self, *, max_levels: int | None = None): + def to_windowed_arrays(self, *, max_levels: int | None = None, time_chunk: int = 1): """Wrap dask-backed field data in rolling time-window caches. Opt-in optimization for forward-marching simulations where all particles @@ -185,13 +185,20 @@ def to_windowed_arrays(self, *, max_levels: int | None = None): spread across many time levels) does the window grow beyond that, and ``max_levels`` then bounds its size. + time_chunk: int, optional + The number of timesteps to be called into memory each time another + time level is needed. If using dask, this should be at least the + chunksize used by xr.open_(mf)dataset times the number of cores, + but should be a number of timesteps that takes less than half the + available memory. + Returns ------- FieldSet ``self``, to allow chaining. """ for model in self.models: - model.to_windowed_arrays(max_levels=max_levels) + model.to_windowed_arrays(max_levels=max_levels, time_chunk=time_chunk) return self def to_chunk_cached_arrays(self, *, max_cache_bytes: int = 600_000_000): diff --git a/src/parcels/_core/model.py b/src/parcels/_core/model.py index e1c44de30..cc5fd666f 100644 --- a/src/parcels/_core/model.py +++ b/src/parcels/_core/model.py @@ -84,7 +84,7 @@ def field_data(self, name: str) -> Any: return windowed[name] return self.data[name] - def to_windowed_arrays(self, *, max_levels: int | None = None) -> Self: + def to_windowed_arrays(self, *, max_levels: int | None = None, time_chunk: int = 1) -> Self: """Wrap dask-backed field data in rolling time-window caches. Opt-in optimization for forward-marching simulations where all particles @@ -109,6 +109,13 @@ def to_windowed_arrays(self, *, max_levels: int | None = None) -> Self: Only when a single call requests a wider time span (e.g. particles spread across many time levels) does the window grow beyond that, and ``max_levels`` then bounds its size. + + time_chunk: int, optional + The number of timesteps to be called into memory each time another + time level is needed. If using dask, this should be at least the + chunksize used by xr.open_(mf)dataset times the number of cores, + but should be a number of timesteps that takes less than half the + available memory. """ windowed = self.__dict__.setdefault("_windowed", {}) for dim in ["lon", "lat", "depth"]: @@ -117,7 +124,7 @@ def to_windowed_arrays(self, *, max_levels: int | None = None) -> Self: self.data[dim].load() for name in self.scalar_field_names: current = windowed.get(name, self.data[name]) - windowed[name] = maybe_windowed(current, max_levels=max_levels) + windowed[name] = maybe_windowed(current, max_levels=max_levels, time_chunk=time_chunk) return self def to_chunk_cached_arrays(self, *, max_cache_bytes: int = 600_000_000) -> Self: