Skip to content
37 changes: 25 additions & 12 deletions src/parcels/_core/_windowed_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
(``dt > 0``) or backward (``dt < 0``). Eviction keeps only the levels each
``isel`` actually requests, which is symmetric in time -- so direction never
enters the logic and no integration-direction flag is needed.
* Time coordinate of U/V data/files is monotonic (ie, grabbing slice(i,i+10) is 10 sequential times)
"""

from __future__ import annotations
Expand All @@ -29,12 +30,13 @@
class WindowedArray:
"""Wrap a lazy DataArray so ``isel`` loads/caches/evicts time levels as NumPy."""

def __init__(self, data: xr.DataArray, time_dim: str = "time", max_levels: int | None = None):
def __init__(self, data: xr.DataArray, time_dim: str = "time", max_levels: int | None = None, time_chunk: int = 1):
if data.dims[0] != time_dim:
raise ValueError(f"WindowedArray expects {time_dim!r} as the leading dimension, got {data.dims}")
self._data = data
self._tdim = time_dim
self._max = max_levels
self._tchunk = time_chunk # Let us load a bunch of stuff in at once (e.g. 1 day, or 1 month)
self._cache: xr.DataArray = xr.DataArray(
np.empty((0, *data.shape[1:]), dtype=data.dtype),
dims=data.dims,
Expand All @@ -43,7 +45,9 @@ def __init__(self, data: xr.DataArray, time_dim: str = "time", max_levels: int |
# diagnostics
self.loads = 0
self.bytes_read = 0
self._slab_bytes = int(np.prod(data.isel({time_dim: 0}).shape)) * data.dtype.itemsize
self._slab_bytes = (
int(np.prod(data.isel({time_dim: 0}).shape)) * data.dtype.itemsize * self._tchunk
) # If I understand this variable correctly

# -- transparency: forward everything we don't override -------------------
def __getattr__(self, name):
Expand All @@ -57,9 +61,9 @@ def __repr__(self):
)

# -- window management ----------------------------------------------------
def _read_level(self, lvl: int) -> np.ndarray:
"""Bulk, sequential read of one time level into NumPy (the dask->NumPy step)."""
return np.asarray(self._data.isel({self._tdim: int(lvl)}).values)
def _read_level(self, min_lvl: int, max_lvl: int) -> np.ndarray:
"""Bulk, sequential read of self._tchunk time levels into NumPy (the dask->NumPy step)."""
return np.asarray(self._data.isel({self._tdim: slice(min_lvl, max_lvl)}).values)

def _ensure(self, levels: np.ndarray) -> None:
if self._max is not None and levels.size > self._max:
Expand All @@ -79,8 +83,11 @@ def _ensure(self, levels: np.ndarray) -> None:
)

lo, hi = int(np.min(levels)), int(np.max(levels))
# Expand these out for the full number of chunks we're keeping
lo = lo // self._tchunk * self._tchunk
hi = hi // self._tchunk * self._tchunk + self._tchunk # Suggest using [ ) rather than [ ] sets with a -1?
coord = self._cache[self._tdim].values
keep = (coord >= lo) & (coord <= hi)
keep = (coord >= lo) & (coord < hi)

if self._max is not None:
cached_in_span = np.flatnonzero(keep)
Expand All @@ -100,19 +107,25 @@ def _ensure(self, levels: np.ndarray) -> None:
if lvl in self._cache[self._tdim].values:
continue

slab = self._read_level(lvl)
# What are the bounds of the chunk that this lvl goes into?
min_lvl = lvl // self._tchunk * self._tchunk
max_lvl = min(
min_lvl + self._tchunk, len(self._data[self._tdim])
) # pretty sure this is fine but would appreciate a second pair of eyes for off-by-one errors and not grabbing the end of the array
slab = self._read_level(min_lvl, max_lvl)
self.loads += 1
self.bytes_read += self._slab_bytes
slab_as_xr = xr.DataArray(slab[None], dims=self._data.dims, coords={self._tdim: [lvl]})

slab_as_xr = xr.DataArray(
slab, dims=self._data.dims, coords={self._tdim: np.arange(min_lvl, max_lvl, dtype=int)}
) # Have replaced slab[None] with slab because read_level now keeps dimensions with the slice
# The cache is ordered based on time index
coord = self._cache[self._tdim].values
pos = int(np.searchsorted(coord, lvl))

self._cache = xr.concat(
[
self._cache.isel({self._tdim: slice(0, pos)}),
slab_as_xr,
slab_as_xr, # With modified _read_level to give multiple timesteps at once this is multiple timesteps, but .concat should still be fine
self._cache.isel({self._tdim: slice(pos, None)}),
],
dim=self._tdim,
Expand Down Expand Up @@ -146,7 +159,7 @@ def isel(self, indexers: dict | None = None, **kwargs):
return self._cache.isel(sel) # return the requested isel directly from the cached DataArray


def maybe_windowed(data: xr.DataArray, max_levels: int | None = None):
def maybe_windowed(data: xr.DataArray, max_levels: int | None = None, time_chunk: int = 1):
"""Wrap dask-backed, field data in a ``WindowedArray``; else pass through.

NumPy-backed fields (already resident) and fields without a leading ``time``
Expand All @@ -156,7 +169,7 @@ def maybe_windowed(data: xr.DataArray, max_levels: int | None = None):
if isinstance(data, WindowedArray):
return data
if data.dims and data.dims[0] == "time" and is_dask_collection(data.data):
return WindowedArray(data, max_levels=max_levels)
return WindowedArray(data, max_levels=max_levels, time_chunk=time_chunk)
elif data.dims and data.dims[0] == "mockT" and is_dask_collection(data.data):
return data.compute()
return data
11 changes: 9 additions & 2 deletions src/parcels/_core/fieldset.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ def time_interval(self):

return overlap

def to_windowed_arrays(self, *, max_levels: int | None = None):
def to_windowed_arrays(self, *, max_levels: int | None = None, time_chunk: int = 1):
"""Wrap dask-backed field data in rolling time-window caches.

Opt-in optimization for forward-marching simulations where all particles
Expand All @@ -185,13 +185,20 @@ def to_windowed_arrays(self, *, max_levels: int | None = None):
spread across many time levels) does the window grow beyond that,
and ``max_levels`` then bounds its size.

time_chunk: int, optional
The number of timesteps to be called into memory each time another
time level is needed. If using dask, this should be at least the
chunksize used by xr.open_(mf)dataset times the number of cores,
but should be a number of timesteps that takes less than half the
available memory.

Returns
-------
FieldSet
``self``, to allow chaining.
"""
for model in self.models:
model.to_windowed_arrays(max_levels=max_levels)
model.to_windowed_arrays(max_levels=max_levels, time_chunk=time_chunk)
return self

def to_chunk_cached_arrays(self, *, max_cache_bytes: int = 600_000_000):
Expand Down
11 changes: 9 additions & 2 deletions src/parcels/_core/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def field_data(self, name: str) -> Any:
return windowed[name]
return self.data[name]

def to_windowed_arrays(self, *, max_levels: int | None = None) -> Self:
def to_windowed_arrays(self, *, max_levels: int | None = None, time_chunk: int = 1) -> Self:
"""Wrap dask-backed field data in rolling time-window caches.

Opt-in optimization for forward-marching simulations where all particles
Expand All @@ -109,6 +109,13 @@ def to_windowed_arrays(self, *, max_levels: int | None = None) -> Self:
Only when a single call requests a wider time span (e.g. particles
spread across many time levels) does the window grow beyond that,
and ``max_levels`` then bounds its size.

time_chunk: int, optional
The number of timesteps to be called into memory each time another
time level is needed. If using dask, this should be at least the
chunksize used by xr.open_(mf)dataset times the number of cores,
but should be a number of timesteps that takes less than half the
available memory.
"""
windowed = self.__dict__.setdefault("_windowed", {})
for dim in ["lon", "lat", "depth"]:
Expand All @@ -117,7 +124,7 @@ def to_windowed_arrays(self, *, max_levels: int | None = None) -> Self:
self.data[dim].load()
for name in self.scalar_field_names:
current = windowed.get(name, self.data[name])
windowed[name] = maybe_windowed(current, max_levels=max_levels)
windowed[name] = maybe_windowed(current, max_levels=max_levels, time_chunk=time_chunk)
return self

def to_chunk_cached_arrays(self, *, max_cache_bytes: int = 600_000_000) -> Self:
Expand Down
Loading