From acddd51fdc28193d95cc0d869c4d15d73037a3af Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 12:26:12 +0000 Subject: [PATCH 1/2] feat(spec): add dim_order to Model.from_spec and evaluate The dims dim_order names come first in every variable, constraint and named expression the spec builds, before labels are allocated; the rest keep today's layout. None keeps the name order exactly. --- doc/release-notes.md | 1 + linopy/model.py | 19 ++++++- linopy/spec/api.py | 69 ++++++++++++++++++++++--- linopy/spec/builder.py | 32 ++++++++---- linopy/spec/context.py | 9 +++- linopy/spec/evaluator.py | 6 ++- test/test_spec_api.py | 107 ++++++++++++++++++++++++++++++++++++++- 7 files changed, 221 insertions(+), 22 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index 2a88e23f..23dc3882 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -28,6 +28,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds no term over a dimension without a coordinate: such a term, and a sum of one, is the constant zero, while the data it reads is still read and checked. An assumption over such a dimension is skipped unread. A lookup through a relation without a row no longer lays out where the relation has one. ([\#1155](https://github.com/PyPSA/linopy/pull/1155)) - `Model.from_spec` builds faster: it reads a shifted axis by position, checks a relation's labels against the hash table of their dimension, decides a `where` early when a condition holds nowhere, and sets a scalar scaling factor without broadcasting it. ([\#1169](https://github.com/PyPSA/linopy/pull/1169)) - `Model.from_spec(..., rename=...)` builds each variable and constraint under the name a mapping or a function gives it, e.g. `Generator-p` for the spec's `Generator_p`. Pass the same `rename` to `linopy.spec.evaluate`. ([\#1160](https://github.com/PyPSA/linopy/pull/1160)) +- `Model.from_spec(..., dim_order=...)` sets the axis order of every variable, constraint and named expression it builds: the dims it names come first, the rest keep their name order. E.g. `dim_order=["store"]` builds `(store, snapshot)`, where name order gives `(snapshot, store)`. Pass the same `dim_order` to `linopy.spec.evaluate`. (PRNUM) - A missing row takes the value that `missing:` sets, e.g. `missing: 0`, in bounds, expressions and conditions alike. A parameter's `where:` limits its coordinates, and a dimension's `one_of:` its labels. Built objects follow the name order of their dimensions, as mathspec sorts every `dims:` list. ([\#1137](https://github.com/PyPSA/linopy/pull/1137)) - Limits: a constraint row that absence leaves without terms raises, where mathspec skips it ([\#993](https://github.com/PyPSA/linopy/issues/993)). `store=True` stores only relations with one key and one value column. A product of two variables in a constraint is accepted only where the data leaves it no live row. diff --git a/linopy/model.py b/linopy/model.py index ba07b436..11933447 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -491,6 +491,7 @@ def from_spec( store: bool = False, expressions: bool = True, rename: Mapping[str, str] | Callable[[str], str] | None = None, + dim_order: Sequence[str] | None = None, **model_kwargs: Any, ) -> Model: """ @@ -535,6 +536,18 @@ def from_spec( name, and the entries a ``piecewise:`` curve expands into are named too. Expressions and duals in the spec keep reading spec names. Pass the same *rename* to :func:`linopy.spec.evaluate`. + dim_order : sequence of str, optional + The axis order of each variable, constraint and named expression + the spec builds, and so the order its labels are allocated in. + The dims of an entry named here come first, in this order; the + rest keep the order ``None`` gives them: by name, except that a + dim only the data side of a constraint carries, such as a + piecewise breakpoint dim, comes last. A name an entry lacks is + ignored there. For instance, ``dim_order=["store"]`` builds a + variable over ``snapshot`` and ``store`` as + ``(store, snapshot)``, where ``None`` gives + ``(snapshot, store)``. Pass the same *dim_order* to + :func:`linopy.spec.evaluate`. **model_kwargs Passed to :class:`Model`, such as ``sparse=True``. @@ -552,6 +565,10 @@ def from_spec( If a key of a mapping *rename* names no variable or constraint. ValueError If *rename* gives two variables, or two constraints, one name. + TypeError + If *dim_order* is a ``str``. + ValueError + If *dim_order* names a dim twice. Warns ----- @@ -562,7 +579,7 @@ def from_spec( from linopy.spec.api import build_into model = cls(**model_kwargs) - build_into(model, spec, sources, store, expressions, rename) + build_into(model, spec, sources, store, expressions, rename, dim_order) return model @property diff --git a/linopy/spec/api.py b/linopy/spec/api.py index 72f2c65a..675ed809 100644 --- a/linopy/spec/api.py +++ b/linopy/spec/api.py @@ -13,7 +13,7 @@ import functools from collections import Counter -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from pathlib import Path from types import MappingProxyType from typing import Any, TypeAlias @@ -57,13 +57,15 @@ def build_into( store: bool, expressions: bool, rename: Rename | None = None, + dim_order: Sequence[str] | None = None, ) -> None: """ Build *spec* with *sources* into the fresh *model*. With *store* its data goes into ``model.parameters``, with *expressions* its named expressions into ``model.expressions``, and - each variable and constraint under the name *rename* gives it. + each variable and constraint under the name *rename* gives it, its dims + in the order *dim_order* sets. Raises ------ @@ -81,10 +83,13 @@ def build_into( A constraint multiplies two variable-carrying operands. KeyError, ValueError What :func:`_renamer` raises for *rename*. + TypeError, ValueError + What :func:`_dim_order` raises for *dim_order*. """ _warn_evolving_api("spec", EVOLVING_MESSAGE, stacklevel=4) + order = _dim_order(dim_order) program, uses = _normalized(spec) - _build(model, program, uses, sources, store, expressions, rename) + _build(model, program, uses, sources, store, expressions, rename, order) def _build( @@ -95,6 +100,7 @@ def _build( store: bool, expressions: bool, rename: Rename | None, + dim_order: tuple[str, ...], ) -> None: """ The body of :func:`build_into`, in a frame of its own. @@ -112,10 +118,37 @@ def _build( if store: unfilled = {name: raw(name).values for name in program.parameters} model.parameters = xr.Dataset(unfilled | relations, coords=attached.coords) - ctx = _context(attached, uses, model, raw, solved=False, rename=named) + ctx = _context( + attached, uses, model, raw, solved=False, rename=named, dim_order=dim_order + ) build(ctx, expressions) +def _dim_order(dim_order: Sequence[str] | None) -> tuple[str, ...]: + """ + *dim_order* as the tuple :meth:`Context.frame` reads, empty for ``None``. + + Raises + ------ + TypeError + *dim_order* is a ``str``, which would read as one dim per character. + ValueError + *dim_order* names a dim twice. + """ + if dim_order is None: + return () + if isinstance(dim_order, str): + raise TypeError( + f"dim_order takes a sequence of dim names, not the str {dim_order!r}; " + f"pass [{dim_order!r}]" + ) + order = tuple(dim_order) + twice = sorted(d for d, count in Counter(order).items() if count > 1) + if twice: + raise ValueError(f"dim_order names the dim {twice[0]!r} twice") + return order + + def _renamer(program: ms.Program, rename: Rename | None) -> Callable[[str], str]: """ The model name of each variable and constraint of *program*: what *rename* gives, its own name where it gives none. @@ -192,6 +225,7 @@ def evaluate( *names: str, sources: Sources | None = None, rename: Rename | None = None, + dim_order: Sequence[str] | None = None, ) -> dict[str, xr.DataArray]: """ The named expressions *names* of *spec*, all in name order when none is named, folded over *model*. @@ -201,7 +235,8 @@ def evaluate( ``Model.from_spec(..., store=True)`` writes it: a body over data alone needs no solve. A model built with ``Model.from_spec(..., rename=...)`` is read with the - same *rename*. + same *rename*, and each expression is laid out in the order *dim_order* + sets, as ``Model.from_spec(..., dim_order=...)`` lays out what it builds. Raises ------ @@ -215,10 +250,15 @@ def evaluate( SpecDataError The data does not fit the spec, or a body reads a missing row of a ``missing: refused`` parameter. + TypeError + *dim_order* is a ``str``. + ValueError + *dim_order* names a dim twice. """ _warn_evolving_api("spec", EVOLVING_MESSAGE) + order = _dim_order(dim_order) program, uses = _normalized(spec) - return _folded(program, uses, model, names, sources, rename) + return _folded(program, uses, model, names, sources, rename, order) def _folded( @@ -228,6 +268,7 @@ def _folded( names: tuple[str, ...], sources: Sources | None, rename: Rename | None, + dim_order: tuple[str, ...], ) -> dict[str, xr.DataArray]: """The body of :func:`evaluate`, in a frame of its own for the reason :func:`_build` gives.""" for name in names: @@ -235,7 +276,15 @@ def _folded( raise unknown("expression", name, program.expressions) named = _renamer(program, rename) attached = attach(program, _stored(program, model) if sources is None else sources) - ctx = _context(attached, uses, model, attached.read, solved=True, rename=named) + ctx = _context( + attached, + uses, + model, + attached.read, + solved=True, + rename=named, + dim_order=dim_order, + ) return {name: fold(name, ctx) for name in names or program.expressions} @@ -273,6 +322,7 @@ def _context( *, solved: bool, rename: Callable[[str], str], + dim_order: tuple[str, ...], ) -> Context: program = attached.program tables = (model, program, attached.coords, attached.relations) @@ -282,7 +332,9 @@ def unmasked(name: str) -> Parameter: parameter = raw(name) return Parameter(fill(program, name, parameter), parameter.present) - conditions = Context(*tables, unmasked, solved=solved, rename=rename) + conditions = Context( + *tables, unmasked, solved=solved, rename=rename, dim_order=dim_order + ) def read(name: str) -> Parameter: if program.parameters[name].where is None: @@ -298,6 +350,7 @@ def read(name: str) -> Parameter: solved=solved, memo=Memo(uses=dict(uses)), rename=rename, + dim_order=dim_order, ) diff --git a/linopy/spec/builder.py b/linopy/spec/builder.py index fb37c3c5..630df173 100644 --- a/linopy/spec/builder.py +++ b/linopy/spec/builder.py @@ -5,7 +5,9 @@ sets, constraints, the objective and the named expressions that carry a variable term; which linopy call each construct becomes is one branch of :func:`linopy.spec.evaluator.evaluate`. Everything built is framed in the -name order of its dims, the order mathspec sorts every ``dims:`` list into. +order :meth:`~linopy.spec.context.Context.frame` gives its dims: the dims +``dim_order`` names first, the rest in the name order mathspec sorts every +``dims:`` list into. """ from __future__ import annotations @@ -104,7 +106,7 @@ def _variables(ctx: Context) -> None: ctx.model.add_variables( lower=lower, upper=upper, - coords={d: ctx.coords[d] for d in declared.dims}, + coords={d: ctx.coords[d] for d in ctx.frame(declared.dims)}, name=ctx.rename(name), mask=mask, binary=declared.domain == "binary", @@ -162,7 +164,11 @@ def _constraints(ctx: Context) -> None: ) term, other = carried(term, other) ctx.model.add_constraints( - _framed(term, row.dims), sense, other, name=ctx.rename(name), mask=mask + _framed(term, row.dims, ctx), + sense, + other, + name=ctx.rename(name), + mask=mask, ) @@ -306,19 +312,27 @@ def _expressions(ctx: Context) -> None: continue reference = ms.ExpressionReference(name, body, dims=declared.dims) value = cast(Term, evaluate(reference, ctx)) - ctx.model.add_expressions(_framed(value, declared.dims), name=name) + ctx.model.add_expressions(_framed(value, declared.dims, ctx), name=name) def _framed( - term: Term, dims: tuple[str, ...] + term: Term, dims: tuple[str, ...], ctx: Context ) -> LinearExpression | QuadraticExpression: """ - *term* with its dimensions in the order of the entry's dims, sorted by - name, without stray coordinates. A dimension the entry declares and the term lacks is - what the other side of a comparison brings in. + *term* with its dimensions in the order :meth:`Context.frame` gives the + entry's *dims*, without stray coordinates. A dimension the entry declares + and the term lacks is what the other side of a comparison brings in, and + lands after the term's own, as a piecewise breakpoint dim does; one + ``dim_order`` names is spread onto the term to take its place in front. """ expression = without_strays(as_expression(term)) - order = [d for d in dims if d in expression.coord_dims] + order = [ + d for d in ctx.frame(dims) if d in expression.coord_dims or d in ctx.dim_order + ] + lacking = {d: ctx.coords[d] for d in order if d not in expression.coord_dims} + if lacking: + shape = [len(index) for index in lacking.values()] + expression = expression + xr.DataArray(np.zeros(shape), coords=lacking) if isinstance(expression, LinearExpression) and expression.is_sparse: store = expression._store.transpose(order) return LinearExpression._from_store(store, expression.model) diff --git a/linopy/spec/context.py b/linopy/spec/context.py index 79c368a1..3dbda318 100644 --- a/linopy/spec/context.py +++ b/linopy/spec/context.py @@ -137,7 +137,8 @@ class Context: narrows the finiteness check to the rows a constraint builds, and ``subject`` names what is being built in its refusals. ``memo`` is shared by every context replaced from one. ``rename`` gives the model - name of a variable or constraint the program declares. + name of a variable or constraint the program declares. ``dim_order`` + names the dims :meth:`frame` puts first. """ model: Model @@ -150,6 +151,12 @@ class Context: subject: str = "the spec" memo: Memo = field(default_factory=Memo, compare=False, repr=False) rename: Callable[[str], str] = field(default=str, compare=False, repr=False) + dim_order: tuple[str, ...] = () + + def frame(self, dims: tuple[str, ...]) -> tuple[str, ...]: + """*dims* of an entry, those ``dim_order`` names first, in its order, the rest after in name order, as mathspec states them.""" + first = tuple(d for d in self.dim_order if d in dims) + return (*first, *(d for d in dims if d not in first)) def is_empty(dims: Iterable[str], ctx: Context) -> bool: diff --git a/linopy/spec/evaluator.py b/linopy/spec/evaluator.py index c67a7a24..9f52dd5b 100644 --- a/linopy/spec/evaluator.py +++ b/linopy/spec/evaluator.py @@ -45,7 +45,9 @@ def fold(name: str, ctx: Context) -> xr.DataArray: refuse_holes(f"expression '{name}'", [declared.expression], ctx) value = evaluate(declared.expression, ctx) if isinstance(value, xr.DataArray): - return without_strays(value).rename(name).transpose(*declared.dims, ...) + return ( + without_strays(value).rename(name).transpose(*ctx.frame(declared.dims), ...) + ) return xr.DataArray(float(cast(float, value)), name=name) @@ -197,7 +199,7 @@ def _dual(constraint: str, ctx: Context) -> xr.DataArray: if renamed in built: reported = built[renamed]._dual elif any(c._dual is not None for c in built.values()): - dims = ctx.program.constraints[constraint].dims + dims = ctx.frame(ctx.program.constraints[constraint].dims) shape = [len(ctx.coords[d]) for d in dims] coords = {d: ctx.coords[d] for d in dims} reported = xr.DataArray(np.full(shape, np.nan), coords, dims=dims) diff --git a/test/test_spec_api.py b/test/test_spec_api.py index 056f8c25..d07df714 100644 --- a/test/test_spec_api.py +++ b/test/test_spec_api.py @@ -21,13 +21,18 @@ pytest.importorskip("highspy") from spec_helpers import ( # noqa: E402 + BP, + CURVE_DATA, + CURVE_SPEC, DISPATCH_DATA, DISPATCH_P, EXAMPLE_DISPATCH, GENERATOR, TT, + UNITS, WHERE_DATA, WHERE_SPEC, + curve, solved, with_, yaml_dict, @@ -37,7 +42,7 @@ from linopy import LinearExpression, Model # noqa: E402 from linopy.spec import SpecDataError, evaluate # noqa: E402 from linopy.spec.api import lower # noqa: E402 -from linopy.testing import synthetic_sources # noqa: E402 +from linopy.testing import assert_model_equal, synthetic_sources # noqa: E402 # --------------------------------------------------------------------------- # inputs and model integration @@ -213,6 +218,106 @@ def test_rename_is_refused_before_the_build( Model.from_spec(spec, DISPATCH_DATA, rename=rename) +def curves(method: str) -> dict[str, Any]: + return with_( + CURVE_SPEC, + piecewise={ + "cost_curve": {**CURVE_SPEC["piecewise"]["cost_curve"], "method": method} + }, + expressions={ + "net": {"dims": ["snapshot", "generator"], "expression": "op_cost - p"} + }, + ) + + +CURVE_SOURCES = { + **CURVE_DATA, + "snapshot": [0, 1, 2], + "load": pd.DataFrame({"snapshot": [0, 1, 2], "load": [50.0, 55.0, 80.0]}), + "bp_x": curve({(g, k): x for g in UNITS for k, x in zip(BP, [0, 20, 50, 80])}), + "bp_y": curve( + { + **{("hydro", k): y for k, y in zip(BP, [0, 100, 300, 600])}, + **{("gas", k): y for k, y in zip(BP, [0, 150, 450, 900])}, + } + ), +} + + +def framed(default: tuple[str, ...], order: list[str]) -> tuple[str, ...]: + """The dims of an entry built as *default*, the ones *order* names moved to the front.""" + first = tuple(d for d in order if d in default) + return (*first, *(d for d in default if d not in first)) + + +@pytest.mark.parametrize("method", ["lp", "sos2", "incremental"]) +@pytest.mark.parametrize( + "order", [["snapshot"], ["snapshot", "generator"], ["bp", "snapshot", "nowhere"]] +) +def test_dim_order_lays_out_every_entry( + method: str, order: list[str], backend: Model +) -> None: + spec = curves(method) + default = Model.from_spec(spec, CURVE_SOURCES, sparse=backend.sparse) + m = Model.from_spec(spec, CURVE_SOURCES, dim_order=order, sparse=backend.sparse) + for name, variable in default.variables.items(): + assert m.variables[name].labels.dims == framed(variable.labels.dims, order) + for name, constraint in default.constraints.items(): + dims = framed(constraint.labels.dims, order) + assert m.constraints[name].labels.dims == dims + assert m.constraints[name].lhs.coord_dims == dims + for name, expression in default.expressions.items(): + assert m.expressions[name].coord_dims == framed(expression.coord_dims, order) + + +@pytest.mark.parametrize("order", [None, [], ["nowhere"], ["bp", "generator"]]) +def test_a_dim_order_the_name_order_meets_builds_the_default( + order: list[str] | None, backend: Model +) -> None: + spec = curves("sos2") + default = Model.from_spec(spec, CURVE_SOURCES, sparse=backend.sparse) + ordered = Model.from_spec( + spec, CURVE_SOURCES, dim_order=order, sparse=backend.sparse + ) + assert_model_equal(ordered, default) + + +@pytest.mark.parametrize("method", ["lp", "sos2"]) +def test_dim_order_solves_to_the_default_up_to_transposition( + method: str, backend: Model +) -> None: + spec, order = curves(method), ["snapshot", "bp"] + default = solved(spec, CURVE_SOURCES, sparse=backend.sparse) + ordered = solved(spec, CURVE_SOURCES, sparse=backend.sparse, dim_order=order) + assert ordered.objective.value == pytest.approx(default.objective.value) + pairs = [ + (default.solution, ordered.solution), + (default.dual, ordered.dual), + (evaluate(spec, default), evaluate(spec, ordered, dim_order=order)), + ] + for expected, got in pairs: + assert set(got) == set(expected) + for name, value in expected.items(): + assert got[name].dims == framed(value.dims, order) + xr.testing.assert_allclose(got[name].transpose(*value.dims), value) + + +@pytest.mark.parametrize( + ("dim_order", "error", "match"), + [ + pytest.param("snapshot", TypeError, "not the str 'snapshot'", id="str"), + pytest.param(["generator"] * 2, ValueError, "'generator' twice", id="twice"), + ], +) +def test_dim_order_is_refused_before_the_build( + dim_order: Any, error: type[Exception], match: str +) -> None: + with pytest.raises(error, match=match): + Model.from_spec(yaml_dict(), DISPATCH_DATA, dim_order=dim_order) + with pytest.raises(error, match=match): + evaluate(yaml_dict(), Model(), dim_order=dim_order) + + def test_the_spec_package_exports() -> None: assert isinstance(linopy.spec.attach, types.ModuleType) assert set(linopy.spec.__all__) == {"SpecDataError", "evaluate"} From 2abf49578390824d65bd070ff04be496a3bcdecf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 12:26:48 +0000 Subject: [PATCH 2/2] docs: link the release note to its PR --- doc/release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index 23dc3882..eaf418f6 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -28,7 +28,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds no term over a dimension without a coordinate: such a term, and a sum of one, is the constant zero, while the data it reads is still read and checked. An assumption over such a dimension is skipped unread. A lookup through a relation without a row no longer lays out where the relation has one. ([\#1155](https://github.com/PyPSA/linopy/pull/1155)) - `Model.from_spec` builds faster: it reads a shifted axis by position, checks a relation's labels against the hash table of their dimension, decides a `where` early when a condition holds nowhere, and sets a scalar scaling factor without broadcasting it. ([\#1169](https://github.com/PyPSA/linopy/pull/1169)) - `Model.from_spec(..., rename=...)` builds each variable and constraint under the name a mapping or a function gives it, e.g. `Generator-p` for the spec's `Generator_p`. Pass the same `rename` to `linopy.spec.evaluate`. ([\#1160](https://github.com/PyPSA/linopy/pull/1160)) -- `Model.from_spec(..., dim_order=...)` sets the axis order of every variable, constraint and named expression it builds: the dims it names come first, the rest keep their name order. E.g. `dim_order=["store"]` builds `(store, snapshot)`, where name order gives `(snapshot, store)`. Pass the same `dim_order` to `linopy.spec.evaluate`. (PRNUM) +- `Model.from_spec(..., dim_order=...)` sets the axis order of every variable, constraint and named expression it builds: the dims it names come first, the rest keep their name order. E.g. `dim_order=["store"]` builds `(store, snapshot)`, where name order gives `(snapshot, store)`. Pass the same `dim_order` to `linopy.spec.evaluate`. ([\#1179](https://github.com/PyPSA/linopy/pull/1179)) - A missing row takes the value that `missing:` sets, e.g. `missing: 0`, in bounds, expressions and conditions alike. A parameter's `where:` limits its coordinates, and a dimension's `one_of:` its labels. Built objects follow the name order of their dimensions, as mathspec sorts every `dims:` list. ([\#1137](https://github.com/PyPSA/linopy/pull/1137)) - Limits: a constraint row that absence leaves without terms raises, where mathspec skips it ([\#993](https://github.com/PyPSA/linopy/issues/993)). `store=True` stores only relations with one key and one value column. A product of two variables in a constraint is accepted only where the data leaves it no live row.