diff --git a/doc/release-notes.md b/doc/release-notes.md index a1c3ff1e..2a88e23f 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -131,6 +131,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) +- `Model.from_spec` no longer lets a `where` guard what a `shift`, window, `sum`, `count` or `at()` lookup reads at another coordinate, so a non-finite value there raises instead of entering a row or a `when:`. ([\#1173](https://github.com/PyPSA/linopy/pull/1173)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) diff --git a/linopy/spec/context.py b/linopy/spec/context.py index 12a0ac34..79c368a1 100644 --- a/linopy/spec/context.py +++ b/linopy/spec/context.py @@ -12,7 +12,7 @@ from collections import Counter from collections.abc import Callable, Hashable, Iterable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, TypeVar, cast, overload import pandas as pd @@ -155,3 +155,28 @@ class Context: def is_empty(dims: Iterable[str], ctx: Context) -> bool: """Whether one of *dims* has no coordinate, so what is over them has no cell.""" return any(not len(ctx.coords[d]) for d in dims) + + +def widened(node: ms.Expression | ms.Condition, ctx: Context) -> Context: + """ + *ctx* for the operand of *node*: a sum, a window, a shift, a lookup and a + count read their operand at other coordinates along the axes they move, so + the rows *ctx* checks are widened along those axes to every row there. + """ + if ctx.rows is None: + return ctx + along = [d for d in _moved_dims(node) if d in ctx.rows.dims] + return replace(ctx, rows=ctx.rows.any(along)) if along else ctx + + +def _moved_dims(node: ms.Expression | ms.Condition) -> tuple[str, ...]: + """The dimensions along which *node* reads its operand at other coordinates than its own.""" + if isinstance(node, ms.Sum): + return tuple(axis.dimension for axis in node.over) + if isinstance(node, ms.CountComparison): + return (node.over,) + if isinstance(node, ms.Translate | ms.WindowSum | ms.TranslatedCondition): + return (node.along,) + if isinstance(node, ms.Join | ms.JoinedCondition): + return node.columns.dropped_dims + return () diff --git a/linopy/spec/evaluator.py b/linopy/spec/evaluator.py index a51b2ca3..c67a7a24 100644 --- a/linopy/spec/evaluator.py +++ b/linopy/spec/evaluator.py @@ -14,7 +14,15 @@ from linopy.constants import FACTOR_DIM from linopy.expressions import LinearExpression, QuadraticExpression from linopy.spec import operators -from linopy.spec.context import Array, Context, Term, Value, as_expression, is_empty +from linopy.spec.context import ( + Array, + Context, + Term, + Value, + as_expression, + is_empty, + widened, +) from linopy.spec.errors import SpecDataError, first_coordinates from linopy.spec.where import admitted, evaluate_where, groups_of, looked_up from linopy.variables import Variable @@ -79,11 +87,11 @@ def _evaluated(node: ms.Expression, ctx: Context) -> Value: if isinstance(node, ms.Sum): return _sum(node, ctx) if isinstance(node, ms.Join): - return looked_up(_operand(node.operand, ctx), node.columns, ctx) + return looked_up(_operand(node.operand, widened(node, ctx)), node.columns, ctx) if isinstance(node, ms.Translate): - return _moved(node, _operand(node.operand, ctx), ctx, node.fill) + return _moved(node, _operand(node.operand, widened(node, ctx)), ctx, node.fill) if isinstance(node, ms.WindowSum): - return _moved(node, _operand(node.operand, ctx), ctx, None) + return _moved(node, _operand(node.operand, widened(node, ctx)), ctx, None) if isinstance(node, ms.Cases): regions = ( _in_region(evaluate(region.value, ctx), evaluate_where(region.when, ctx)) @@ -121,12 +129,13 @@ def _data_under(node: ms.Expression, ctx: Context) -> None: if isinstance(node, ms.Translate | ms.WindowSum): _partition(node.partition, ctx) _amount(_amount_of(node), ctx) + inner = widened(node, ctx) for operand in ms.children(node): if not ms.carries_variable(operand): - evaluate(operand, ctx) + evaluate(operand, inner) continue ctx.memo.used(operand) - _data_under(operand, ctx) + _data_under(operand, inner) def _sum( @@ -137,7 +146,7 @@ def _sum( the operand through the relation, the plain ones by summing them away. *of* evaluates an operand, :func:`evaluate` when not given. """ - value = functools.partial(_operand, ctx=ctx) if of is None else of + value = functools.partial(_operand, ctx=widened(node, ctx)) if of is None else of operand = node.operand plain = [axis.dimension for axis in node.over if axis.column is None] if isinstance(operand, ms.Join) and operand.columns.axes: diff --git a/linopy/spec/where.py b/linopy/spec/where.py index 0079ee19..a42ebf11 100644 --- a/linopy/spec/where.py +++ b/linopy/spec/where.py @@ -11,7 +11,7 @@ from mathspec import program as ms from linopy.spec import operators -from linopy.spec.context import Array, Context +from linopy.spec.context import Array, Context, widened from linopy.spec.errors import SpecDataError _COMPARISONS: dict[str, Callable[..., xr.DataArray]] = { @@ -108,13 +108,15 @@ def _condition(node: ms.Condition, ctx: Context) -> xr.DataArray: if isinstance(node, ms.ExpressionComparison): return _both(node.op, _side(node.left, ctx), _side(node.right, ctx)) if isinstance(node, ms.CountComparison): - count = _node(node.condition.root, ctx).sum(node.over) + count = _node(node.condition.root, widened(node, ctx)).sum(node.over) return _COMPARISONS[node.op](count, node.value) if isinstance(node, ms.TranslatedCondition): - shifted = _node(node.operand.root, ctx).shift({node.along: node.offset}) + inner = widened(node, ctx) + shifted = _node(node.operand.root, inner).shift({node.along: node.offset}) return _bool(shifted) if isinstance(node, ms.JoinedCondition): - pulled = looked_up(_node(node.operand.root, ctx), node.columns, ctx) + inner = widened(node, ctx) + pulled = looked_up(_node(node.operand.root, inner), node.columns, ctx) return _bool(cast(xr.DataArray, pulled)) if isinstance(node, ms.DimensionComparison): labels = ctx.coords[node.name] diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index a699285a..dcfa10b9 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -666,6 +666,122 @@ def test_arithmetic_masked_out_for_one_row_is_refused_for_the_next_reading_it() Model.from_spec(spec, ZERO_AT_0) +SUCC = {"relations": {"succ": {"key": "t", "values": {"u": "t"}}}} +SUCC_DATA = {**ZERO_AT_0, "succ": pd.DataFrame({"t": T, "u": [1, 0, 1]})} + + +@pytest.mark.parametrize( + "expression", + [ + pytest.param("x <= shift(a / b, along=t, offset=1, edge=0)", id="shift"), + pytest.param("x <= sum_back(a / b, along=t, window=2)", id="window"), + pytest.param("x <= at(a / b, by=succ[u])", id="lookup"), + pytest.param("x <= sum(a / b, over=t)", id="sum"), + pytest.param("shift(a / b * x, along=t, offset=1, edge=0) <= 1", id="term"), + ], +) +def test_a_where_does_not_guard_what_a_row_reads_at_another_coordinate( + expression: str, +) -> None: + """``b > 0`` masks t=0 only, and row t=1 reads ``a / b`` at t=0.""" + spec = with_(quotient(expression, where="b > 0"), **SUCC) + with pytest.raises(SpecDataError, match="divide of 'a', 'b' is not finite at t=0"): + Model.from_spec(spec, SUCC_DATA) + + +@pytest.mark.parametrize( + ("expression", "rhs"), + [ + pytest.param("x <= shift(a, along=t, offset=1, edge=0) / b", [0.0, 0.2]), + pytest.param("x <= a / b + shift(a, along=t, offset=1, edge=0)", [0.25, 1.2]), + pytest.param("x <= sum(a, over=t) / b", [0.5, 0.4]), + ], + ids=["shift-then-divide", "beside-a-shift", "sum-then-divide"], +) +def test_a_where_guards_what_a_row_reads_at_its_own_coordinate( + expression: str, rhs: list[float] +) -> None: + m = Model.from_spec(quotient(expression, where="b > 0"), ZERO_AT_0) + built = m.constraints["row"] + assert built.labels.values.tolist()[0] == -1 + assert built.rhs.values.tolist()[1:] == pytest.approx(rhs) + + +@pytest.mark.parametrize( + "when", + [ + pytest.param("shift(a / b > 0.1, along=t, offset=1)", id="shift"), + pytest.param("at(a / b > 0.1, by=succ[u])", id="lookup"), + pytest.param("count(a / b > 0.1, over=t) >= 1", id="count"), + ], +) +def test_a_where_does_not_guard_what_a_condition_reads_at_another_coordinate( + when: str, +) -> None: + region = {"on": {"when": when, "expression": "a"}} + spec = with_( + quotient("x <= r", where="b > 0"), + expressions={"r": {"dims": ["t"], "cases": region, "otherwise": 0}}, + **SUCC, + ) + with pytest.raises(SpecDataError, match="divide of 'a', 'b' is not finite at t=0"): + Model.from_spec(spec, SUCC_DATA) + + +def test_a_where_drops_every_guard_along_the_axis_a_reader_moves() -> None: + """No kept row reads t=0, yet the shift along t is guarded nowhere on t.""" + spec = quotient("x <= shift(a / b, along=t, offset=1, edge=0)", where="t > 1") + with pytest.raises(SpecDataError, match="divide of 'a', 'b' is not finite at t=0"): + Model.from_spec(spec, ZERO_AT_0) + + +def test_a_where_does_not_guard_a_shift_summed_over_an_empty_dimension() -> None: + expression = "x + sum(shift(a / b * y, along=t, offset=1, edge=0), over=e) <= 1" + spec = with_( + quotient(expression, where="b > 0"), + dimensions={"e": {"dtype": "str"}}, + variables={"y": {"dims": ["t", "e"], "bounds": {"lower": 0, "upper": 1}}}, + ) + data = {**ZERO_AT_0, "e": pd.Index([], name="e", dtype=str)} + with pytest.raises(SpecDataError, match="divide of 'a', 'b' is not finite at t=0"): + Model.from_spec(spec, data) + + +@pytest.mark.parametrize( + ("expression", "rhs"), + [ + pytest.param( + "x <= shift(a / b, along=t, offset=1, edge=0)", [0.0, 0.5, 0.5], id="shift" + ), + pytest.param("x <= at(a / b, by=lag[u])", [0.5, 0.5, 0.5], id="lookup"), + ], +) +def test_a_where_guards_a_reader_along_the_axes_it_does_not_move( + expression: str, rhs: list[float] +) -> None: + """``b > 0`` masks s=p, where ``a / b`` is inf, and each reader moves along t only.""" + row = {"dims": ["t", "s"], "where": "b > 0", "expression": expression} + spec = with_( + quotient(""), + dimensions={"s": {"dtype": "str"}}, + relations={"lag": {"key": ["t", "s"], "values": {"u": "t"}}}, + parameters={"a": {"dims": ["t", "s"]}, "b": {"dims": ["t", "s"]}}, + variables={"x": {"dims": ["t", "s"], "bounds": {"lower": 0, "upper": 10}}}, + constraints={"row": row}, + objective={"sense": "minimize", "expression": "sum(x, over=[t, s])"}, + ) + lag = {"t": [0, 0, 1, 1, 2, 2], "s": ["p", "q"] * 3, "u": [0, 0, 0, 0, 1, 1]} + data = { + "t": T, + "s": GROUPS, + "lag": pd.DataFrame(lag), + "a": xr.DataArray(np.ones((3, 2)), coords=[T, GROUPS]), + "b": xr.DataArray([[0.0, 2.0]] * 3, coords=[T, GROUPS]), + } + built = Model.from_spec(spec, data).constraints["row"] + assert built.rhs.sel(s="q").values.tolist() == rhs + + @pytest.mark.parametrize( ("missing", "expression"), [(0, "size >= 0"), (5, "size >= 5")] )