From 5c7ace3d7a66aa1f689baf2afbbc62779cdf66af Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 07:09:44 +0000 Subject: [PATCH 1/6] fix(spec): check otherwise only where no case applies A case value, and the otherwise value, was evaluated over every row and masked to its region only afterwards, so the finiteness check fired at rows another case claims (e.g. `cap * pu` where a case guards `cap == inf`). Each case value is now evaluated with the context's `rows` narrowed to its region, the mechanism that already scopes the check to a constraint's built rows, both when building and when reading the data under an unbuilt sum over an empty dimension. Fixes #1148 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01F4nx8L63c3o8u6QQvr6TGN --- doc/release-notes.md | 1 + linopy/spec/evaluator.py | 28 ++++++++++++------ test/test_spec_builder.py | 61 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index c9836a1d..1f27cbd9 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -130,6 +130,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) +- `Model.from_spec` checks the value of a case, and of `otherwise:`, only where it applies, so a case guarding data like `cap == inf` no longer leaves `otherwise:` refused as not finite there. ([\#1148](https://github.com/PyPSA/linopy/issues/1148)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) diff --git a/linopy/spec/evaluator.py b/linopy/spec/evaluator.py index a51b2ca3..8ea25cda 100644 --- a/linopy/spec/evaluator.py +++ b/linopy/spec/evaluator.py @@ -5,6 +5,7 @@ import functools import operator from collections.abc import Callable, Sequence +from dataclasses import replace from typing import TypeAlias, TypeVar, assert_never, cast import numpy as np @@ -85,10 +86,7 @@ def _evaluated(node: ms.Expression, ctx: Context) -> Value: if isinstance(node, ms.WindowSum): return _moved(node, _operand(node.operand, ctx), ctx, None) if isinstance(node, ms.Cases): - regions = ( - _in_region(evaluate(region.value, ctx), evaluate_where(region.when, ctx)) - for region in node.regions - ) + regions = (_case(region, ctx) for region in node.regions) return functools.reduce(lambda a, b: operator.add(*_paired(a, b)), regions) assert_never(node) @@ -115,18 +113,18 @@ def _data_under(node: ms.Expression, ctx: Context) -> None: variable, the conditions of its cases, the amount and partition of a shift or window. A use of every other operand is consumed unbuilt. """ + scopes = [ctx] * len(ms.children(node)) if isinstance(node, ms.Cases): - for region in node.regions: - evaluate_where(region.when, ctx) + scopes = [_within(evaluate_where(r.when, ctx), ctx) for r in node.regions] if isinstance(node, ms.Translate | ms.WindowSum): _partition(node.partition, ctx) _amount(_amount_of(node), ctx) - for operand in ms.children(node): + for operand, scope in zip(ms.children(node), scopes): if not ms.carries_variable(operand): - evaluate(operand, ctx) + evaluate(operand, scope) continue ctx.memo.used(operand) - _data_under(operand, ctx) + _data_under(operand, scope) def _sum( @@ -357,6 +355,18 @@ def _in_region(value: Value, rows: xr.DataArray) -> Value: return as_expression(value).where(rows, 0) +def _case(region: ms.Region, ctx: Context) -> Value: + """The value of one case of a :class:`~mathspec.program.Cases`, evaluated within its region and a hard zero outside it.""" + rows = evaluate_where(region.when, ctx) + return _in_region(evaluate(region.value, _within(rows, ctx)), rows) + + +def _within(region: xr.DataArray, ctx: Context) -> Context: + """*ctx* narrowed to the rows of a case's *region*, so its value is refused only where it applies, as :func:`_marks` reads it.""" + rows = region if ctx.rows is None else ctx.rows & region + return replace(ctx, rows=rows) + + def _amount(amount: int | str, ctx: Context) -> operators.Amount: """An amount, a ``refused`` hole moving nothing: no live row reads it, as :func:`refuse_holes` checks.""" if not isinstance(amount, str): diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index c1eb74d1..b6801e45 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -627,6 +627,67 @@ def test_arithmetic_is_not_refused_on_a_row_masked_out() -> None: assert int(m.constraints["row"].labels.sel(t=0)) == -1 +def guarded(row: str, cases: dict[str, Any], otherwise: Any) -> dict[str, Any]: + """:func:`quotient` with a row reading the named expression ``used``, defined by *cases*.""" + used = {"dims": ["t"], "cases": cases, "otherwise": otherwise} + return with_(quotient(row), expressions={"used": used}) + + +UNRATED = {"unrated": {"when": "a == inf", "expression": 1}} +RATED = {"rated": {"when": "a < inf", "expression": "a * b"}} + + +@pytest.mark.parametrize( + ("cases", "otherwise"), + [ + pytest.param(UNRATED, "a * b", id="otherwise"), + pytest.param(RATED, 1, id="case"), + ], +) +def test_a_case_value_is_not_refused_where_another_case_applies( + cases: dict[str, Any], otherwise: Any +) -> None: + """``a * b`` is inf * 0 on ``t=0``, which a case claims: a value is read only within its region (#1148).""" + m = Model.from_spec(guarded("x <= used", cases, otherwise), INF_AT_0) + row = m.constraints["row"] + assert row.rhs.values.tolist() == [1.0, 4.0, 5.0] + assert row.coeffs.squeeze().values.tolist() == [1.0, 1.0, 1.0] + + +def test_a_case_term_is_not_refused_where_another_case_applies() -> None: + spec = guarded( + "used <= 3", {"unrated": {"when": "a == inf", "expression": "x"}}, "a * b * x" + ) + row = Model.from_spec(spec, INF_AT_0).constraints["row"] + assert row.coeffs.where(row.vars != -1).sum("_term").values.tolist() == [1, 4, 5] + assert row.rhs.values.tolist() == [3.0, 3.0, 3.0] + + +def test_a_case_value_is_refused_within_its_region() -> None: + spec = guarded("x <= used", {"high": {"when": "b > 4", "expression": 1}}, "a * b") + with pytest.raises(SpecDataError, match=r"constraint 'row'.*not finite at t=0"): + Model.from_spec(spec, INF_AT_0) + + +def test_a_case_value_over_an_empty_dimension_is_read_within_its_region() -> None: + """A sum over a dimension without a coordinate builds no term but reads its data, a case's value within its region.""" + spec = with_( + quotient("x <= sum(used, over=s)"), + dimensions={"s": {"dtype": "int"}}, + variables={"y": {"dims": ["s"], "bounds": {"lower": 0, "upper": 1}}}, + expressions={ + "used": { + "dims": ["t", "s"], + "cases": {"unrated": {"when": "a == inf", "expression": "y"}}, + "otherwise": "a * b * y", + } + }, + ) + data = {**INF_AT_0, "s": pd.Index([], dtype=int, name="s")} + row = Model.from_spec(spec, data).constraints["row"] + assert row.rhs.values.tolist() == [0.0, 0.0, 0.0] + + GATED_ON = { "variables": { "on": { From 6412918b3380cc37bf118acd553bdbf1dfc7a58c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:01:48 +0000 Subject: [PATCH 2/6] fix(spec): a where does not guard what a shift, window or lookup reads elsewhere The rows a constraint's where keeps narrow the finiteness checks of what it reads. A sum, a window, a shift and a lookup read their operand at other coordinates along the axes they move. So the narrowing is widened along those axes to every row there. It stays along the other axes, where they read their own coordinate. Before, where: b > 0 with x <= shift(a / b, edge=0) built x[1] <= inf without a word. The widening reads only ctx.rows, so it applies to any narrowing. --- doc/release-notes.md | 1 + linopy/spec/evaluator.py | 37 ++++++++++++++++++---- test/test_spec_builder.py | 66 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index a1c3ff1e..1f34b171 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -131,6 +131,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) +- `Model.from_spec` no longer lets a `where` guard what a `shift`, window, `sum` or `at()` lookup reads at another coordinate, so a non-finite value there raises instead of entering a row. ([\#PRNUM](https://github.com/PyPSA/linopy/pull/PRNUM)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) diff --git a/linopy/spec/evaluator.py b/linopy/spec/evaluator.py index a51b2ca3..865c42d3 100644 --- a/linopy/spec/evaluator.py +++ b/linopy/spec/evaluator.py @@ -5,6 +5,7 @@ import functools import operator from collections.abc import Callable, Sequence +from dataclasses import replace from typing import TypeAlias, TypeVar, assert_never, cast import numpy as np @@ -79,11 +80,11 @@ def _evaluated(node: ms.Expression, ctx: Context) -> Value: if isinstance(node, ms.Sum): return _sum(node, ctx) if isinstance(node, ms.Join): - return looked_up(_operand(node.operand, ctx), node.columns, ctx) + return looked_up(_operand(node.operand, _inner(node, ctx)), node.columns, ctx) if isinstance(node, ms.Translate): - return _moved(node, _operand(node.operand, ctx), ctx, node.fill) + return _moved(node, _operand(node.operand, _inner(node, ctx)), ctx, node.fill) if isinstance(node, ms.WindowSum): - return _moved(node, _operand(node.operand, ctx), ctx, None) + return _moved(node, _operand(node.operand, _inner(node, ctx)), ctx, None) if isinstance(node, ms.Cases): regions = ( _in_region(evaluate(region.value, ctx), evaluate_where(region.when, ctx)) @@ -93,6 +94,29 @@ def _evaluated(node: ms.Expression, ctx: Context) -> Value: assert_never(node) +def _inner(node: ms.Expression, ctx: Context) -> Context: + """ + *ctx* for the operand of *node*: a sum, a window, a shift and a lookup + read their operand at other coordinates along the axes they move, so the + rows *ctx* checks are widened along those axes to every row there. + """ + if ctx.rows is None: + return ctx + along = [d for d in _moved_dims(node) if d in ctx.rows.dims] + return replace(ctx, rows=ctx.rows.any(along)) if along else ctx + + +def _moved_dims(node: ms.Expression) -> tuple[str, ...]: + """The dimensions along which *node* reads its operand at other coordinates than its own.""" + if isinstance(node, ms.Sum): + return tuple(axis.dimension for axis in node.over) + if isinstance(node, ms.Translate | ms.WindowSum): + return (node.along,) + if isinstance(node, ms.Join): + return node.columns.dropped_dims + return () + + def _sums_nothing(node: ms.Expression, ctx: Context) -> bool: """Whether *node* is over a dimension without a coordinate, or sums an operand that is.""" return is_empty((node.operand if isinstance(node, ms.Sum) else node).dims, ctx) @@ -121,12 +145,13 @@ def _data_under(node: ms.Expression, ctx: Context) -> None: if isinstance(node, ms.Translate | ms.WindowSum): _partition(node.partition, ctx) _amount(_amount_of(node), ctx) + inner = _inner(node, ctx) for operand in ms.children(node): if not ms.carries_variable(operand): - evaluate(operand, ctx) + evaluate(operand, inner) continue ctx.memo.used(operand) - _data_under(operand, ctx) + _data_under(operand, inner) def _sum( @@ -137,7 +162,7 @@ def _sum( the operand through the relation, the plain ones by summing them away. *of* evaluates an operand, :func:`evaluate` when not given. """ - value = functools.partial(_operand, ctx=ctx) if of is None else of + value = functools.partial(_operand, ctx=_inner(node, ctx)) if of is None else of operand = node.operand plain = [axis.dimension for axis in node.over if axis.column is None] if isinstance(operand, ms.Join) and operand.columns.axes: diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index a699285a..02ae771a 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -666,6 +666,72 @@ def test_arithmetic_masked_out_for_one_row_is_refused_for_the_next_reading_it() Model.from_spec(spec, ZERO_AT_0) +SUCC = {"relations": {"succ": {"key": "t", "values": {"u": "t"}}}} +SUCC_DATA = {**ZERO_AT_0, "succ": pd.DataFrame({"t": T, "u": [1, 0, 1]})} + + +@pytest.mark.parametrize( + "expression", + [ + pytest.param("x <= shift(a / b, along=t, offset=1, edge=0)", id="shift"), + pytest.param("x <= sum_back(a / b, along=t, window=2)", id="window"), + pytest.param("x <= at(a / b, by=succ[u])", id="lookup"), + pytest.param("x <= sum(a / b, over=t)", id="sum"), + pytest.param("shift(a / b * x, along=t, offset=1, edge=0) <= 1", id="term"), + ], +) +def test_a_where_does_not_guard_what_a_row_reads_at_another_coordinate( + expression: str, +) -> None: + """``b > 0`` masks t=0 only, and row t=1 reads ``a / b`` at t=0.""" + spec = with_(quotient(expression, where="b > 0"), **SUCC) + with pytest.raises(SpecDataError, match="divide of 'a', 'b' is not finite at t=0"): + Model.from_spec(spec, SUCC_DATA) + + +@pytest.mark.parametrize( + ("expression", "rhs"), + [ + pytest.param("x <= shift(a, along=t, offset=1, edge=0) / b", [0.0, 0.2]), + pytest.param("x <= a / b + shift(a, along=t, offset=1, edge=0)", [0.25, 1.2]), + pytest.param("x <= sum(a, over=t) / b", [0.5, 0.4]), + ], + ids=["shift-then-divide", "beside-a-shift", "sum-then-divide"], +) +def test_a_where_guards_what_a_row_reads_at_its_own_coordinate( + expression: str, rhs: list[float] +) -> None: + m = Model.from_spec(quotient(expression, where="b > 0"), ZERO_AT_0) + built = m.constraints["row"] + assert built.labels.values.tolist()[0] == -1 + assert built.rhs.values.tolist()[1:] == pytest.approx(rhs) + + +def test_a_where_guards_a_shift_along_the_axes_it_does_not_move() -> None: + """``b > 0`` masks s=p, where ``a / b`` is inf, and the shift reads s=q only.""" + row = { + "dims": ["t", "s"], + "where": "b > 0", + "expression": "x <= shift(a / b, along=t, offset=1, edge=0)", + } + spec = with_( + quotient(""), + dimensions={"s": {"dtype": "str"}}, + parameters={"a": {"dims": ["t", "s"]}, "b": {"dims": ["s"]}}, + variables={"x": {"dims": ["t", "s"], "bounds": {"lower": 0, "upper": 10}}}, + constraints={"row": row}, + objective={"sense": "minimize", "expression": "sum(x, over=[t, s])"}, + ) + data = { + "t": T, + "s": GROUPS, + "a": xr.DataArray(np.ones((3, 2)), coords=[T, GROUPS]), + "b": xr.DataArray([0.0, 2.0], coords=[GROUPS]), + } + built = Model.from_spec(spec, data).constraints["row"] + assert built.rhs.sel(s="q").values.tolist() == [0.0, 0.5, 0.5] + + @pytest.mark.parametrize( ("missing", "expression"), [(0, "size >= 0"), (5, "size >= 5")] ) From 7cd6e72466f81508e86b6875ce39e671181a91fd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:02:31 +0000 Subject: [PATCH 3/6] docs: name #1173 in the release note --- doc/release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index 1f34b171..8124d770 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -131,7 +131,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) -- `Model.from_spec` no longer lets a `where` guard what a `shift`, window, `sum` or `at()` lookup reads at another coordinate, so a non-finite value there raises instead of entering a row. ([\#PRNUM](https://github.com/PyPSA/linopy/pull/PRNUM)) +- `Model.from_spec` no longer lets a `where` guard what a `shift`, window, `sum` or `at()` lookup reads at another coordinate, so a non-finite value there raises instead of entering a row. ([\#1173](https://github.com/PyPSA/linopy/pull/1173)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) From 8e6cedc59b92405085a68def06183371721fa559 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:11:58 +0000 Subject: [PATCH 4/6] fix(spec): skip a case that applies nowhere A case whose region holds no row is not evaluated, and its uses are consumed unread. A zero over the dimensions of the cases stands in for it, a term when the cases carry a variable. Its data is no longer checked, so a window wider than its axis there no longer raises. Tests pin that a shift or window in a case value is still refused where it reads a non-finite value, and cover the fold path. The release note is one line naming this PR. --- doc/release-notes.md | 2 +- linopy/spec/evaluator.py | 70 ++++++++++++++++++++++++++++----------- test/test_spec_builder.py | 66 +++++++++++++++++++++++++++++++++--- 3 files changed, 113 insertions(+), 25 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index f2da9ece..da5f89c2 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -131,8 +131,8 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) -- `Model.from_spec` checks the value of a case, and of `otherwise:`, only where it applies, so a case guarding data like `cap == inf` no longer leaves `otherwise:` refused as not finite there. ([\#1148](https://github.com/PyPSA/linopy/issues/1148)) - `Model.from_spec` no longer lets a `where` guard what a `shift`, window, `sum` or `at()` lookup reads at another coordinate, so a non-finite value there raises instead of entering a row. ([\#1173](https://github.com/PyPSA/linopy/pull/1173)) +- `Model.from_spec` checks a case's value, `otherwise:` included, only where that case applies, and skips a case that applies nowhere, so a guard like `cap == inf` no longer leaves `otherwise:` refused there. ([\#1163](https://github.com/PyPSA/linopy/pull/1163)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) diff --git a/linopy/spec/evaluator.py b/linopy/spec/evaluator.py index d6e15df1..caa4ea7b 100644 --- a/linopy/spec/evaluator.py +++ b/linopy/spec/evaluator.py @@ -4,7 +4,7 @@ import functools import operator -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence from dataclasses import replace from typing import TypeAlias, TypeVar, assert_never, cast @@ -86,8 +86,7 @@ def _evaluated(node: ms.Expression, ctx: Context) -> Value: if isinstance(node, ms.WindowSum): return _moved(node, _operand(node.operand, _inner(node, ctx)), ctx, None) if isinstance(node, ms.Cases): - regions = (_case(region, ctx) for region in node.regions) - return functools.reduce(lambda a, b: operator.add(*_paired(a, b)), regions) + return _cases(node, ctx) assert_never(node) @@ -136,18 +135,25 @@ def _data_under(node: ms.Expression, ctx: Context) -> None: variable, the conditions of its cases, the amount and partition of a shift or window. A use of every other operand is consumed unbuilt. """ - scopes = [_inner(node, ctx)] * len(ms.children(node)) if isinstance(node, ms.Cases): - scopes = [_within(evaluate_where(r.when, ctx), ctx) for r in node.regions] + for region, rows in _regions(node, ctx): + _data_of(region.value, _narrowed(ctx, rows)) + return if isinstance(node, ms.Translate | ms.WindowSum): _partition(node.partition, ctx) _amount(_amount_of(node), ctx) - for operand, scope in zip(ms.children(node), scopes): - if not ms.carries_variable(operand): - evaluate(operand, scope) - continue - ctx.memo.used(operand) - _data_under(operand, scope) + inner = _inner(node, ctx) + for operand in ms.children(node): + _data_of(operand, inner) + + +def _data_of(operand: ms.Expression, ctx: Context) -> None: + """Evaluate *operand* if it holds no variable, else what it reads beside its terms.""" + if not ms.carries_variable(operand): + evaluate(operand, ctx) + return + ctx.memo.used(operand) + _data_under(operand, ctx) def _sum( @@ -378,16 +384,40 @@ def _in_region(value: Value, rows: xr.DataArray) -> Value: return as_expression(value).where(rows, 0) -def _case(region: ms.Region, ctx: Context) -> Value: - """The value of one case of a :class:`~mathspec.program.Cases`, evaluated within its region and a hard zero outside it.""" - rows = evaluate_where(region.when, ctx) - return _in_region(evaluate(region.value, _within(rows, ctx)), rows) - +def _cases(node: ms.Cases, ctx: Context) -> Value: + """ + The cases of *node* added up, each its value where it applies and a hard + zero elsewhere. -def _within(region: xr.DataArray, ctx: Context) -> Context: - """*ctx* narrowed to the rows of a case's *region*, so its value is refused only where it applies, as :func:`_marks` reads it.""" - rows = region if ctx.rows is None else ctx.rows & region - return replace(ctx, rows=rows) + A case's value is evaluated and checked only where it applies. A case + that applies nowhere is not evaluated, as its value is never read: a + zero over the dimensions of *node* stands in for it. + """ + parts = [ + _in_region(evaluate(region.value, _narrowed(ctx, rows)), rows) + for region, rows in _regions(node, ctx) + ] + if len(parts) < len(node.regions): + parts.append(_zeros(node, ctx)) + total = functools.reduce(lambda a, b: operator.add(*_paired(a, b)), parts) + if not ctx.solved and ms.carries_variable(node) and isinstance(total, xr.DataArray): + return operators.constant_term(ctx.model, total) + return total + + +def _regions(node: ms.Cases, ctx: Context) -> Iterator[tuple[ms.Region, xr.DataArray]]: + """The cases of *node* that apply somewhere, each with where it applies; a use of every other is consumed unread.""" + for region in node.regions: + rows = evaluate_where(region.when, ctx) + if bool(rows.any()): + yield region, rows + else: + ctx.memo.skipped(region.value) + + +def _narrowed(ctx: Context, rows: xr.DataArray) -> Context: + """*ctx* checking only the rows *rows* keeps of the ones it checks, so a case's value is refused only where it applies, as :func:`_marks` reads it.""" + return replace(ctx, rows=rows if ctx.rows is None else ctx.rows & rows) def _amount(amount: int | str, ctx: Context) -> operators.Amount: diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index 54d549d5..f6409e04 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -44,6 +44,7 @@ import linopy # noqa: E402 from linopy import Model # noqa: E402 +from linopy.constants import TERM_DIM # noqa: E402 from linopy.spec import SpecDataError, evaluate # noqa: E402 from linopy.spec.api import lower # noqa: E402 from linopy.testing import ( # noqa: E402 @@ -650,10 +651,13 @@ def test_a_case_value_is_not_refused_where_another_case_applies( cases: dict[str, Any], otherwise: Any ) -> None: """``a * b`` is inf * 0 on ``t=0``, which a case claims: a value is read only within its region (#1148).""" - m = Model.from_spec(guarded("x <= used", cases, otherwise), INF_AT_0) + spec = guarded("x <= used", cases, otherwise) + m = Model.from_spec(spec, INF_AT_0) row = m.constraints["row"] assert row.rhs.values.tolist() == [1.0, 4.0, 5.0] assert row.coeffs.squeeze().values.tolist() == [1.0, 1.0, 1.0] + folded = evaluate(spec, m, "used", sources=INF_AT_0)["used"] + assert folded.values.tolist() == [1.0, 4.0, 5.0] def test_a_case_term_is_not_refused_where_another_case_applies() -> None: @@ -661,16 +665,70 @@ def test_a_case_term_is_not_refused_where_another_case_applies() -> None: "used <= 3", {"unrated": {"when": "a == inf", "expression": "x"}}, "a * b * x" ) row = Model.from_spec(spec, INF_AT_0).constraints["row"] - assert row.coeffs.where(row.vars != -1).sum("_term").values.tolist() == [1, 4, 5] + assert row.coeffs.where(row.vars != -1).sum(TERM_DIM).values.tolist() == [1, 4, 5] assert row.rhs.values.tolist() == [3.0, 3.0, 3.0] -def test_a_case_value_is_refused_within_its_region() -> None: - spec = guarded("x <= used", {"high": {"when": "b > 4", "expression": 1}}, "a * b") +SHIFTED_PRODUCT = "shift(a * b, along=t, offset=1, edge=0)" + + +@pytest.mark.parametrize( + ("cases", "otherwise"), + [ + pytest.param({"high": {"when": "b > 4", "expression": 1}}, "a * b", id="own"), + pytest.param( + {"pos": {"when": "b > 0", "expression": SHIFTED_PRODUCT}}, 0, id="shift" + ), + pytest.param(UNRATED, SHIFTED_PRODUCT, id="otherwise-shift"), + pytest.param( + UNRATED, "sum_back(a * b, along=t, window=2)", id="otherwise-window" + ), + ], +) +def test_a_case_value_is_refused_where_it_is_read( + cases: dict[str, Any], otherwise: Any +) -> None: + """``a * b`` is inf * 0 on ``t=0``, read there by the case that applies, or from ``t=1`` by a shift or window.""" + spec = guarded("x <= used", cases, otherwise) with pytest.raises(SpecDataError, match=r"constraint 'row'.*not finite at t=0"): Model.from_spec(spec, INF_AT_0) +@pytest.mark.parametrize( + ("row", "dims", "case", "rhs"), + [ + pytest.param( + "x <= used", + ["t"], + "sum_back(b, along=t, window=5)", + [0.0, 4.0, 5.0], + id="unread", + ), + pytest.param( + "x <= sum(used, over=s)", ["t", "s"], "y", [0.0, 8.0, 10.0], id="zero" + ), + ], +) +def test_a_case_that_applies_nowhere_is_not_evaluated( + row: str, dims: list[str], case: str, rhs: list[float] +) -> None: + """No ``b`` is negative, so a window wider than ``t`` is never read, and the case is zero over its dimensions.""" + spec = with_( + quotient(row), + dimensions={"s": {"dtype": "str"}}, + variables={"y": {"dims": ["s"], "bounds": {"lower": 0, "upper": 1}}}, + expressions={ + "used": { + "dims": dims, + "cases": {"neg": {"when": "b < 0", "expression": case}}, + "otherwise": "b", + } + }, + ) + built = Model.from_spec(spec, {**INF_AT_0, "s": GROUPS}).constraints["row"] + assert built.rhs.values.tolist() == rhs + + def test_a_case_value_over_an_empty_dimension_is_read_within_its_region() -> None: """A sum over a dimension without a coordinate builds no term but reads its data, a case's value within its region.""" spec = with_( From ddf74f990c619a8b64fb16591f2ccd026aacf4cf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:21:25 +0000 Subject: [PATCH 5/6] fix(spec): a where does not guard what a shifted, looked-up or counted condition reads elsewhere A shift, an at() and a count of a condition read their operand at other coordinates, as their expression forms do. So the rows a where keeps are widened along the axes they move, the same way. Before, where: b > 0 with a case on shift(a / b > 0.1, along=t, offset=1) read the NaN of a / b at t=0 as false and built row t=1 without a word. The widening moves to context.py, so the evaluator and the conditions share it. --- doc/release-notes.md | 2 +- linopy/spec/context.py | 27 +++++++++++++++++++++++- linopy/spec/evaluator.py | 44 +++++++++++++-------------------------- linopy/spec/where.py | 10 +++++---- test/test_spec_builder.py | 21 +++++++++++++++++++ 5 files changed, 68 insertions(+), 36 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index 8124d770..2a88e23f 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -131,7 +131,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) -- `Model.from_spec` no longer lets a `where` guard what a `shift`, window, `sum` or `at()` lookup reads at another coordinate, so a non-finite value there raises instead of entering a row. ([\#1173](https://github.com/PyPSA/linopy/pull/1173)) +- `Model.from_spec` no longer lets a `where` guard what a `shift`, window, `sum`, `count` or `at()` lookup reads at another coordinate, so a non-finite value there raises instead of entering a row or a `when:`. ([\#1173](https://github.com/PyPSA/linopy/pull/1173)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) diff --git a/linopy/spec/context.py b/linopy/spec/context.py index 12a0ac34..79c368a1 100644 --- a/linopy/spec/context.py +++ b/linopy/spec/context.py @@ -12,7 +12,7 @@ from collections import Counter from collections.abc import Callable, Hashable, Iterable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, TypeVar, cast, overload import pandas as pd @@ -155,3 +155,28 @@ class Context: def is_empty(dims: Iterable[str], ctx: Context) -> bool: """Whether one of *dims* has no coordinate, so what is over them has no cell.""" return any(not len(ctx.coords[d]) for d in dims) + + +def widened(node: ms.Expression | ms.Condition, ctx: Context) -> Context: + """ + *ctx* for the operand of *node*: a sum, a window, a shift, a lookup and a + count read their operand at other coordinates along the axes they move, so + the rows *ctx* checks are widened along those axes to every row there. + """ + if ctx.rows is None: + return ctx + along = [d for d in _moved_dims(node) if d in ctx.rows.dims] + return replace(ctx, rows=ctx.rows.any(along)) if along else ctx + + +def _moved_dims(node: ms.Expression | ms.Condition) -> tuple[str, ...]: + """The dimensions along which *node* reads its operand at other coordinates than its own.""" + if isinstance(node, ms.Sum): + return tuple(axis.dimension for axis in node.over) + if isinstance(node, ms.CountComparison): + return (node.over,) + if isinstance(node, ms.Translate | ms.WindowSum | ms.TranslatedCondition): + return (node.along,) + if isinstance(node, ms.Join | ms.JoinedCondition): + return node.columns.dropped_dims + return () diff --git a/linopy/spec/evaluator.py b/linopy/spec/evaluator.py index 865c42d3..c67a7a24 100644 --- a/linopy/spec/evaluator.py +++ b/linopy/spec/evaluator.py @@ -5,7 +5,6 @@ import functools import operator from collections.abc import Callable, Sequence -from dataclasses import replace from typing import TypeAlias, TypeVar, assert_never, cast import numpy as np @@ -15,7 +14,15 @@ from linopy.constants import FACTOR_DIM from linopy.expressions import LinearExpression, QuadraticExpression from linopy.spec import operators -from linopy.spec.context import Array, Context, Term, Value, as_expression, is_empty +from linopy.spec.context import ( + Array, + Context, + Term, + Value, + as_expression, + is_empty, + widened, +) from linopy.spec.errors import SpecDataError, first_coordinates from linopy.spec.where import admitted, evaluate_where, groups_of, looked_up from linopy.variables import Variable @@ -80,11 +87,11 @@ def _evaluated(node: ms.Expression, ctx: Context) -> Value: if isinstance(node, ms.Sum): return _sum(node, ctx) if isinstance(node, ms.Join): - return looked_up(_operand(node.operand, _inner(node, ctx)), node.columns, ctx) + return looked_up(_operand(node.operand, widened(node, ctx)), node.columns, ctx) if isinstance(node, ms.Translate): - return _moved(node, _operand(node.operand, _inner(node, ctx)), ctx, node.fill) + return _moved(node, _operand(node.operand, widened(node, ctx)), ctx, node.fill) if isinstance(node, ms.WindowSum): - return _moved(node, _operand(node.operand, _inner(node, ctx)), ctx, None) + return _moved(node, _operand(node.operand, widened(node, ctx)), ctx, None) if isinstance(node, ms.Cases): regions = ( _in_region(evaluate(region.value, ctx), evaluate_where(region.when, ctx)) @@ -94,29 +101,6 @@ def _evaluated(node: ms.Expression, ctx: Context) -> Value: assert_never(node) -def _inner(node: ms.Expression, ctx: Context) -> Context: - """ - *ctx* for the operand of *node*: a sum, a window, a shift and a lookup - read their operand at other coordinates along the axes they move, so the - rows *ctx* checks are widened along those axes to every row there. - """ - if ctx.rows is None: - return ctx - along = [d for d in _moved_dims(node) if d in ctx.rows.dims] - return replace(ctx, rows=ctx.rows.any(along)) if along else ctx - - -def _moved_dims(node: ms.Expression) -> tuple[str, ...]: - """The dimensions along which *node* reads its operand at other coordinates than its own.""" - if isinstance(node, ms.Sum): - return tuple(axis.dimension for axis in node.over) - if isinstance(node, ms.Translate | ms.WindowSum): - return (node.along,) - if isinstance(node, ms.Join): - return node.columns.dropped_dims - return () - - def _sums_nothing(node: ms.Expression, ctx: Context) -> bool: """Whether *node* is over a dimension without a coordinate, or sums an operand that is.""" return is_empty((node.operand if isinstance(node, ms.Sum) else node).dims, ctx) @@ -145,7 +129,7 @@ def _data_under(node: ms.Expression, ctx: Context) -> None: if isinstance(node, ms.Translate | ms.WindowSum): _partition(node.partition, ctx) _amount(_amount_of(node), ctx) - inner = _inner(node, ctx) + inner = widened(node, ctx) for operand in ms.children(node): if not ms.carries_variable(operand): evaluate(operand, inner) @@ -162,7 +146,7 @@ def _sum( the operand through the relation, the plain ones by summing them away. *of* evaluates an operand, :func:`evaluate` when not given. """ - value = functools.partial(_operand, ctx=_inner(node, ctx)) if of is None else of + value = functools.partial(_operand, ctx=widened(node, ctx)) if of is None else of operand = node.operand plain = [axis.dimension for axis in node.over if axis.column is None] if isinstance(operand, ms.Join) and operand.columns.axes: diff --git a/linopy/spec/where.py b/linopy/spec/where.py index 0079ee19..a42ebf11 100644 --- a/linopy/spec/where.py +++ b/linopy/spec/where.py @@ -11,7 +11,7 @@ from mathspec import program as ms from linopy.spec import operators -from linopy.spec.context import Array, Context +from linopy.spec.context import Array, Context, widened from linopy.spec.errors import SpecDataError _COMPARISONS: dict[str, Callable[..., xr.DataArray]] = { @@ -108,13 +108,15 @@ def _condition(node: ms.Condition, ctx: Context) -> xr.DataArray: if isinstance(node, ms.ExpressionComparison): return _both(node.op, _side(node.left, ctx), _side(node.right, ctx)) if isinstance(node, ms.CountComparison): - count = _node(node.condition.root, ctx).sum(node.over) + count = _node(node.condition.root, widened(node, ctx)).sum(node.over) return _COMPARISONS[node.op](count, node.value) if isinstance(node, ms.TranslatedCondition): - shifted = _node(node.operand.root, ctx).shift({node.along: node.offset}) + inner = widened(node, ctx) + shifted = _node(node.operand.root, inner).shift({node.along: node.offset}) return _bool(shifted) if isinstance(node, ms.JoinedCondition): - pulled = looked_up(_node(node.operand.root, ctx), node.columns, ctx) + inner = widened(node, ctx) + pulled = looked_up(_node(node.operand.root, inner), node.columns, ctx) return _bool(cast(xr.DataArray, pulled)) if isinstance(node, ms.DimensionComparison): labels = ctx.coords[node.name] diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index 02ae771a..304a3c72 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -707,6 +707,27 @@ def test_a_where_guards_what_a_row_reads_at_its_own_coordinate( assert built.rhs.values.tolist()[1:] == pytest.approx(rhs) +@pytest.mark.parametrize( + "when", + [ + pytest.param("shift(a / b > 0.1, along=t, offset=1)", id="shift"), + pytest.param("at(a / b > 0.1, by=succ[u])", id="lookup"), + pytest.param("count(a / b > 0.1, over=t) >= 1", id="count"), + ], +) +def test_a_where_does_not_guard_what_a_condition_reads_at_another_coordinate( + when: str, +) -> None: + region = {"on": {"when": when, "expression": "a"}} + spec = with_( + quotient("x <= r", where="b > 0"), + expressions={"r": {"dims": ["t"], "cases": region, "otherwise": 0}}, + **SUCC, + ) + with pytest.raises(SpecDataError, match="divide of 'a', 'b' is not finite at t=0"): + Model.from_spec(spec, SUCC_DATA) + + def test_a_where_guards_a_shift_along_the_axes_it_does_not_move() -> None: """``b > 0`` masks s=p, where ``a / b`` is inf, and the shift reads s=q only.""" row = { From 2a52459892e3e26e01f93b01a0c7a0e79aae74a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:21:25 +0000 Subject: [PATCH 6/6] test(spec): pin what a where guards along the axes a reader moves and keeps A guard over t and s keeps column p out of a shift and a lookup along t, so dropping the guard fails. The lookup keyed by t and s also fails when it widens along the key column it carries. A shift summed over an empty dimension and a where: t > 1 around a shift are refused, the second as mathspec's literal rule asks. --- test/test_spec_builder.py | 49 +++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index 304a3c72..dcfa10b9 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -728,29 +728,58 @@ def test_a_where_does_not_guard_what_a_condition_reads_at_another_coordinate( Model.from_spec(spec, SUCC_DATA) -def test_a_where_guards_a_shift_along_the_axes_it_does_not_move() -> None: - """``b > 0`` masks s=p, where ``a / b`` is inf, and the shift reads s=q only.""" - row = { - "dims": ["t", "s"], - "where": "b > 0", - "expression": "x <= shift(a / b, along=t, offset=1, edge=0)", - } +def test_a_where_drops_every_guard_along_the_axis_a_reader_moves() -> None: + """No kept row reads t=0, yet the shift along t is guarded nowhere on t.""" + spec = quotient("x <= shift(a / b, along=t, offset=1, edge=0)", where="t > 1") + with pytest.raises(SpecDataError, match="divide of 'a', 'b' is not finite at t=0"): + Model.from_spec(spec, ZERO_AT_0) + + +def test_a_where_does_not_guard_a_shift_summed_over_an_empty_dimension() -> None: + expression = "x + sum(shift(a / b * y, along=t, offset=1, edge=0), over=e) <= 1" + spec = with_( + quotient(expression, where="b > 0"), + dimensions={"e": {"dtype": "str"}}, + variables={"y": {"dims": ["t", "e"], "bounds": {"lower": 0, "upper": 1}}}, + ) + data = {**ZERO_AT_0, "e": pd.Index([], name="e", dtype=str)} + with pytest.raises(SpecDataError, match="divide of 'a', 'b' is not finite at t=0"): + Model.from_spec(spec, data) + + +@pytest.mark.parametrize( + ("expression", "rhs"), + [ + pytest.param( + "x <= shift(a / b, along=t, offset=1, edge=0)", [0.0, 0.5, 0.5], id="shift" + ), + pytest.param("x <= at(a / b, by=lag[u])", [0.5, 0.5, 0.5], id="lookup"), + ], +) +def test_a_where_guards_a_reader_along_the_axes_it_does_not_move( + expression: str, rhs: list[float] +) -> None: + """``b > 0`` masks s=p, where ``a / b`` is inf, and each reader moves along t only.""" + row = {"dims": ["t", "s"], "where": "b > 0", "expression": expression} spec = with_( quotient(""), dimensions={"s": {"dtype": "str"}}, - parameters={"a": {"dims": ["t", "s"]}, "b": {"dims": ["s"]}}, + relations={"lag": {"key": ["t", "s"], "values": {"u": "t"}}}, + parameters={"a": {"dims": ["t", "s"]}, "b": {"dims": ["t", "s"]}}, variables={"x": {"dims": ["t", "s"], "bounds": {"lower": 0, "upper": 10}}}, constraints={"row": row}, objective={"sense": "minimize", "expression": "sum(x, over=[t, s])"}, ) + lag = {"t": [0, 0, 1, 1, 2, 2], "s": ["p", "q"] * 3, "u": [0, 0, 0, 0, 1, 1]} data = { "t": T, "s": GROUPS, + "lag": pd.DataFrame(lag), "a": xr.DataArray(np.ones((3, 2)), coords=[T, GROUPS]), - "b": xr.DataArray([0.0, 2.0], coords=[GROUPS]), + "b": xr.DataArray([[0.0, 2.0]] * 3, coords=[T, GROUPS]), } built = Model.from_spec(spec, data).constraints["row"] - assert built.rhs.sel(s="q").values.tolist() == [0.0, 0.5, 0.5] + assert built.rhs.sel(s="q").values.tolist() == rhs @pytest.mark.parametrize(