From 80b3e7ea7dd11a484cd061c882054b678bb6812e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:02:57 +0000 Subject: [PATCH 1/2] fix(spec): a window sum over absent terms is 0, like sum sum_back left a row absent where every term of its window was absent. sum over the same terms gives 0. A window now fills each absent term with 0 and sums, as sum does, so the row stays. This also holds where the window reaches nothing: a member a partition places nowhere, or a width no member carries. A shift without edge= still leaves its row absent. --- doc/release-notes.md | 1 + doc/user-guide/spec-data.md | 1 + linopy/spec/operators.py | 14 ++++++-------- test/test_spec_builder.py | 1 - test/test_spec_operators.py | 36 ++++++++++++++++++++++++++++++++---- 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index a1c3ff1e..8f9d9dac 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -131,6 +131,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) +- `Model.from_spec` builds a `sum_back` window that holds only absent terms as 0, like `sum`, where it dropped the row. ([\#1174](https://github.com/PyPSA/linopy/pull/1174)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) diff --git a/doc/user-guide/spec-data.md b/doc/user-guide/spec-data.md index 91c2fa95..f8606072 100644 --- a/doc/user-guide/spec-data.md +++ b/doc/user-guide/spec-data.md @@ -297,6 +297,7 @@ The other operators read past an edge in their own way: - `sum_back(p, along=s, window=3)` is a sum, so it shortens its window at the start rather than dropping the row: the first `s` reads `p[0]` alone. + A window that holds only absent terms is 0, as a `sum` of them is. `edge='wrap'` reaches back into the end, `within=rel[col]` stops at each group's edge. - `at(p, by=rel[col])` reads nothing for a member the relation maps nowhere, diff --git a/linopy/spec/operators.py b/linopy/spec/operators.py index 6217d28d..20356566 100644 --- a/linopy/spec/operators.py +++ b/linopy/spec/operators.py @@ -367,8 +367,9 @@ def sum_back( """ Sum *array* over a trailing window along *over*: positions ``t - within + 1`` through ``t``. - A position the window cannot reach contributes a zero; a window that - reaches nothing keeps no row. *by* stops the window at each group's edge. + A position the window cannot reach contributes a zero, as an absent term + does, so a window of nothing but such positions is 0, as :func:`sum_over` + is. *by* stops the window at each group's edge. With *wrap* the window continues from the other end of *over*, so a width above the axis size still reads every position and is taken as the whole @@ -386,15 +387,12 @@ def sum_back( ) widest = max(1, min(asked, size)) lagged_terms: list[Array] = [] - reached: list[xr.DataArray] = [] for lag in range(widest): - lagged = _gather(array, over, lag, partition, wrap, None) - live, term = ~lagged.isnull(), _filled(lagged, 0.0) + term = _filled(_gather(array, over, lag, partition, wrap, None), 0.0) if isinstance(within, xr.DataArray): - live, term = live & (within > lag), term * (within > lag).astype(float) + term = term * (within > lag).astype(float) lagged_terms.append(term) - reached.append(live) - return _merged(lagged_terms).where(reduce(operator.or_, reached)) + return _merged(lagged_terms) def _widest(within: Amount) -> int: diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index a699285a..cc12ae3e 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -344,7 +344,6 @@ def absent_row(expression: str) -> dict[str, Any]: pytest.param("x <= a", [1, 2], id="constant-side"), pytest.param("x <= shift(a, along=t, offset=1, edge=0)", [0, 2], id="shift"), pytest.param("x <= at(z, by=grp[s])", [2], id="at"), - pytest.param("sum_back(a * x, along=t, window=2) <= 4", [1, 2], id="sum-back"), ], ) def test_an_absent_row_masks_the_row_it_reaches( diff --git a/test/test_spec_operators.py b/test/test_spec_operators.py index bb9d3f58..1d93339d 100644 --- a/test/test_spec_operators.py +++ b/test/test_spec_operators.py @@ -278,6 +278,33 @@ def test_a_trailing_sum_wider_than_its_axis_wraps_onto_the_whole_axis() -> None: ) +ABSENT_WINDOWS: dict[str, list[float]] = { + "sum-back": [1, 1, 0, 8], + "sum-back-wrap": [9, 1, 0, 8], + "sum-back-in-groups": [1, 1, 0, 8], + "sum-back-group-width": [1, 0, 0, 8], +} + + +@pytest.mark.parametrize("key", ABSENT_WINDOWS) +def test_a_window_of_absent_terms_sums_to_zero_like_sum(key: str) -> None: + live = xr.DataArray([True, False, False, True], coords=[TT]) + x = {**operator_spec()["variables"]["x"], "where": "live"} + spec = with_( + operator_spec(), + parameters={"live": {"dims": ["t"], "dtype": "bool"}}, + variables={"x": x}, + ) + m = solved(spec, {**OPERATOR_DATA, "live": live}) + name = key.replace("-", "_") + want = xr.DataArray(ABSENT_WINDOWS[key], coords=[TT], name=f"probe_{name}") + assert bool((m.constraints[f"link_{name}"].labels != -1).all()) + xr.testing.assert_allclose(m.solution[f"y_{name}"], want.rename(f"y_{name}")) + xr.testing.assert_allclose( + evaluate(spec, m, f"probe_{name}")[f"probe_{name}"], want + ) + + # --------------------------------------------------------------------------- # grouped sum # --------------------------------------------------------------------------- @@ -913,7 +940,7 @@ def test_a_position_no_coordinate_holds_is_refused(predicate: str, match: str) - "sum-by": [3.0, 4.0], "at": [10.0, 20.0, 80.0, np.nan], "shift-wrap-in-groups": [2.0, 1.0, 4.0, np.nan], - "sum-back-in-groups": [1.0, 3.0, 4.0, np.nan], + "sum-back-in-groups": [1.0, 3.0, 4.0, 0.0], } @@ -929,7 +956,8 @@ def test_a_member_a_relation_sends_nowhere_reaches_nothing(key: str) -> None: ) xr.testing.assert_allclose(folded, want.rename(folded.name)) if dims == ["t"]: - assert int(m.constraints[f"link_{name}"].labels.sel(t=3)) == -1 + built = int(m.constraints[f"link_{name}"].labels.sel(t=3)) != -1 + assert built == bool(folded.sel(t=3).notnull()) def test_a_sum_beside_an_empty_dimension_is_the_empty_sum() -> None: @@ -943,7 +971,7 @@ def test_a_sum_beside_an_empty_dimension_is_the_empty_sum() -> None: assert "cap" not in m.constraints -def test_a_window_width_no_member_carries_is_a_window_of_nothing() -> None: +def test_a_window_width_no_member_carries_is_the_empty_sum() -> None: data = { **OPERATOR_DATA, "season_of": pd.DataFrame( @@ -954,7 +982,7 @@ def test_a_window_width_no_member_carries_is_a_window_of_nothing() -> None: folded = evaluate(operator_spec(), m, "probe_sum_back_group_width")[ "probe_sum_back_group_width" ] - assert bool(folded.isnull().all()) + assert bool((folded == 0).all()) def test_a_lookup_into_an_empty_dimension_reads_nothing() -> None: From 1c1b43a7d72e91a71750880433915b14930b2a37 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:16:34 +0000 Subject: [PATCH 2/2] fix(spec): an undefined window keeps no row A window sum over absent terms stays 0, like sum. A member that within= places in no group has no window. A member whose width is absent has no window either. Both keep no row, as on master and as shift does. A width of 0 is a defined, empty window and gives 0. Test the refusal of a row left with no variable term. Test a where that reads sum_back over an absent parameter. --- doc/release-notes.md | 2 +- doc/user-guide/spec-data.md | 2 ++ linopy/spec/operators.py | 10 ++++-- test/test_spec_builder.py | 23 +++++++++++++ test/test_spec_operators.py | 67 ++++++++++++++++++++++++++----------- 5 files changed, 81 insertions(+), 23 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index 8f9d9dac..a3834e62 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -131,7 +131,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) -- `Model.from_spec` builds a `sum_back` window that holds only absent terms as 0, like `sum`, where it dropped the row. ([\#1174](https://github.com/PyPSA/linopy/pull/1174)) +- `Model.from_spec` reads a `sum_back` window that holds only absent terms as 0, like `sum`, in a constraint and in a `where:`, where it was absent; a row left with no variable term is then refused, as for `sum`. ([\#1174](https://github.com/PyPSA/linopy/pull/1174)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) diff --git a/doc/user-guide/spec-data.md b/doc/user-guide/spec-data.md index f8606072..c6f4477b 100644 --- a/doc/user-guide/spec-data.md +++ b/doc/user-guide/spec-data.md @@ -298,6 +298,8 @@ The other operators read past an edge in their own way: - `sum_back(p, along=s, window=3)` is a sum, so it shortens its window at the start rather than dropping the row: the first `s` reads `p[0]` alone. A window that holds only absent terms is 0, as a `sum` of them is. + A member that `within=` places in no group, or one with no width, has + no window, and the row goes with it. `edge='wrap'` reaches back into the end, `within=rel[col]` stops at each group's edge. - `at(p, by=rel[col])` reads nothing for a member the relation maps nowhere, diff --git a/linopy/spec/operators.py b/linopy/spec/operators.py index 20356566..a9a7aac9 100644 --- a/linopy/spec/operators.py +++ b/linopy/spec/operators.py @@ -369,7 +369,9 @@ def sum_back( A position the window cannot reach contributes a zero, as an absent term does, so a window of nothing but such positions is 0, as :func:`sum_over` - is. *by* stops the window at each group's edge. + is. A window that is not defined there, at a member *by* places in no + group or where the width is absent, keeps no row. *by* stops the window + at each group's edge. With *wrap* the window continues from the other end of *over*, so a width above the axis size still reads every position and is taken as the whole @@ -392,7 +394,11 @@ def sum_back( if isinstance(within, xr.DataArray): term = term * (within > lag).astype(float) lagged_terms.append(term) - return _merged(lagged_terms) + defined: list[xr.DataArray] = [] if partition is None else [partition.grouped] + if isinstance(within, xr.DataArray): + defined.append(within.notnull()) + summed = _merged(lagged_terms) + return summed.where(reduce(operator.and_, defined)) if defined else summed def _widest(within: Amount) -> int: diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index cc12ae3e..501d29c3 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -353,6 +353,12 @@ def test_an_absent_row_masks_the_row_it_reaches( assert T[(row.labels != -1).to_numpy()].tolist() == kept +def test_a_row_of_nothing_but_an_absent_window_is_refused_like_sum() -> None: + spec = absent_row("sum_back(a * x, along=t, window=2) <= 4") + with pytest.raises(SpecDataError, match="1 row.*hold no variable term.*at t=0"): + Model.from_spec(spec, ABSENT_DATA) + + def test_an_absent_summand_is_dropped() -> None: spec = with_( ABSENT_SPEC, @@ -777,6 +783,23 @@ def test_a_missing_row_in_a_where_reads_the_way_missing_says( assert (m.variables["x"].labels != -1).values.tolist() == [True, kept] +def test_a_where_reads_a_window_of_absent_rows_as_the_sum_of_the_rest() -> None: + spec = { + "dimensions": {"t": {"dtype": "int", "ordered": True}}, + "parameters": {"p": {"dims": ["t"], "missing": "absent"}}, + "variables": { + "x": { + "dims": ["t"], + "where": "sum_back(p, along=t, window=2) <= 1", + "bounds": {"lower": 0, "upper": 1}, + } + }, + } + p = pd.DataFrame({"t": [1, 2], "p": [1.0, 5.0]}) + m = Model.from_spec(spec, {"t": T, "p": p}) + assert (m.variables["x"].labels != -1).values.tolist() == [True, True, False] + + @pytest.mark.parametrize( ("missing", "rows", "x"), [ diff --git a/test/test_spec_operators.py b/test/test_spec_operators.py index 1d93339d..0c63a501 100644 --- a/test/test_spec_operators.py +++ b/test/test_spec_operators.py @@ -278,28 +278,56 @@ def test_a_trailing_sum_wider_than_its_axis_wraps_onto_the_whole_axis() -> None: ) -ABSENT_WINDOWS: dict[str, list[float]] = { - "sum-back": [1, 1, 0, 8], - "sum-back-wrap": [9, 1, 0, 8], - "sum-back-in-groups": [1, 1, 0, 8], - "sum-back-group-width": [1, 0, 0, 8], -} - - -@pytest.mark.parametrize("key", ABSENT_WINDOWS) -def test_a_window_of_absent_terms_sums_to_zero_like_sum(key: str) -> None: +@pytest.mark.parametrize( + ("key", "data", "expected"), + [ + pytest.param("sum-back", {}, [1, 1, 0, 8], id="sum-back"), + pytest.param("sum-back-wrap", {}, [9, 1, 0, 8], id="sum-back-wrap"), + pytest.param("sum-back-in-groups", {}, [1, 1, 0, 8], id="sum-back-in-groups"), + pytest.param( + "sum-back-group-width", {}, [1, 0, 0, 8], id="sum-back-group-width" + ), + pytest.param( + "sum-back-group-width", + {"width": pd.DataFrame({"s": S, "width": [1, 0]})}, + [1, 0, 0, 0], + id="width-of-zero-is-zero", + ), + pytest.param( + "sum-back-in-groups", + {"season_of": pd.DataFrame({"t": TT[:3], "s": list("aab")})}, + [1, 1, 0, np.nan], + id="member-in-no-group-keeps-no-row", + ), + pytest.param( + "sum-back-group-width", + {"width": pd.DataFrame({"s": S[:1], "width": [1]})}, + [1, 0, np.nan, np.nan], + id="absent-width-keeps-no-row", + ), + ], +) +def test_a_defined_window_of_absent_terms_sums_to_zero_like_sum( + key: str, data: dict[str, Any], expected: list[float] +) -> None: live = xr.DataArray([True, False, False, True], coords=[TT]) x = {**operator_spec()["variables"]["x"], "where": "live"} spec = with_( operator_spec(), - parameters={"live": {"dims": ["t"], "dtype": "bool"}}, + parameters={ + "live": {"dims": ["t"], "dtype": "bool"}, + "width": {"dims": ["s"], "dtype": "int", "missing": "absent"}, + }, variables={"x": x}, ) - m = solved(spec, {**OPERATOR_DATA, "live": live}) + m = solved(spec, {**OPERATOR_DATA, "live": live, **data}) name = key.replace("-", "_") - want = xr.DataArray(ABSENT_WINDOWS[key], coords=[TT], name=f"probe_{name}") - assert bool((m.constraints[f"link_{name}"].labels != -1).all()) - xr.testing.assert_allclose(m.solution[f"y_{name}"], want.rename(f"y_{name}")) + want = xr.DataArray(expected, coords=[TT], name=f"probe_{name}") + built = m.constraints[f"link_{name}"].labels != -1 + assert built.values.tolist() == want.notnull().values.tolist() + xr.testing.assert_allclose( + m.solution[f"y_{name}"].where(built), want.rename(f"y_{name}") + ) xr.testing.assert_allclose( evaluate(spec, m, f"probe_{name}")[f"probe_{name}"], want ) @@ -940,7 +968,7 @@ def test_a_position_no_coordinate_holds_is_refused(predicate: str, match: str) - "sum-by": [3.0, 4.0], "at": [10.0, 20.0, 80.0, np.nan], "shift-wrap-in-groups": [2.0, 1.0, 4.0, np.nan], - "sum-back-in-groups": [1.0, 3.0, 4.0, 0.0], + "sum-back-in-groups": [1.0, 3.0, 4.0, np.nan], } @@ -956,8 +984,7 @@ def test_a_member_a_relation_sends_nowhere_reaches_nothing(key: str) -> None: ) xr.testing.assert_allclose(folded, want.rename(folded.name)) if dims == ["t"]: - built = int(m.constraints[f"link_{name}"].labels.sel(t=3)) != -1 - assert built == bool(folded.sel(t=3).notnull()) + assert int(m.constraints[f"link_{name}"].labels.sel(t=3)) == -1 def test_a_sum_beside_an_empty_dimension_is_the_empty_sum() -> None: @@ -971,7 +998,7 @@ def test_a_sum_beside_an_empty_dimension_is_the_empty_sum() -> None: assert "cap" not in m.constraints -def test_a_window_width_no_member_carries_is_the_empty_sum() -> None: +def test_a_window_width_no_member_carries_is_a_window_of_nothing() -> None: data = { **OPERATOR_DATA, "season_of": pd.DataFrame( @@ -982,7 +1009,7 @@ def test_a_window_width_no_member_carries_is_the_empty_sum() -> None: folded = evaluate(operator_spec(), m, "probe_sum_back_group_width")[ "probe_sum_back_group_width" ] - assert bool((folded == 0).all()) + assert bool(folded.isnull().all()) def test_a_lookup_into_an_empty_dimension_reads_nothing() -> None: