From ba8d7a6a006bd3d3234521fe3fed7d2411d1b13f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 07:11:29 +0000 Subject: [PATCH 1/7] fix(spec): drop a row emptied by a stated edge value when it holds A row that the data empties of every variable term was dropped with a "trivially true" warning only when every variable declared a missing: number. A row emptied by a shift's stated edge, e.g. sum(shift(x, along=t, offset=1, edge=0), over=k) == load at the first t, was refused even when it read 0 == 0. Count such a row as valued when the spec states every value it can be left with: variables without where: (or with a missing: number), shifts with an edge= number, and data that is never absent. A valued row whose sides differ is now refused as one that cannot hold, naming the first comparison (e.g. 0 == 5). A row emptied by an absent term, such as a shift without edge=, a window, a lookup or a sum over no member, is still refused as before. Fixes #1147 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01F4nx8L63c3o8u6QQvr6TGN --- doc/release-notes.md | 1 + linopy/spec/builder.py | 86 +++++++++++++++++++++++++++++++++++---- test/test_spec_builder.py | 45 ++++++++++++++++++++ 3 files changed, 123 insertions(+), 9 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index c9836a1d..b2ad97f5 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -130,6 +130,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) +- `Model.from_spec` drops a constraint row that a stated edge value empties of every variable term, e.g. `sum(shift(x, along=t, offset=1, edge=0), over=k) == load` at the first `t`, with a warning when it holds (`0 == 0`), and refuses one that cannot hold (`0 == 5`) as such. A row emptied by an absent term, such as a shift without `edge=`, is still refused. ([\#1147](https://github.com/PyPSA/linopy/issues/1147)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) diff --git a/linopy/spec/builder.py b/linopy/spec/builder.py index fb37c3c5..015adafd 100644 --- a/linopy/spec/builder.py +++ b/linopy/spec/builder.py @@ -203,9 +203,11 @@ def _check_live( Such a row reads as ``0 sense rhs``: linopy carries no column there, the row leaves the problem and the constraint silently stops binding. Where - every variable of the row declares a ``missing:`` number, the constant - left is what the math says, so a row whose two sides are equal there is - warned about rather than refused. + every variable of the row declares a ``missing:`` number, or the spec + states every value the row can be left with (see :func:`_stated`), the + constant left is what the math says: a row whose two sides are equal there + is warned about, and one whose sides differ is refused as one that cannot + hold. An absent row is not emptied: absence spread through arithmetic, a masked variable or a vacated translation, takes the row with it. A bare variable term is never emptied: a row of it exists exactly where the variable does. @@ -222,18 +224,21 @@ def _check_live( f"is attached, the first at {first_coordinates(dead, 3)}. Nothing is left to constrain there, so " f"the row leaves the problem without saying so." ) + sides = (declared.lhs, declared.rhs) valued = all( - ctx.program.variables[v].missing != "absent" - for v in ms.variables_of(declared.lhs, declared.rhs) - ) - if valued and not _binds(other, constant_of(term), dead): + ctx.program.variables[v].missing != "absent" for v in ms.variables_of(*sides) + ) or (not isinstance(other, Term) and all(_stated(side, ctx) for side in sides)) + const = constant_of(term) + if valued and not _binds(other, const, dead): warnings.warn( - f"{said} Every variable there takes its missing: number and the two sides " - f"are equal, so the row is trivially true.", + f"{said} Every term there takes the value the spec states for it and the " + f"two sides are equal, so the row is trivially true.", UserWarning, stacklevel=2, ) return + if valued: + said = _cannot_hold(name, declared, const, other, dead) supply = ( " Supply the rows of the variables, if the row is meant to bind." if valued @@ -252,6 +257,69 @@ def _binds(other: Value, const: xr.DataArray, dead: xr.DataArray) -> bool: return bool((differs & dead).any()) +def _cannot_hold( + name: str, + declared: ms.Constraint, + const: xr.DataArray, + other: Value, + dead: xr.DataArray, +) -> str: + """The refusal of the rows *dead* names whose constant *const* differs from *other*, the first one's comparison spelled out.""" + failing = dead & (xr.DataArray(other) != const) + first = dict( + zip( + failing.dims, + np.unravel_index(int(np.argmax(failing.values)), failing.shape), + ) + ) + left, right = ( + float(xr.DataArray(side).broadcast_like(failing).isel(first)) + for side in (const, other) + ) + sense = _FLIPPED[declared.sense] if flipped(declared) else declared.sense + return ( + f"constraint '{name}': {int(failing.sum())} row(s) hold no variable term once the " + f"data is attached and cannot hold, the first at {first_coordinates(failing, 3)}, " + f"where it reads {left:g} {sense} {right:g}. Every term there takes the value " + f"the spec states for it, so the two sides are constants that differ." + ) + + +def _stated(node: ms.Expression, ctx: Context) -> bool: + """ + Whether the spec states every value *node* can leave a row with. + + A term under it is then present at every row or stands as a number the + spec gives: a variable without a ``where:``, or with a ``missing:`` number, + shifted only with an ``edge=`` number, times data that is never absent. + Anything that can leave a term absent is not stated, so a row emptied by + it is still refused: a shift without ``edge=``, a window, a lookup, a + partition, data that can be absent and a sum over no member. + """ + summed = node.operand.dims if isinstance(node, ms.Sum) else () + if is_empty((*node.dims, *summed), ctx): + return False + if isinstance(node, ms.VariableReference): + variable = ctx.program.variables[node.name] + return variable.where is None or variable.missing != "absent" + if isinstance(node, ms.ParameterReference): + return _never_absent(node.name, ctx) + if isinstance(node, ms.Translate): + if node.fill is None or node.partition is not None: + return False + if isinstance(node.offset, str) and not _never_absent(node.offset, ctx): + return False + elif isinstance(node, ms.WindowSum | ms.Join | ms.Dual): + return False + return all(_stated(child, ctx) for child in ms.children(node)) + + +def _never_absent(parameter: str, ctx: Context) -> bool: + """Whether *parameter* has a value at every coordinate: one ``missing:`` does not leave absent, under no ``where:``.""" + declared = ctx.program.parameters[parameter] + return declared.missing != "absent" and declared.where is None + + def _objective(ctx: Context) -> None: declared = ctx.program.objective if declared is None: diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index c1eb74d1..f15b29e5 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -754,6 +754,51 @@ def test_a_dimension_with_no_members_empties_a_summed_row(row: str) -> None: Model.from_spec(spec, {"t": empty, "w": none, "c": none}) +def delayed(edge: str) -> dict[str, Any]: + """A balance over ``t`` that a delay of one empties of every variable term at ``t=0``.""" + return { + "dimensions": {"t": {"dtype": "int", "ordered": True}, "k": {"dtype": "str"}}, + "parameters": {"load": {"dims": ["t"]}}, + "variables": {"x": {"dims": ["t", "k"], "bounds": {"lower": 0}}}, + "constraints": { + "balance": { + "dims": ["t"], + "expression": f"sum(shift(x, along=t, offset=1{edge}), over=k) == load", + } + }, + "objective": {"sense": "minimize", "expression": "sum(x, over=[t, k])"}, + } + + +def delayed_data(first: float) -> dict[str, Any]: + return {"t": T, "k": ["a"], "load": xr.DataArray([first, 10.0, 10.0], coords=[T])} + + +def test_a_row_emptied_by_a_stated_edge_that_holds_is_only_warned_about() -> None: + with pytest.warns(UserWarning, match=r"constraint 'balance'.*trivially true"): + m = solved(delayed(", edge=0"), delayed_data(0.0)) + assert m.termination_condition == "optimal" + assert (m.constraints["balance"].vars.sel(t=0) == -1).all() + assert float(m.solution["x"].sel(t=1).sum()) == pytest.approx(10.0) + + +def test_a_row_emptied_by_a_stated_edge_that_cannot_hold_is_refused() -> None: + with pytest.raises( + SpecDataError, + match=r"constraint 'balance'.*cannot hold, the first at t=0, where it reads 0 == 5", + ): + Model.from_spec(delayed(", edge=0"), delayed_data(5.0)) + + +@pytest.mark.parametrize("first", [0.0, 5.0]) +def test_a_row_emptied_by_a_vacated_shift_is_refused(first: float) -> None: + """Without ``edge=`` the vacated term is absent, not a stated zero.""" + with pytest.raises( + SpecDataError, match=r"(?s)constraint 'balance'.*Declare missing: 0" + ): + Model.from_spec(delayed(""), delayed_data(first)) + + @pytest.mark.parametrize(("missing", "kept"), [("absent", False), (0, True)]) def test_a_missing_row_in_a_where_reads_the_way_missing_says( missing: Any, kept: bool From 0d1744ec048f74984602217067d60e07502f4ecb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 07:23:20 +0000 Subject: [PATCH 2/7] test(spec): cover the exclusions of a stated row value A shift whose offset parameter can be absent, and a window, keep an emptied row refused even under edge=0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01F4nx8L63c3o8u6QQvr6TGN --- test/test_spec_builder.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index f15b29e5..14ffbaef 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -799,6 +799,33 @@ def test_a_row_emptied_by_a_vacated_shift_is_refused(first: float) -> None: Model.from_spec(delayed(""), delayed_data(first)) +@pytest.mark.parametrize( + "balance", + [ + pytest.param( + "sum(shift(x, along=t, offset=lag, edge=0), over=k) == load", + id="offset-may-be-absent", + ), + pytest.param( + "sum(sum_back(shift(x, along=t, offset=1, edge=0), along=t, window=1), " + "over=k) == load", + id="window", + ), + ], +) +def test_a_row_emptied_where_a_term_may_be_absent_is_still_refused( + balance: str, +) -> None: + """An ``edge=0`` under an offset that can be absent, or a window, is no stated value.""" + spec = delayed(", edge=0") + spec["parameters"]["lag"] = {"dims": [], "dtype": "int", "missing": "absent"} + spec["constraints"]["balance"]["expression"] = balance + with pytest.raises( + SpecDataError, match=r"(?s)constraint 'balance'.*Declare missing: 0" + ): + Model.from_spec(spec, {**delayed_data(0.0), "lag": 1}) + + @pytest.mark.parametrize(("missing", "kept"), [("absent", False), (0, True)]) def test_a_missing_row_in_a_where_reads_the_way_missing_says( missing: Any, kept: bool From 278f5ba2a5046d56a4e952f177ffe8f88d73f2e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:14:00 +0000 Subject: [PATCH 3/7] Revert "test(spec): cover the exclusions of a stated row value" This reverts commit 0d1744ec048f74984602217067d60e07502f4ecb. --- test/test_spec_builder.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index b5e87a9c..532797a4 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -801,33 +801,6 @@ def test_a_row_emptied_by_a_vacated_shift_is_refused(first: float) -> None: Model.from_spec(delayed(""), delayed_data(first)) -@pytest.mark.parametrize( - "balance", - [ - pytest.param( - "sum(shift(x, along=t, offset=lag, edge=0), over=k) == load", - id="offset-may-be-absent", - ), - pytest.param( - "sum(sum_back(shift(x, along=t, offset=1, edge=0), along=t, window=1), " - "over=k) == load", - id="window", - ), - ], -) -def test_a_row_emptied_where_a_term_may_be_absent_is_still_refused( - balance: str, -) -> None: - """An ``edge=0`` under an offset that can be absent, or a window, is no stated value.""" - spec = delayed(", edge=0") - spec["parameters"]["lag"] = {"dims": [], "dtype": "int", "missing": "absent"} - spec["constraints"]["balance"]["expression"] = balance - with pytest.raises( - SpecDataError, match=r"(?s)constraint 'balance'.*Declare missing: 0" - ): - Model.from_spec(spec, {**delayed_data(0.0), "lag": 1}) - - @pytest.mark.parametrize(("missing", "kept"), [("absent", False), (0, True)]) def test_a_missing_row_in_a_where_reads_the_way_missing_says( missing: Any, kept: bool From 0ad32838c1812c8c4c52a921cfe665513f40e762 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:14:00 +0000 Subject: [PATCH 4/7] Revert "fix(spec): drop a row emptied by a stated edge value when it holds" This reverts commit ba8d7a6a006bd3d3234521fe3fed7d2411d1b13f. --- doc/release-notes.md | 1 - linopy/spec/builder.py | 86 ++++----------------------------------- test/test_spec_builder.py | 45 -------------------- 3 files changed, 9 insertions(+), 123 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index fa9792c3..a1c3ff1e 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -131,7 +131,6 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) -- `Model.from_spec` drops a constraint row that a stated edge value empties of every variable term, e.g. `sum(shift(x, along=t, offset=1, edge=0), over=k) == load` at the first `t`, with a warning when it holds (`0 == 0`), and refuses one that cannot hold (`0 == 5`) as such. A row emptied by an absent term, such as a shift without `edge=`, is still refused. ([\#1147](https://github.com/PyPSA/linopy/issues/1147)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) diff --git a/linopy/spec/builder.py b/linopy/spec/builder.py index 015adafd..fb37c3c5 100644 --- a/linopy/spec/builder.py +++ b/linopy/spec/builder.py @@ -203,11 +203,9 @@ def _check_live( Such a row reads as ``0 sense rhs``: linopy carries no column there, the row leaves the problem and the constraint silently stops binding. Where - every variable of the row declares a ``missing:`` number, or the spec - states every value the row can be left with (see :func:`_stated`), the - constant left is what the math says: a row whose two sides are equal there - is warned about, and one whose sides differ is refused as one that cannot - hold. + every variable of the row declares a ``missing:`` number, the constant + left is what the math says, so a row whose two sides are equal there is + warned about rather than refused. An absent row is not emptied: absence spread through arithmetic, a masked variable or a vacated translation, takes the row with it. A bare variable term is never emptied: a row of it exists exactly where the variable does. @@ -224,21 +222,18 @@ def _check_live( f"is attached, the first at {first_coordinates(dead, 3)}. Nothing is left to constrain there, so " f"the row leaves the problem without saying so." ) - sides = (declared.lhs, declared.rhs) valued = all( - ctx.program.variables[v].missing != "absent" for v in ms.variables_of(*sides) - ) or (not isinstance(other, Term) and all(_stated(side, ctx) for side in sides)) - const = constant_of(term) - if valued and not _binds(other, const, dead): + ctx.program.variables[v].missing != "absent" + for v in ms.variables_of(declared.lhs, declared.rhs) + ) + if valued and not _binds(other, constant_of(term), dead): warnings.warn( - f"{said} Every term there takes the value the spec states for it and the " - f"two sides are equal, so the row is trivially true.", + f"{said} Every variable there takes its missing: number and the two sides " + f"are equal, so the row is trivially true.", UserWarning, stacklevel=2, ) return - if valued: - said = _cannot_hold(name, declared, const, other, dead) supply = ( " Supply the rows of the variables, if the row is meant to bind." if valued @@ -257,69 +252,6 @@ def _binds(other: Value, const: xr.DataArray, dead: xr.DataArray) -> bool: return bool((differs & dead).any()) -def _cannot_hold( - name: str, - declared: ms.Constraint, - const: xr.DataArray, - other: Value, - dead: xr.DataArray, -) -> str: - """The refusal of the rows *dead* names whose constant *const* differs from *other*, the first one's comparison spelled out.""" - failing = dead & (xr.DataArray(other) != const) - first = dict( - zip( - failing.dims, - np.unravel_index(int(np.argmax(failing.values)), failing.shape), - ) - ) - left, right = ( - float(xr.DataArray(side).broadcast_like(failing).isel(first)) - for side in (const, other) - ) - sense = _FLIPPED[declared.sense] if flipped(declared) else declared.sense - return ( - f"constraint '{name}': {int(failing.sum())} row(s) hold no variable term once the " - f"data is attached and cannot hold, the first at {first_coordinates(failing, 3)}, " - f"where it reads {left:g} {sense} {right:g}. Every term there takes the value " - f"the spec states for it, so the two sides are constants that differ." - ) - - -def _stated(node: ms.Expression, ctx: Context) -> bool: - """ - Whether the spec states every value *node* can leave a row with. - - A term under it is then present at every row or stands as a number the - spec gives: a variable without a ``where:``, or with a ``missing:`` number, - shifted only with an ``edge=`` number, times data that is never absent. - Anything that can leave a term absent is not stated, so a row emptied by - it is still refused: a shift without ``edge=``, a window, a lookup, a - partition, data that can be absent and a sum over no member. - """ - summed = node.operand.dims if isinstance(node, ms.Sum) else () - if is_empty((*node.dims, *summed), ctx): - return False - if isinstance(node, ms.VariableReference): - variable = ctx.program.variables[node.name] - return variable.where is None or variable.missing != "absent" - if isinstance(node, ms.ParameterReference): - return _never_absent(node.name, ctx) - if isinstance(node, ms.Translate): - if node.fill is None or node.partition is not None: - return False - if isinstance(node.offset, str) and not _never_absent(node.offset, ctx): - return False - elif isinstance(node, ms.WindowSum | ms.Join | ms.Dual): - return False - return all(_stated(child, ctx) for child in ms.children(node)) - - -def _never_absent(parameter: str, ctx: Context) -> bool: - """Whether *parameter* has a value at every coordinate: one ``missing:`` does not leave absent, under no ``where:``.""" - declared = ctx.program.parameters[parameter] - return declared.missing != "absent" and declared.where is None - - def _objective(ctx: Context) -> None: declared = ctx.program.objective if declared is None: diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index 532797a4..a699285a 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -756,51 +756,6 @@ def test_a_dimension_with_no_members_empties_a_summed_row(row: str) -> None: Model.from_spec(spec, {"t": empty, "w": none, "c": none}) -def delayed(edge: str) -> dict[str, Any]: - """A balance over ``t`` that a delay of one empties of every variable term at ``t=0``.""" - return { - "dimensions": {"t": {"dtype": "int", "ordered": True}, "k": {"dtype": "str"}}, - "parameters": {"load": {"dims": ["t"]}}, - "variables": {"x": {"dims": ["t", "k"], "bounds": {"lower": 0}}}, - "constraints": { - "balance": { - "dims": ["t"], - "expression": f"sum(shift(x, along=t, offset=1{edge}), over=k) == load", - } - }, - "objective": {"sense": "minimize", "expression": "sum(x, over=[t, k])"}, - } - - -def delayed_data(first: float) -> dict[str, Any]: - return {"t": T, "k": ["a"], "load": xr.DataArray([first, 10.0, 10.0], coords=[T])} - - -def test_a_row_emptied_by_a_stated_edge_that_holds_is_only_warned_about() -> None: - with pytest.warns(UserWarning, match=r"constraint 'balance'.*trivially true"): - m = solved(delayed(", edge=0"), delayed_data(0.0)) - assert m.termination_condition == "optimal" - assert (m.constraints["balance"].vars.sel(t=0) == -1).all() - assert float(m.solution["x"].sel(t=1).sum()) == pytest.approx(10.0) - - -def test_a_row_emptied_by_a_stated_edge_that_cannot_hold_is_refused() -> None: - with pytest.raises( - SpecDataError, - match=r"constraint 'balance'.*cannot hold, the first at t=0, where it reads 0 == 5", - ): - Model.from_spec(delayed(", edge=0"), delayed_data(5.0)) - - -@pytest.mark.parametrize("first", [0.0, 5.0]) -def test_a_row_emptied_by_a_vacated_shift_is_refused(first: float) -> None: - """Without ``edge=`` the vacated term is absent, not a stated zero.""" - with pytest.raises( - SpecDataError, match=r"(?s)constraint 'balance'.*Declare missing: 0" - ): - Model.from_spec(delayed(""), delayed_data(first)) - - @pytest.mark.parametrize(("missing", "kept"), [("absent", False), (0, True)]) def test_a_missing_row_in_a_where_reads_the_way_missing_says( missing: Any, kept: bool From eb1ab2f46656ab350069791107c3bebd24eba432 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:24:00 +0000 Subject: [PATCH 5/7] fix(spec): check a row the data empties against its constants Port of 1ea671d from spec/prune-unbuilt onto master. A sum of absent terms counts as 0. A row the data leaves without a variable term reads constant sense constant. It is dropped with linopy.EmptiedRowWarning where it holds and refused, quoting the row, where it cannot. A term the data scales by zero is pruned, so 0 * x >= 5 is refused instead of built. The objective is evaluated again without that pruning when it would lose every variable. 1ea671d's _variable hunk is left out: it made an objective over a fully masked variable unsolvable. The product of two terms converts one constant side only, as in 7ea5b20. --- linopy/__init__.py | 2 + linopy/constants.py | 14 ++++ linopy/spec/builder.py | 154 +++++++++++++++++++++++++------------- linopy/spec/context.py | 3 + linopy/spec/evaluator.py | 40 +++++++++- test/test_spec_builder.py | 133 ++++++++++++++++++++++++++++---- 6 files changed, 276 insertions(+), 70 deletions(-) diff --git a/linopy/__init__.py b/linopy/__init__.py index 9a0ff6e7..9be4bb11 100644 --- a/linopy/__init__.py +++ b/linopy/__init__.py @@ -19,6 +19,7 @@ GREATER_EQUAL, LESS_EQUAL, AbsorbedTermsWarning, + EmptiedRowWarning, EvolvingAPIWarning, PerformanceWarning, ) @@ -45,6 +46,7 @@ "Constraints", "Constraint", "EQUAL", + "EmptiedRowWarning", "PerformanceWarning", "EvolvingAPIWarning", "GREATER_EQUAL", diff --git a/linopy/constants.py b/linopy/constants.py index 4a1c9758..26b58388 100644 --- a/linopy/constants.py +++ b/linopy/constants.py @@ -108,6 +108,20 @@ class AbsorbedTermsWarning(UserWarning): SOS_BIG_M_ATTR = "big_m_upper" +class EmptiedRowWarning(UserWarning): + """ + Signals constraint rows that ``Model.from_spec`` dropped because the data + left them without a variable term and they hold, e.g. ``0 <= 5``. + + Filter it with:: + + import warnings + import linopy + + warnings.filterwarnings("ignore", category=linopy.EmptiedRowWarning) + """ + + class EvolvingAPIWarning(FutureWarning): """ Signals a newly-added API whose details may evolve in minor releases. diff --git a/linopy/spec/builder.py b/linopy/spec/builder.py index fb37c3c5..3ba1c392 100644 --- a/linopy/spec/builder.py +++ b/linopy/spec/builder.py @@ -19,7 +19,7 @@ import xarray as xr from mathspec import program as ms -from linopy.constants import FACTOR_DIM, TERM_DIM +from linopy.constants import FACTOR_DIM, TERM_DIM, EmptiedRowWarning from linopy.expressions import LinearExpression, QuadraticExpression from linopy.spec.context import Context, Term, Value, as_expression, is_empty from linopy.spec.errors import SpecDataError, first_coordinates @@ -36,6 +36,7 @@ from linopy.variables import Variable _FLIPPED: dict[str, ms.ConstraintSense] = {"==": "==", "<=": ">=", ">=": "<="} +_COMPARE = {"==": operator.eq, "<=": operator.le, ">=": operator.ge} _SENSE = {"minimize": "min", "maximize": "max"} @@ -139,7 +140,6 @@ def _constraints(ctx: Context) -> None: ctx.memo.skipped(row.lhs, row.rhs) continue rows = evaluate_where(row.where, ctx) - mask = as_linopy_mask(rows) scoped = replace(ctx, rows=rows, subject=f"constraint '{name}'") refuse_holes(scoped.subject, [row.lhs, row.rhs], ctx, rows) lhs, rhs, sense = row.lhs, row.rhs, row.sense @@ -151,9 +151,11 @@ def _constraints(ctx: Context) -> None: term = cast(Term, term) if any(term.sizes[d] == 0 for d in term.coord_dims): continue - _check_live(name, row, term, other, rows, ctx) - sides = [term, other] if isinstance(other, Term) else [term] - if all(as_expression(side).is_constant for side in sides): + emptied = _emptied(term, other, rows) + if bool(emptied.any()): + _check_emptied(name, row, term, other, emptied) + rows = rows & ~emptied + if _constant(term, other): continue if isinstance(other, xr.DataArray): const = constant_of(term) @@ -162,10 +164,21 @@ def _constraints(ctx: Context) -> None: ) term, other = carried(term, other) ctx.model.add_constraints( - _framed(term, row.dims), sense, other, name=ctx.rename(name), mask=mask + _framed(term, row.dims), + sense, + other, + name=ctx.rename(name), + mask=as_linopy_mask(rows), ) +def _constant(*sides: Value) -> bool: + """Whether none of *sides*, the two of a comparison or an objective, holds a variable term.""" + return all( + not isinstance(side, Term) or as_expression(side).is_constant for side in sides + ) + + def _linear(name: str, side: Value, rows: xr.DataArray) -> Value: """ One *side* of a constraint as a linear expression, refused where the data leaves @@ -190,66 +203,84 @@ def _linear(name: str, side: Value, rows: xr.DataArray) -> Value: return LinearExpression(side.data.assign(vars=vars), side.model) -def _check_live( +def _emptied(term: Term, other: Value, rows: xr.DataArray) -> xr.DataArray: + """ + The rows *rows* keeps where neither side holds a variable term and neither is absent. + + An absent row is not emptied: absence spread through arithmetic, a masked + variable or a vacated translation, takes the row with it. A row of a bare + variable exists exactly where the variable does, so it is never emptied. + """ + held = _held(term) | _held(other) + absent = _constant_of(term).isnull() | _constant_of(other).isnull() + return rows & ~held & ~absent + + +def _held(side: Value) -> xr.DataArray: + """Per row, whether *side* holds a variable term.""" + if isinstance(side, Variable): + return side.labels != -1 + if not isinstance(side, LinearExpression): + return xr.DataArray(False) + store = side._store + return store.dataarray(store.has_terms(), "held") + + +def _constant_of(side: Value) -> xr.DataArray: + """The constant of *side*, NaN where it is absent.""" + if isinstance(side, Variable): + return xr.where(side.labels != -1, 0.0, np.nan) + return constant_of(side) + + +def _check_emptied( name: str, declared: ms.Constraint, term: Term, other: Value, - rows: xr.DataArray, - ctx: Context, + emptied: xr.DataArray, ) -> None: """ - Refuse a row the data emptied of every variable term. + Drop with a warning the rows *emptied* names where they hold, refuse them where they cannot. - Such a row reads as ``0 sense rhs``: linopy carries no column there, the - row leaves the problem and the constraint silently stops binding. Where - every variable of the row declares a ``missing:`` number, the constant - left is what the math says, so a row whose two sides are equal there is - warned about rather than refused. - An absent row is not emptied: absence spread through arithmetic, a masked - variable or a vacated translation, takes the row with it. A bare variable - term is never emptied: a row of it exists exactly where the variable does. + The data left such a row without a variable term, so it reads as + ``constant sense constant``, compared exactly as written. A row that + holds constrains nothing; one that cannot hold makes the problem + infeasible, and linopy, which carries no row without a term, would drop + it silently. """ - if isinstance(term, Variable): - return - store = cast(LinearExpression, term)._store - alive = store.dataarray(store.has_terms() | np.isnan(store.const), "alive") - dead = rows & ~alive - if not bool(dead.any()): - return - said = ( - f"constraint '{name}': {int(dead.sum())} row(s) hold no variable term once the data " - f"is attached, the first at {first_coordinates(dead, 3)}. Nothing is left to constrain there, so " - f"the row leaves the problem without saying so." - ) - valued = all( - ctx.program.variables[v].missing != "absent" - for v in ms.variables_of(declared.lhs, declared.rhs) - ) - if valued and not _binds(other, constant_of(term), dead): - warnings.warn( - f"{said} Every variable there takes its missing: number and the two sides " - f"are equal, so the row is trivially true.", - UserWarning, - stacklevel=2, + lhs, rhs = _constant_of(term), _constant_of(other) + if flipped(declared): + lhs, rhs = rhs, lhs + holds = _COMPARE[declared.sense](lhs, rhs) + failing = emptied & ~holds + said = f"constraint '{name}': {{}} row(s) hold no variable term once the data is attached" + if bool(failing.any()): + raise SpecDataError( + f"{said.format(int(failing.sum()))} and cannot hold, the first at " + f"{_quoted(failing, lhs, rhs, declared.sense)}.\n" + f" Mask them out with a where on the constraint, if the row should not exist there.\n" + f" Supply the rows of the variables, if the row is meant to bind." ) - return - supply = ( - " Supply the rows of the variables, if the row is meant to bind." - if valued - else " Declare missing: 0 on the variables, if an absent term is a zero there." - ) - raise SpecDataError( - f"{said}\n" - f" Mask them out with a where on the constraint, if the row should not exist there.\n" - f"{supply}" + warnings.warn( + f"{said.format(int(emptied.sum()))}, the first at " + f"{_quoted(emptied, lhs, rhs, declared.sense)}. Each holds, so it is dropped.", + EmptiedRowWarning, + stacklevel=2, ) -def _binds(other: Value, const: xr.DataArray, dead: xr.DataArray) -> bool: - """Whether the side without the variable term differs from the constant *const* left on a row *dead* names.""" - differs = xr.DataArray(other) != const - return bool((differs & dead).any()) +def _quoted( + flags: xr.DataArray, lhs: xr.DataArray, rhs: xr.DataArray, sense: str +) -> str: + """The first row *flags* names, with the constants its two sides reduce to.""" + first = np.unravel_index(int(np.argmax(flags.values)), flags.shape) + at = dict(zip(flags.dims, first)) + left, right = ( + float(side.broadcast_like(flags).transpose(*flags.dims)[at]) + for side in (lhs, rhs) + ) + return f"{first_coordinates(flags, 1)}: {left!r} {sense} {right!r}" def _objective(ctx: Context) -> None: @@ -258,6 +289,9 @@ def _objective(ctx: Context) -> None: return refuse_holes("the objective", [declared.expression], ctx) expr = evaluate(declared.expression, ctx) + if _constant(expr): + expr = evaluate(declared.expression, _terms_dropped(ctx, zeroes=False)) + _terms_dropped(ctx) if not isinstance(expr, Term): raise SpecDataError( "the objective carries no variable term once the data is attached, so there is nothing to optimize" @@ -267,6 +301,18 @@ def _objective(ctx: Context) -> None: ) +def _terms_dropped(ctx: Context, zeroes: bool = True) -> Context: + """*ctx* with the terms its memo keeps dropped, a term the data scales by zero pruned as *zeroes* says.""" + values = ctx.memo.values + for key in [ + k + for k, v in values.items() + if isinstance(v, LinearExpression | QuadraticExpression) + ]: + del values[key] + return replace(ctx, zeroes=zeroes) + + def _linear_if_no_product(expr: Term) -> Term: """ A quadratic *expr* whose products the data zeroes, as the linear expression it is. diff --git a/linopy/spec/context.py b/linopy/spec/context.py index 12a0ac34..31714f5c 100644 --- a/linopy/spec/context.py +++ b/linopy/spec/context.py @@ -138,6 +138,8 @@ class Context: ``subject`` names what is being built in its refusals. ``memo`` is shared by every context replaced from one. ``rename`` gives the model name of a variable or constraint the program declares. + ``zeroes`` makes a term the data scales by zero the constant it is + there, so a row left with such terms alone holds no variable term. """ model: Model @@ -150,6 +152,7 @@ class Context: subject: str = "the spec" memo: Memo = field(default_factory=Memo, compare=False, repr=False) rename: Callable[[str], str] = field(default=str, compare=False, repr=False) + zeroes: bool = True def is_empty(dims: Iterable[str], ctx: Context) -> bool: diff --git a/linopy/spec/evaluator.py b/linopy/spec/evaluator.py index a51b2ca3..d6aef6d3 100644 --- a/linopy/spec/evaluator.py +++ b/linopy/spec/evaluator.py @@ -228,13 +228,26 @@ def _combine(node: _Binary, ctx: Context) -> Value: op, second = operator.sub, second.operand left, right = evaluate(first, ctx), evaluate(second, ctx) kind = type(node).__name__.lower() + if ( + isinstance(node, ms.Multiply) + and isinstance(left, Term) + and isinstance(right, Term) + ): + if _is_constant(right): + right = cast(LinearExpression, right).const + elif _is_constant(left): + left = cast(LinearExpression, left).const if isinstance(left, Term) or isinstance(right, Term): left, right = _off_rows(left, ctx), _off_rows(right, ctx) if isinstance(node, ms.Add): a, b = constant_of(left), constant_of(right) refuse_made(kind, (node,), op, a, b, ctx) made = op(*_paired(left, right)) - return _without_dead_terms(made) if isinstance(node, ms.Multiply) else made + if not isinstance(node, ms.Multiply): + return made + if ctx.zeroes: + made = _without_zeroed_terms(made, left, right) + return _without_dead_terms(made) if not isinstance(left, xr.DataArray) and not isinstance(right, xr.DataArray): return op(left, right) a, b = xr.DataArray(left), xr.DataArray(right) @@ -248,6 +261,31 @@ def _combine(node: _Binary, ctx: Context) -> Value: return refuse_made(kind, (node,), op, a, b, ctx) +def _is_constant(value: Value) -> bool: + """ + Whether *value* is a term without a variable term: a product of it and + another term scales that term by its constant, as data, its absence a + hole, and stays a term. + """ + return isinstance(value, LinearExpression) and value.is_constant + + +def _without_zeroed_terms(made: Value, left: Value, right: Value) -> Value: + """ + A linear *made*, the product of a term and data, without the terms the data zeroes. + + Where the data is zero the product is the term's constant times zero: + a constant, and a row left with nothing but such terms holds no + variable term. + """ + if not isinstance(made, LinearExpression) or type(made) is not LinearExpression: + return made + data = right if isinstance(left, Term) else left + if isinstance(data, Term) or not bool(np.any(np.asarray(data) == 0)): + return made + return LinearExpression._from_store(made._store.prune(), made.model) + + def _without_dead_terms(made: Value) -> Value: """ A quadratic *made* whose terms the data zeroes throughout, as its constant. diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index a699285a..80466ad6 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -7,6 +7,7 @@ from __future__ import annotations import glob +import re import warnings from pathlib import Path from typing import Any @@ -558,12 +559,22 @@ def test_an_edge_value_stands_in_for_the_vacated_row() -> None: assert float(first.rhs) == 0.0 -def test_a_row_the_data_emptied_of_variables_is_refused() -> None: - """A zero absence leaves ``0 <= relmax`` standing, which linopy cannot carry and a solver would never see.""" +@pytest.mark.parametrize( + ("missing", "expression", "row"), + [ + pytest.param(0, "size >= relmax", "0.0 >= 0.5", id="zero"), + pytest.param(1, "size == relmax", "1.0 == 0.5", id="value"), + ], +) +def test_a_row_the_data_emptied_that_cannot_hold_is_refused( + missing: float, expression: str, row: str +) -> None: + """``0 >= 0.5`` binds nothing linopy can carry, and dropping it would drop an infeasibility.""" with pytest.raises( - SpecDataError, match=r"(?s)constraint 'sized'.*Supply the rows of the variables" + SpecDataError, + match=rf"(?s)constraint 'sized'.*cannot hold, the first at f='b': {row}.*Supply the rows", ): - Model.from_spec(sized(0, "size <= relmax"), ENVELOPE_DATA) + Model.from_spec(sized(missing, expression), ENVELOPE_DATA) def quotient(expression: str, where: str | None = None) -> dict[str, Any]: @@ -667,15 +678,19 @@ def test_arithmetic_masked_out_for_one_row_is_refused_for_the_next_reading_it() @pytest.mark.parametrize( - ("missing", "expression"), [(0, "size >= 0"), (5, "size >= 5")] + ("missing", "expression"), + [(0, "size >= 0"), (5, "size >= 5"), (0, "size <= relmax")], ) -def test_a_dead_row_of_missing_values_whose_sides_are_equal_is_only_warned_about( +def test_a_row_the_data_emptied_that_holds_is_dropped_with_a_warning( missing: float, expression: str ) -> None: - with pytest.warns(UserWarning, match=r"constraint 'sized'.*trivially true"): + with pytest.warns( + linopy.EmptiedRowWarning, match=r"constraint 'sized'.*at f='b'.*dropped" + ): m = solved(sized(missing, expression), ENVELOPE_DATA) assert m.termination_condition == "optimal" - assert (m.constraints["sized"].vars.sel(f="b") == -1).all() + assert m.constraints["sized"].labels.values.tolist() == [0, -1] + assert np.isnan(m.constraints["sized"].dual.sel(f="b")) def test_a_coefficient_needs_no_row_where_its_variable_is_absent() -> None: @@ -745,15 +760,24 @@ def test_a_constraint_is_framed_in_the_name_order_of_its_dims(expression: str) - assert m.constraints["k"].lhs.coord_dims == ("g", "t") -@pytest.mark.parametrize("row", ["sum(x, over=t) >= 5", "sum(x, over=t) <= 0"]) -def test_a_dimension_with_no_members_empties_a_summed_row(row: str) -> None: - spec = with_(SPARSE_SPEC, constraints={"budget": {"dims": [], "expression": row}}) +def test_a_dimension_with_no_members_empties_a_summed_row() -> None: + """A sum of nothing is 0: ``0 <= 0`` holds and ``0 >= 5`` cannot.""" empty = pd.Index([], name="t", dtype=int) none = xr.DataArray(np.array([], dtype=float), coords=[empty]) - with pytest.raises( - SpecDataError, match="constraint 'budget'.*hold no variable term" - ): - Model.from_spec(spec, {"t": empty, "w": none, "c": none}) + data = {"t": empty, "w": none, "c": none} + + def budget(row: str) -> dict[str, Any]: + return with_( + SPARSE_SPEC, constraints={"budget": {"dims": [], "expression": row}} + ) + + with pytest.warns(linopy.EmptiedRowWarning, match="the only row: 0.0 <= 0.0"): + assert ( + "budget" + not in Model.from_spec(budget("sum(x, over=t) <= 0"), data).constraints + ) + with pytest.raises(SpecDataError, match="the only row: 0.0 >= 5.0"): + Model.from_spec(budget("sum(x, over=t) >= 5"), data) @pytest.mark.parametrize(("missing", "kept"), [("absent", False), (0, True)]) @@ -830,6 +854,85 @@ def test_a_fold_reads_a_masked_slot_the_way_its_absence_says( assert not bool(spend.sel(generator="wind").isnull().any()) +# --------------------------------------------------------------------------- +# rows the data empties of every variable term +# --------------------------------------------------------------------------- + +DELAYED_SPEC: dict[str, Any] = { + "dimensions": {"t": {"dtype": "int", "ordered": True}, "k": {"dtype": "str"}}, + "parameters": {"load": {"dims": ["t"]}, "c": {"dims": ["t"]}}, + "variables": { + "x": {"dims": ["t"], "bounds": {"lower": 0, "upper": 10}}, + "y": {"dims": ["t", "k"], "bounds": {"lower": 0, "upper": 10}}, + }, + "objective": { + "sense": "minimize", + "expression": "sum(x, over=t) + sum(y, over=[t, k])", + }, +} +DELAYED_Y = "sum(shift(y, along=t, offset=1, edge=0), over=k)" +ABSENT_Y = "sum(shift(y, along=t, offset=1), over=k)" + + +def delayed(expression: str, load: float = 0.0) -> tuple[dict[str, Any], Any]: + """A row over ``t`` that the data empties at ``t=0``, where ``load`` is *load* and ``c`` is 0.""" + spec = with_( + DELAYED_SPEC, constraints={"r": {"dims": ["t"], "expression": expression}} + ) + data = { + "t": T, + "k": ["a"], + "load": xr.DataArray([load, 1.0, 1.0], coords=[T]), + "c": FULL_C, + } + return spec, data + + +@pytest.mark.parametrize( + ("expression", "load", "row"), + [ + pytest.param(f"{DELAYED_Y} == load", 0.0, None, id="edge-holds"), + pytest.param(f"{DELAYED_Y} == load", 5.0, "0.0 == 5.0", id="edge-fails"), + pytest.param(f"{ABSENT_Y} == load", 0.0, None, id="absent-sum-holds"), + pytest.param(f"{ABSENT_Y} == load", 5.0, "0.0 == 5.0", id="absent-sum-fails"), + pytest.param("c * x <= load", 0.0, None, id="zero-coefficient-holds"), + pytest.param("c * x >= load + 5", 0.0, "0.0 >= 5.0", id="zero-coefficient"), + pytest.param("load + 5 <= c * x", 0.0, "5.0 <= 0.0", id="written-order"), + pytest.param( + "sum(c * y, over=k) >= 1", 0.0, "0.0 >= 1.0", id="zero-coefficient-sum" + ), + pytest.param(f"{DELAYED_Y} >= c * x", 0.0, None, id="both-sides-hold"), + pytest.param(f"{DELAYED_Y} - 1 >= c * x", 0.0, "-1.0 >= 0.0", id="both-sides"), + ], +) +def test_a_row_the_data_empties_is_checked_against_its_constants( + expression: str, load: float, row: str | None +) -> None: + spec, data = delayed(expression, load) + if row is not None: + message = f"constraint 'r': 1 row(s) hold no variable term once the data is attached and cannot hold, the first at t=0: {row}." + with pytest.raises(SpecDataError, match=re.escape(message)): + Model.from_spec(spec, data) + return + with pytest.warns( + linopy.EmptiedRowWarning, match=r"constraint 'r': 1 row\(s\).*at t=0" + ): + m = Model.from_spec(spec, data) + assert m.constraints["r"].labels.values.tolist() == [-1, 1, 2] + + +@pytest.mark.parametrize( + "expression", [f"{DELAYED_Y} <= x", f"x >= {DELAYED_Y}"], ids=["left", "right"] +) +def test_a_row_is_emptied_only_where_neither_side_holds_a_variable( + expression: str, +) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("error", linopy.EmptiedRowWarning) + m = Model.from_spec(*delayed(expression)) + assert (m.constraints["r"].labels != -1).all() + + # --------------------------------------------------------------------------- # piecewise curves as SOS constraints # --------------------------------------------------------------------------- From 17ee0e2f18267edaa6316f48768c8872a2ef7678 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:24:04 +0000 Subject: [PATCH 6/7] fix(spec): an emptied row holds within DROPPED_ROW_TOL A row emptied of variable terms compared its constants exactly, so 0.1 + 0.2 == 0.3 was refused. It now holds within DROPPED_ROW_TOL, the tolerance Model.restrict uses for rows it drops. An integral constant is quoted without a fraction, e.g. 0 == 5. The refusal hint points to the data that empties the row. Tests cover a left side emptied against a right side that keeps variables, and an objective the data zeroes, also through a named expression. --- linopy/spec/builder.py | 39 +++++++++++++++++++++------------- linopy/spec/evaluator.py | 10 +++------ test/test_spec_builder.py | 44 ++++++++++++++++++++++++++++----------- 3 files changed, 60 insertions(+), 33 deletions(-) diff --git a/linopy/spec/builder.py b/linopy/spec/builder.py index 3ba1c392..a5aa7804 100644 --- a/linopy/spec/builder.py +++ b/linopy/spec/builder.py @@ -12,6 +12,7 @@ import operator import warnings +from collections.abc import Callable from dataclasses import replace from typing import cast @@ -21,6 +22,7 @@ from linopy.constants import FACTOR_DIM, TERM_DIM, EmptiedRowWarning from linopy.expressions import LinearExpression, QuadraticExpression +from linopy.restrict.matrix import DROPPED_ROW_TOL from linopy.spec.context import Context, Term, Value, as_expression, is_empty from linopy.spec.errors import SpecDataError, first_coordinates from linopy.spec.evaluator import ( @@ -36,7 +38,11 @@ from linopy.variables import Variable _FLIPPED: dict[str, ms.ConstraintSense] = {"==": "==", "<=": ">=", ">=": "<="} -_COMPARE = {"==": operator.eq, "<=": operator.le, ">=": operator.ge} +_VIOLATION: dict[str, Callable[[xr.DataArray], xr.DataArray]] = { + "==": abs, + "<=": lambda gap: gap, + ">=": operator.neg, +} _SENSE = {"minimize": "min", "maximize": "max"} @@ -244,23 +250,23 @@ def _check_emptied( Drop with a warning the rows *emptied* names where they hold, refuse them where they cannot. The data left such a row without a variable term, so it reads as - ``constant sense constant``, compared exactly as written. A row that - holds constrains nothing; one that cannot hold makes the problem - infeasible, and linopy, which carries no row without a term, would drop - it silently. + ``constant sense constant``, compared as written within + ``DROPPED_ROW_TOL``, the tolerance of :meth:`linopy.Model.restrict`. A + row that holds constrains nothing; one that cannot hold makes the + problem infeasible, and linopy, which carries no row without a term, + would drop it silently. """ lhs, rhs = _constant_of(term), _constant_of(other) if flipped(declared): lhs, rhs = rhs, lhs - holds = _COMPARE[declared.sense](lhs, rhs) - failing = emptied & ~holds + failing = emptied & (_VIOLATION[declared.sense](lhs - rhs) > DROPPED_ROW_TOL) said = f"constraint '{name}': {{}} row(s) hold no variable term once the data is attached" if bool(failing.any()): raise SpecDataError( f"{said.format(int(failing.sum()))} and cannot hold, the first at " f"{_quoted(failing, lhs, rhs, declared.sense)}.\n" f" Mask them out with a where on the constraint, if the row should not exist there.\n" - f" Supply the rows of the variables, if the row is meant to bind." + f" Fix the data that empties the row, if the row is meant to bind." ) warnings.warn( f"{said.format(int(emptied.sum()))}, the first at " @@ -280,7 +286,12 @@ def _quoted( float(side.broadcast_like(flags).transpose(*flags.dims)[at]) for side in (lhs, rhs) ) - return f"{first_coordinates(flags, 1)}: {left!r} {sense} {right!r}" + return f"{first_coordinates(flags, 1)}: {_number(left)} {sense} {_number(right)}" + + +def _number(value: float) -> str: + """*value* in its shortest exact form, an integral one without a fraction.""" + return repr(value).removesuffix(".0") def _objective(ctx: Context) -> None: @@ -290,8 +301,9 @@ def _objective(ctx: Context) -> None: refuse_holes("the objective", [declared.expression], ctx) expr = evaluate(declared.expression, ctx) if _constant(expr): - expr = evaluate(declared.expression, _terms_dropped(ctx, zeroes=False)) - _terms_dropped(ctx) + _forget_terms(ctx) + expr = evaluate(declared.expression, replace(ctx, zeroes=False)) + _forget_terms(ctx) if not isinstance(expr, Term): raise SpecDataError( "the objective carries no variable term once the data is attached, so there is nothing to optimize" @@ -301,8 +313,8 @@ def _objective(ctx: Context) -> None: ) -def _terms_dropped(ctx: Context, zeroes: bool = True) -> Context: - """*ctx* with the terms its memo keeps dropped, a term the data scales by zero pruned as *zeroes* says.""" +def _forget_terms(ctx: Context) -> None: + """Drop the terms the memo of *ctx* keeps, so the next evaluation makes them again.""" values = ctx.memo.values for key in [ k @@ -310,7 +322,6 @@ def _terms_dropped(ctx: Context, zeroes: bool = True) -> Context: if isinstance(v, LinearExpression | QuadraticExpression) ]: del values[key] - return replace(ctx, zeroes=zeroes) def _linear_if_no_product(expr: Term) -> Term: diff --git a/linopy/spec/evaluator.py b/linopy/spec/evaluator.py index d6aef6d3..94c18915 100644 --- a/linopy/spec/evaluator.py +++ b/linopy/spec/evaluator.py @@ -262,11 +262,7 @@ def _combine(node: _Binary, ctx: Context) -> Value: def _is_constant(value: Value) -> bool: - """ - Whether *value* is a term without a variable term: a product of it and - another term scales that term by its constant, as data, its absence a - hole, and stays a term. - """ + """Whether *value* is a term without a variable term, which scales another term like data.""" return isinstance(value, LinearExpression) and value.is_constant @@ -278,10 +274,10 @@ def _without_zeroed_terms(made: Value, left: Value, right: Value) -> Value: a constant, and a row left with nothing but such terms holds no variable term. """ - if not isinstance(made, LinearExpression) or type(made) is not LinearExpression: + if type(made) is not LinearExpression: return made data = right if isinstance(left, Term) else left - if isinstance(data, Term) or not bool(np.any(np.asarray(data) == 0)): + if not bool(np.any(np.asarray(data) == 0)): return made return LinearExpression._from_store(made._store.prune(), made.model) diff --git a/test/test_spec_builder.py b/test/test_spec_builder.py index 80466ad6..8cf0dd33 100644 --- a/test/test_spec_builder.py +++ b/test/test_spec_builder.py @@ -562,8 +562,8 @@ def test_an_edge_value_stands_in_for_the_vacated_row() -> None: @pytest.mark.parametrize( ("missing", "expression", "row"), [ - pytest.param(0, "size >= relmax", "0.0 >= 0.5", id="zero"), - pytest.param(1, "size == relmax", "1.0 == 0.5", id="value"), + pytest.param(0, "size >= relmax", "0 >= 0.5", id="zero"), + pytest.param(1, "size == relmax", "1 == 0.5", id="value"), ], ) def test_a_row_the_data_emptied_that_cannot_hold_is_refused( @@ -572,7 +572,7 @@ def test_a_row_the_data_emptied_that_cannot_hold_is_refused( """``0 >= 0.5`` binds nothing linopy can carry, and dropping it would drop an infeasibility.""" with pytest.raises( SpecDataError, - match=rf"(?s)constraint 'sized'.*cannot hold, the first at f='b': {row}.*Supply the rows", + match=rf"(?s)constraint 'sized'.*cannot hold, the first at f='b': {row}.*Fix the data", ): Model.from_spec(sized(missing, expression), ENVELOPE_DATA) @@ -771,12 +771,12 @@ def budget(row: str) -> dict[str, Any]: SPARSE_SPEC, constraints={"budget": {"dims": [], "expression": row}} ) - with pytest.warns(linopy.EmptiedRowWarning, match="the only row: 0.0 <= 0.0"): + with pytest.warns(linopy.EmptiedRowWarning, match="the only row: 0 <= 0"): assert ( "budget" not in Model.from_spec(budget("sum(x, over=t) <= 0"), data).constraints ) - with pytest.raises(SpecDataError, match="the only row: 0.0 >= 5.0"): + with pytest.raises(SpecDataError, match="the only row: 0 >= 5"): Model.from_spec(budget("sum(x, over=t) >= 5"), data) @@ -892,17 +892,18 @@ def delayed(expression: str, load: float = 0.0) -> tuple[dict[str, Any], Any]: ("expression", "load", "row"), [ pytest.param(f"{DELAYED_Y} == load", 0.0, None, id="edge-holds"), - pytest.param(f"{DELAYED_Y} == load", 5.0, "0.0 == 5.0", id="edge-fails"), + pytest.param(f"{DELAYED_Y} == load", 5.0, "0 == 5", id="edge-fails"), + pytest.param(f"{DELAYED_Y} + 0.1 + 0.2 == load", 0.3, None, id="rounding"), pytest.param(f"{ABSENT_Y} == load", 0.0, None, id="absent-sum-holds"), - pytest.param(f"{ABSENT_Y} == load", 5.0, "0.0 == 5.0", id="absent-sum-fails"), + pytest.param(f"{ABSENT_Y} == load", 5.0, "0 == 5", id="absent-sum-fails"), pytest.param("c * x <= load", 0.0, None, id="zero-coefficient-holds"), - pytest.param("c * x >= load + 5", 0.0, "0.0 >= 5.0", id="zero-coefficient"), - pytest.param("load + 5 <= c * x", 0.0, "5.0 <= 0.0", id="written-order"), + pytest.param("c * x >= load + 5", 0.0, "0 >= 5", id="zero-coefficient"), + pytest.param("load + 5 <= c * x", 0.0, "5 <= 0", id="written-order"), pytest.param( - "sum(c * y, over=k) >= 1", 0.0, "0.0 >= 1.0", id="zero-coefficient-sum" + "sum(c * y, over=k) >= 1", 0.0, "0 >= 1", id="zero-coefficient-sum" ), pytest.param(f"{DELAYED_Y} >= c * x", 0.0, None, id="both-sides-hold"), - pytest.param(f"{DELAYED_Y} - 1 >= c * x", 0.0, "-1.0 >= 0.0", id="both-sides"), + pytest.param(f"{DELAYED_Y} - 1 >= c * x", 0.0, "-1 >= 0", id="both-sides"), ], ) def test_a_row_the_data_empties_is_checked_against_its_constants( @@ -922,7 +923,9 @@ def test_a_row_the_data_empties_is_checked_against_its_constants( @pytest.mark.parametrize( - "expression", [f"{DELAYED_Y} <= x", f"x >= {DELAYED_Y}"], ids=["left", "right"] + "expression", + [f"{DELAYED_Y} <= x", f"x >= {DELAYED_Y}", f"{DELAYED_Y} == sum(y, over=k) - load"], + ids=["left", "right", "left-against-a-sum"], ) def test_a_row_is_emptied_only_where_neither_side_holds_a_variable( expression: str, @@ -933,6 +936,23 @@ def test_a_row_is_emptied_only_where_neither_side_holds_a_variable( assert (m.constraints["r"].labels != -1).all() +@pytest.mark.parametrize( + ("objective", "expressions"), + [ + pytest.param("sum(c * x, over=t)", {}, id="inline"), + pytest.param("spend", {"spend": "sum(c * x, over=t)"}, id="named"), + ], +) +def test_an_objective_the_data_zeroes_still_solves( + objective: str, expressions: dict[str, str] +) -> None: + """Dropping the terms the data zeroes must not leave the objective without a variable.""" + spec, data = delayed("x >= load") + spec = with_(spec, objective={"expression": objective}, expressions=expressions) + m = solved(spec, {**data, "c": xr.zeros_like(FULL_C)}) + assert m.termination_condition == "optimal" + + # --------------------------------------------------------------------------- # piecewise curves as SOS constraints # --------------------------------------------------------------------------- From 68fdc0c50b406aef330e59a40635c3abafedba64 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:24:05 +0000 Subject: [PATCH 7/7] docs(spec): describe how a row the data empties is checked A row the data leaves without a variable term is dropped with EmptiedRowWarning where it holds and refused where it cannot. The overview and the spec data guide described the old refusal. EmptiedRowWarning is listed with the other warnings. --- doc/api/warnings.md | 4 ++++ doc/release-notes.md | 3 ++- doc/user-guide/overview.md | 11 +++++++---- doc/user-guide/spec-data.md | 14 +++++++++----- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/doc/api/warnings.md b/doc/api/warnings.md index 3e897036..920034f4 100644 --- a/doc/api/warnings.md +++ b/doc/api/warnings.md @@ -5,6 +5,10 @@ These warning classes can be silenced or filtered via [`warnings.filterwarnings()`][warnings.filterwarnings]. +::: linopy.EmptiedRowWarning + options: + heading_level: 3 + ::: linopy.EvolvingAPIWarning options: heading_level: 3 diff --git a/doc/release-notes.md b/doc/release-notes.md index a1c3ff1e..a8590b1b 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -29,7 +29,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds faster: it reads a shifted axis by position, checks a relation's labels against the hash table of their dimension, decides a `where` early when a condition holds nowhere, and sets a scalar scaling factor without broadcasting it. ([\#1169](https://github.com/PyPSA/linopy/pull/1169)) - `Model.from_spec(..., rename=...)` builds each variable and constraint under the name a mapping or a function gives it, e.g. `Generator-p` for the spec's `Generator_p`. Pass the same `rename` to `linopy.spec.evaluate`. ([\#1160](https://github.com/PyPSA/linopy/pull/1160)) - A missing row takes the value that `missing:` sets, e.g. `missing: 0`, in bounds, expressions and conditions alike. A parameter's `where:` limits its coordinates, and a dimension's `one_of:` its labels. Built objects follow the name order of their dimensions, as mathspec sorts every `dims:` list. ([\#1137](https://github.com/PyPSA/linopy/pull/1137)) -- Limits: a constraint row that absence leaves without terms raises, where mathspec skips it ([\#993](https://github.com/PyPSA/linopy/issues/993)). `store=True` stores only relations with one key and one value column. A product of two variables in a constraint is accepted only where the data leaves it no live row. +- Limits: a constraint row that the data leaves without terms raises where it cannot hold, where mathspec skips it ([\#993](https://github.com/PyPSA/linopy/issues/993)). `store=True` stores only relations with one key and one value column. A product of two variables in a constraint is accepted only where the data leaves it no live row. **Other features** @@ -131,6 +131,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de - `Model.from_spec` builds a linear objective when the data sets all quadratic coefficients to zero. ([\#1141](https://github.com/PyPSA/linopy/pull/1141)) - `Model.from_spec` no longer refuses an infinite parameter times a variable absent there, where no row is built. ([\#1143](https://github.com/PyPSA/linopy/pull/1143)) - `Model.from_spec` adds no constraint where every variable of it is masked, where it raised that both sides are constant. ([\#1154](https://github.com/PyPSA/linopy/pull/1154)) +- `Model.from_spec` checks a constraint row that the data leaves without a variable term, e.g. through a shift's edge, a sum of absent terms or a zero coefficient: it drops the row with a [`linopy.EmptiedRowWarning`][linopy.EmptiedRowWarning] where it holds and refuses it where it cannot, e.g. `0 == 5`, where it refused both or built `0 * x >= 5`. ([\#1164](https://github.com/PyPSA/linopy/pull/1164)) - `LinearExpression(data, model)` no longer raises on a `.sel` over full coordinates. ([\#1144](https://github.com/PyPSA/linopy/pull/1144)) - `LinearExpression.from_constant` builds on the model's store, so a sparse model's constant stays sparse and adding it emits no `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) - `Model.from_spec(..., sparse=True)` keeps a `shift` whose offset differs per entity sparse, without a `PerformanceWarning`. ([\#1145](https://github.com/PyPSA/linopy/pull/1145)) diff --git a/doc/user-guide/overview.md b/doc/user-guide/overview.md index c68a0a35..5d076ce5 100644 --- a/doc/user-guide/overview.md +++ b/doc/user-guide/overview.md @@ -111,10 +111,13 @@ data, and let linopy build it. data in `model.parameters`, reading results, saving and rebuilding, typesetting and variants. -linopy supports a subset of mathspec. A constraint row that absence -leaves without terms (a `missing: absent` parameter, a `missing: 0` parameter such as `0 == load`, or a `where:`-masked -variable inside a sum) is refused, where mathspec skips and reports it; -linopy is stricter here ([\#993](https://github.com/PyPSA/linopy/issues/993)). `store=True` stores +linopy supports a subset of mathspec. A constraint row that the data leaves +without a variable term (a `missing: 0` parameter such as `0 == load`, a shift +past its edge, or a sum whose terms are all absent or `where:`-masked) reads +`constant sense constant`. linopy drops it with a `linopy.EmptiedRowWarning` +where it holds and refuses it where it cannot, e.g. `0 == 5`. mathspec skips +and reports such a row; linopy is stricter here +([\#993](https://github.com/PyPSA/linopy/issues/993)). `store=True` stores only relations of one key column and one value column, and raises `SpecDataError` for any other. diff --git a/doc/user-guide/spec-data.md b/doc/user-guide/spec-data.md index 91c2fa95..db642004 100644 --- a/doc/user-guide/spec-data.md +++ b/doc/user-guide/spec-data.md @@ -251,11 +251,15 @@ variables: propagate to the whole row. An `absent` row read outside a sum takes the row with it, so `x <= c` has no row where `c` is absent; write `where: c` on the constraint to make that visible in the spec. -- Inside a sum, an absent term is left out. A row the data leaves without any - variable term, such as a sum over variables a `where:` masked away - entirely, is refused rather than dropped: nothing is left to constrain, and - the row would leave the problem without saying so. Mask it with a `where:` - on the constraint ([\#993](https://github.com/PyPSA/linopy/issues/993)). +- Inside a sum, an absent term is left out, and a sum of absent terms only + is 0. A row the data leaves without any variable term, such as a sum over + variables a `where:` masked away entirely, a shift past its edge or a + coefficient of 0, reads `constant sense constant`, compared within a + tolerance of `1e-9`. Where it holds, e.g. `0 <= 5`, the row is dropped with + a `linopy.EmptiedRowWarning`. Where it cannot, e.g. `0 >= 5`, it is refused + with the first such row: dropping it would hide an infeasible problem. + Mask such rows with a `where:` on the constraint + ([\#993](https://github.com/PyPSA/linopy/issues/993)). - linopy carries a product of two variables in the objective only. A constraint whose spec multiplies two variables builds wherever the data leaves no such product on a row its `where:` keeps: its coefficient is