Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/release-notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ linopy 1.0 makes the strict arithmetic convention the only behaviour and puts de

*Solvers*

- `solve`, `to_file` and `Model.restrict` raise for a constraint row without a nonzero term that cannot hold, e.g. `0 * x >= 5`, instead of leaving it out and solving "optimal". ([\#1175](https://github.com/PyPSA/linopy/pull/1175))
- With `add_objective(..., scaling=...)`, the solve log reports objective and dual bound in model units. ([\#1077](https://github.com/PyPSA/linopy/issues/1077))
- CPLEX: "optimal with unscaled infeasibilities" is reported as `imprecise` and its solution is imported. ([\#368](https://github.com/PyPSA/linopy/issues/368))
- HiGHS: a solve that stops at a limit without a solution no longer returns all-zero values, and a MIP no longer returns all-zero duals. `log_fn` and solver options also apply to a re-solve of a direct-API model. ([\#1130](https://github.com/PyPSA/linopy/issues/1130))
Expand Down
15 changes: 15 additions & 0 deletions linopy/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -1250,6 +1250,21 @@ def is_constant(x: SideLike) -> bool:
)


def unsatisfied(
rhs: np.ndarray, sense: np.ndarray, tol: float | np.ndarray
) -> np.ndarray:
"""
Per row, whether ``0 sense rhs`` fails by more than ``tol``.

``sense`` holds one character per row: ``"="``, ``"<"`` or ``">"``.
"""
return (
((sense == "=") & (np.abs(rhs) > tol))
| ((sense == "<") & (rhs < -tol))
| ((sense == ">") & (rhs > tol))
)


def values_to_lookup_array(
values: np.ndarray, labels: np.ndarray, size: int | None = None
) -> np.ndarray:
Expand Down
2 changes: 2 additions & 0 deletions linopy/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
GREATER_EQUAL = ">="
LESS_EQUAL = "<="

DROPPED_ROW_TOL = 1e-9


class PerformanceWarning(UserWarning):
"""Warning raised when an operation triggers expensive Dataset reconstruction."""
Expand Down
34 changes: 34 additions & 0 deletions linopy/constraints.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,11 @@
skip_full_indexers,
slice_indexers,
to_dataframe,
unsatisfied,
)
from linopy.config import options
from linopy.constants import (
DROPPED_ROW_TOL,
EQUAL,
GREATER_EQUAL,
HELPER_DIMS,
Expand Down Expand Up @@ -1007,6 +1009,18 @@ def active_row_mask(self) -> np.ndarray:
"""Boolean mask over raveled rows: label set and at least one variable present."""
return (self._label_values() != -1) & self._lhs._store.has_terms()

def unsatisfiable_empty_rows(self) -> np.ndarray:
"""
Boolean mask over raveled rows: label set, no nonzero term, and
``0 sign rhs`` fails.
"""
sense = self.sign.values.ravel().astype("U1")
rows = self._label_values() != -1
rows &= unsatisfied(self.rhs.values.ravel(), sense, DROPPED_ROW_TOL)
if rows.any():
rows[rows] = ~self._lhs._store.has_live_terms(rows)
return rows

def active_labels(self) -> np.ndarray:
"""Active constraint labels in build order, without building the CSR."""
return self._label_values()[self.active_row_mask()]
Expand Down Expand Up @@ -1522,6 +1536,26 @@ def sanitize_infinities(self) -> None:
for con in self.data.values():
con.sanitize_infinities()

def check_empty_rows(self) -> None:
"""
Raise for a row without a nonzero term whose ``0 sign rhs`` fails.

Matrices and LP files leave such rows out, so the solver would never
see them. Rows without a nonzero term that hold are left out silently.
"""
for con in self.data.values():
if con.is_indicator:
continue
rows = con.unsatisfiable_empty_rows()
if rows.any():
labels = con._label_values()[rows]
raise ValueError(
f"Constraint `{con.name}` has {labels.size} row(s) without a "
f"nonzero term that cannot hold, first: "
f"{self.format_labels(labels[:1])}. Fix their coefficients "
"or right-hand side, or mask them."
)

def format_labels(
self, values: Sequence[int], display_max_terms: int | None = None
) -> str:
Expand Down
9 changes: 9 additions & 0 deletions linopy/csr.py
Original file line number Diff line number Diff line change
Expand Up @@ -583,6 +583,15 @@ def terms(
counts -= np.bincount(rows, minlength=self.n_cells).astype(counts.dtype)
return counts, self.csr.indices[keep], data[keep]

def has_live_terms(self, cells: np.ndarray) -> np.ndarray:
indptr = self.csr.indptr
counts = np.diff(indptr)
zeros = np.flatnonzero(self.csr.data == 0)
if zeros.size:
rows = np.searchsorted(indptr, zeros, side="right") - 1
counts = counts - np.bincount(rows, minlength=self.n_cells)
return counts[cells] > 0

def variable_labels(self) -> np.ndarray:
return self.csr.indices

Expand Down
2 changes: 2 additions & 0 deletions linopy/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -812,6 +812,8 @@ def to_file(
if progress is None:
progress = m._xCounter > 10_000

m.constraints.check_empty_rows()

if io_api == "lp" or io_api == "lp-polars":
to_lp_file(
m,
Expand Down
5 changes: 4 additions & 1 deletion linopy/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
to_path,
)
from linopy.constants import (
DROPPED_ROW_TOL,
FACTOR_DIM,
GREATER_EQUAL,
LESS_EQUAL,
Expand Down Expand Up @@ -92,7 +93,6 @@
add_piecewise_formulation,
)
from linopy.remote import RemoteHandler
from linopy.restrict.matrix import DROPPED_ROW_TOL
from linopy.scaling import scalar_scaling, validate_scaling
from linopy.semantics import check_user_nan, enforce_no_multiindex
from linopy.solver_capabilities import solver_supports
Expand Down Expand Up @@ -2205,6 +2205,7 @@ def solve(
"`m.add_objective(...)` first (e.g. `m.add_objective(0 * x)` "
"for a pure feasibility problem)."
)
self.constraints.check_empty_rows()
solved = remote.solve_on_remote(
self,
solver_name=solver_name,
Expand Down Expand Up @@ -2620,6 +2621,8 @@ def _mock_solve(
if sanitize_infinities:
self.constraints.sanitize_infinities()

self.constraints.check_empty_rows()

self.objective._value = 0.0
self.status = "ok"
self.termination_condition = TerminationCondition.optimal.value
Expand Down
111 changes: 74 additions & 37 deletions linopy/restrict/matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,20 @@
import numpy as np
import pandas as pd
import scipy.sparse
from numpy.typing import DTypeLike
from xarray import DataArray
from xarray.indexes import PandasIndex

from linopy.alignment import as_dataarray
from linopy.common import lookup_table
from linopy.common import lookup_table, unsatisfied
from linopy.constants import DROPPED_ROW_TOL
from linopy.csr import indptr_from_counts, repeat_per_row
from linopy.matrices import Matrices

if TYPE_CHECKING:
from linopy.model import Model

__all__ = ["DROPPED_ROW_TOL", "MatrixWindow", "RestrictionError"]

DROPPED_ROW_TOL = 1e-9
__all__ = ["MatrixWindow", "RestrictionError"]


class RestrictionError(ValueError):
Expand Down Expand Up @@ -102,12 +102,16 @@ def _select(containers: Any, indexers: _Indexers) -> dict[str, DataArray]:
return selected


def _flat(selected: dict[str, DataArray], dtype: DTypeLike) -> np.ndarray:
"""The selected labels of all containers, raveled, in container order."""
empty = np.empty(0, dtype=dtype)
return np.concatenate([empty, *(lab.values.ravel() for lab in selected.values())])


def _positions(
selected: dict[str, DataArray], label_to_pos: np.ndarray, distinct: bool
flat: np.ndarray, label_to_pos: np.ndarray, distinct: bool
) -> np.ndarray:
"""Parent matrix positions of the selected active labels, in container order."""
empty = np.empty(0, dtype=label_to_pos.dtype)
flat = np.concatenate([empty, *(lab.values.ravel() for lab in selected.values())])
positions = _active_positions(flat, label_to_pos)
if not distinct and np.unique(positions).size != positions.size:
raise RestrictionError("Cannot restrict model: the selection repeats labels.")
Expand Down Expand Up @@ -146,28 +150,24 @@ def _equal(value: DataArray, indexes: dict[str, pd.Index]) -> bool:
)


def _check_dropped(
rhs: np.ndarray,
sense: np.ndarray,
magnitude: np.ndarray,
clabels: np.ndarray,
tol: float,
) -> None:
"""Raise when a row that lost every term violates its sign beyond ``tol``."""
tolerance = tol * np.maximum(1.0, magnitude)
violated = (
((sense == "=") & (np.abs(rhs) > tolerance))
| ((sense == "<") & (rhs < -tolerance))
| ((sense == ">") & (rhs > tolerance))
def _empty_rows(model: Model) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
Labels, right-hand sides and signs of the labelled rows without a term,
which the model's matrices leave out.
"""
labels, rhs, sense = [], [], []
for con in model.constraints.data.values():
own = con.labels.values.ravel()
rows = (own != -1) & ~con.active_row_mask()
labels.append(own[rows])
rhs.append(con.rhs.values.ravel()[rows])
sense.append(con.sign.values.ravel()[rows].astype("U1"))
dtype = model._dtypes["labels"]
return (
np.concatenate([np.empty(0, dtype=dtype), *labels]),
np.concatenate([np.empty(0), *rhs]),
np.concatenate([np.empty(0, dtype="U1"), *sense]),
)
if violated.any():
raise RestrictionError(
"Cannot window the matrices: folding the dangling terms makes "
f"constraint labels {clabels[violated].tolist()} unsatisfiable "
f"(folded rhs {rhs[violated].tolist()} against sign "
f"{sense[violated].tolist()}). Raise `tol` or exclude the rows "
"via `drop`."
)


class MatrixWindow:
Expand Down Expand Up @@ -197,6 +197,7 @@ def __init__(self, model: Model) -> None:
raise NotImplementedError("Matrix windows require at least one constraint.")
self._model = model
self._matrices = matrices
self._empty = _empty_rows(model)
self._A = matrices.A
frame = {f.name: vars(matrices)[f.name] for f in fields(Matrices)}
self._frame = Matrices(**{**frame, "A": None})
Expand All @@ -217,7 +218,8 @@ def select(
``fix`` maps variable names to the values of out-of-window terms, which
are folded into the right-hand side at their actual coefficient; a row
that loses every term is dropped when its folded right-hand side
satisfies its sign within ``tol``, relative to the folded magnitude.
satisfies its sign within ``tol``, relative to the folded magnitude. A
selected row without any term in the parent is checked alike.
``drop`` maps constraint names to boolean masks of rows removed from
the window before folding.

Expand All @@ -236,13 +238,19 @@ def select(
indexers = _Indexers(selection)
variables = _select(m.variables, indexers)
constraints = _select(m.constraints, indexers)
dtype = m._dtypes["labels"]
cols = _positions(
variables, m.variables.label_index.label_to_pos, indexers.distinct
_flat(variables, dtype),
m.variables.label_index.label_to_pos,
indexers.distinct,
)
clabels = _flat(constraints, dtype)
rows = _positions(
constraints, m.constraints.label_index.label_to_pos, indexers.distinct
clabels, m.constraints.label_index.label_to_pos, indexers.distinct
)
rows = rows[~self._excluded(drop)[rows]]
excluded = self._excluded(drop)
rows = rows[~excluded[mat.clabels[rows]]]
self._check_empty(clabels, excluded, tol)
if not cols.size:
raise RestrictionError(
f"Cannot restrict model: the selection {dict(selection)} is empty."
Expand Down Expand Up @@ -273,7 +281,7 @@ def select(
keep = counts > 0
magnitude = np.bincount(outer_rows, np.abs(folded), minlength=rows.size)
dropped = rows[~keep]
_check_dropped(
self._check_dropped(
rhs[~keep], mat.sense[dropped], magnitude[~keep], mat.clabels[dropped], tol
)
indptr = indptr_from_counts(counts[keep], A.indptr.dtype)
Expand Down Expand Up @@ -301,10 +309,39 @@ def _fixed(self, fix: Mapping[str, Any] | None, cols: np.ndarray) -> np.ndarray:
values[own] = aligned[labels[own] - start]
return values * mat.var_scaling[cols]

def _check_dropped(
self,
rhs: np.ndarray,
sense: np.ndarray,
magnitude: np.ndarray,
clabels: np.ndarray,
tol: float,
) -> None:
"""Raise when a row without a term in the window violates its sign beyond ``tol``."""
violated = unsatisfied(rhs, sense, tol * np.maximum(1.0, magnitude))
if violated.any():
first = self._model.constraints.format_labels(clabels[violated][:1])
raise RestrictionError(
"Cannot window the matrices: constraint labels "
f"{clabels[violated].tolist()} have no term in the window and "
f"are unsatisfiable (rhs {rhs[violated].tolist()} against sign "
f"{sense[violated].tolist()}), first: {first}. Raise `tol` or "
"exclude the rows via `drop`."
)

def _check_empty(
self, clabels: np.ndarray, excluded: np.ndarray, tol: float
) -> None:
"""Raise for a selected row without a term that ``drop`` keeps and that fails."""
labels, rhs, sense = self._empty
kept = np.isin(labels, clabels) & ~excluded[labels]
magnitude = np.zeros(int(kept.sum()))
self._check_dropped(rhs[kept], sense[kept], magnitude, labels[kept], tol)

def _excluded(self, drop: Mapping[str, Any] | None) -> np.ndarray:
"""Whether each parent row is excluded by ``drop``, by matrix position."""
"""Whether each parent row is excluded by ``drop``, by constraint label."""
constraints = self._model.constraints
excluded = np.zeros(self._matrices.clabels.size, dtype=bool)
excluded = np.zeros(self._model._cCounter, dtype=bool)
for name, given in (drop or {}).items():
if name not in constraints:
raise RestrictionError(
Expand All @@ -317,6 +354,6 @@ def _excluded(self, drop: Mapping[str, Any] | None) -> np.ndarray:
raise RestrictionError(
f"{what} must be boolean, got dtype {mask.dtype}."
)
label_to_pos = constraints.label_index.label_to_pos
excluded[_active_positions(labels.values[mask], label_to_pos)] = True
dropped = labels.values[mask]
excluded[dropped[dropped != -1]] = True
return excluded
3 changes: 2 additions & 1 deletion linopy/restrict/window.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
from xarray import DataArray

from linopy.common import to_path
from linopy.restrict.matrix import DROPPED_ROW_TOL, MatrixWindow, RestrictionError
from linopy.constants import DROPPED_ROW_TOL
from linopy.restrict.matrix import MatrixWindow, RestrictionError

if TYPE_CHECKING:
from linopy.constants import Result, Solution
Expand Down
2 changes: 2 additions & 0 deletions linopy/solvers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1008,6 +1008,7 @@ def _build_direct(
"""Build the native solver model from the matrices of ``self.model``."""
model = self.model
assert model is not None
model.constraints.check_empty_rows()
M = model.matrices
names = (
linopy.io.get_printers_scalar(
Expand Down Expand Up @@ -1233,6 +1234,7 @@ def _apply_locked(
if isinstance(diff, RebuildReason):
self._fall_back_to_rebuild(diff, rebuild, disallow_rebuild)
return diff
model.constraints.check_empty_rows()
n_vars = model.variables.label_index.n_active_vars
n_cons = model.constraints.label_index.n_active_cons
if self._apply_or_rebuild(diff, n_vars, n_cons, rebuild, disallow_rebuild):
Expand Down
13 changes: 13 additions & 0 deletions linopy/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,13 @@ def terms(
per-term index.
"""

@abstractmethod
def has_live_terms(self, cells: np.ndarray) -> np.ndarray:
"""
Per cell selected by the boolean ``cells``, whether it holds a term
with a label and a nonzero coefficient.
"""

@abstractmethod
def variable_labels(self) -> np.ndarray:
"""
Expand Down Expand Up @@ -757,6 +764,12 @@ def terms(
counts = np.count_nonzero(keep, axis=-1).reshape(-1)
return counts, vars_[keep], coeffs[keep]

def has_live_terms(self, cells: np.ndarray) -> np.ndarray:
coeffs, vars_ = self._term_views()
live = vars_ != -1
live &= coeffs != 0
return live.any(axis=-1).reshape(-1)[cells]

def variable_labels(self) -> np.ndarray:
return self.ds.vars.values.ravel()

Expand Down
Loading
Loading