From 69a318878cdec4f2d62fbf741b5c0c5947435af7 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:15:42 +0200 Subject: [PATCH 01/66] Changes by JD before edits of MS --- edisgo/edisgo.py | 21 + edisgo/run/__init__.py | 31 ++ edisgo/run/config.py | 410 ++++++++++++++++++ edisgo/run/context.py | 115 +++++ edisgo/run/presets/basic.yaml | 22 + .../run/presets/r4mu_base_and_scenario.yaml | 49 +++ edisgo/run/presets/uc1_loads_worst_case.yaml | 32 ++ edisgo/run/presets/uc2_flex_opf.yaml | 43 ++ edisgo/run/presets/uc3_oedb_ts.yaml | 36 ++ edisgo/run/registry.py | 113 +++++ edisgo/run/runner.py | 261 +++++++++++ edisgo/run/tasks/__init__.py | 25 ++ edisgo/run/tasks/io.py | 179 ++++++++ edisgo/run/validator.py | 201 +++++++++ setup.py | 1 + tests/run/__init__.py | 1 + tests/run/test_config.py | 154 +++++++ tests/run/test_registry.py | 35 ++ tests/run/test_runner.py | 118 +++++ tests/run/test_validator.py | 97 +++++ 20 files changed, 1944 insertions(+) create mode 100644 edisgo/run/__init__.py create mode 100644 edisgo/run/config.py create mode 100644 edisgo/run/context.py create mode 100644 edisgo/run/presets/basic.yaml create mode 100644 edisgo/run/presets/r4mu_base_and_scenario.yaml create mode 100644 edisgo/run/presets/uc1_loads_worst_case.yaml create mode 100644 edisgo/run/presets/uc2_flex_opf.yaml create mode 100644 edisgo/run/presets/uc3_oedb_ts.yaml create mode 100644 edisgo/run/registry.py create mode 100644 edisgo/run/runner.py create mode 100644 edisgo/run/tasks/__init__.py create mode 100644 edisgo/run/tasks/io.py create mode 100644 edisgo/run/validator.py create mode 100644 tests/run/__init__.py create mode 100644 tests/run/test_config.py create mode 100644 tests/run/test_registry.py create mode 100644 tests/run/test_runner.py create mode 100644 tests/run/test_validator.py diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 07324d010..63b7753b9 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -232,6 +232,27 @@ def config(self): def config(self, kwargs): self._config = Config(**kwargs) + def run_pipeline(self, config): + """ + Run a YAML/JSON task pipeline on this EDisGo instance. + + See :mod:`edisgo.run` for the config schema and task list. + + Parameters + ---------- + config : str, :class:`pathlib.Path`, or dict + Pipeline config as path to a YAML/JSON file or as a dict. + + Returns + ------- + :class:`~.EDisGo` + The EDisGo instance after the pipeline has run. + + """ + from edisgo.run import _run_pipeline_on + + return _run_pipeline_on(self, config) + def import_ding0_grid(self, path, legacy_ding0_grids=True): """ Import ding0 topology data from csv files in the format as diff --git a/edisgo/run/__init__.py b/edisgo/run/__init__.py new file mode 100644 index 000000000..cd202ddef --- /dev/null +++ b/edisgo/run/__init__.py @@ -0,0 +1,31 @@ +""" +YAML/JSON-driven pipeline runner for eDisGo. + +Two entry points share the same core: + + from edisgo.run import run_edisgo + edisgo = run_edisgo("presets/uc2_flex_opf.yaml") + + # or, on an existing EDisGo instance: + edisgo = EDisGo(ding0_grid="30879") + edisgo.run_pipeline("my_run.yaml") + +Pipelines are lists of named tasks from :mod:`edisgo.run.tasks`. Each step +is either a string (``worst_case_ts``) or a single-key mapping with +parameters (``import_electromobility: {charging_strategy: dumb}``). Tasks +can be grouped into ordered ``stages`` that can save artifacts and reload +them with ``load_from``, enabling two-phase workflows (base reinforce + +per-scenario reinforce). +""" + +from edisgo.run.context import RunContext +from edisgo.run.registry import known_tasks, register_task +from edisgo.run.runner import _run_pipeline_on, run_edisgo + +__all__ = [ + "RunContext", + "_run_pipeline_on", + "known_tasks", + "register_task", + "run_edisgo", +] diff --git a/edisgo/run/config.py b/edisgo/run/config.py new file mode 100644 index 000000000..5c4ee8573 --- /dev/null +++ b/edisgo/run/config.py @@ -0,0 +1,410 @@ +""" +Config loader and schema normalizer for the eDisGo pipeline runner. + +The loader turns a YAML file, JSON file, or Python dict into the +canonical internal schema consumed by :mod:`edisgo.run.runner`. It +handles four concerns in a fixed order: + +1. **Read** — parse YAML/JSON (auto-detected by extension; unknown + extensions are tried as JSON first, then YAML). +2. **extends** — resolve a ``extends:`` key recursively into the + parent config and deep-merge; the child overrides parent keys. The + ``extends:`` value may be a path (relative to the including file) + or a bare preset name (resolved against + :mod:`edisgo.run.presets`). +3. **external_config** — merge machine-specific overrides from an + ``external_config:`` path (typically ``~/.edisgo/secrets.json`` + with DB credentials). Keys in the external file override keys in + the main config. +4. **eGo-legacy adaptation** — if the config looks like an eGo + ``scenario_setting_*.json`` (has top-level ``eDisGo.tasks``), map + it onto the new schema so old eGo configs run unchanged. +5. **Stage normalization** — collapse a flat ``pipeline:`` into a + single-stage ``stages: [{name: main, pipeline: [...]}]`` so the + runner only ever deals with the stage form. + +Only :func:`load_config` is public. Everything else is implementation +detail. +""" +from __future__ import annotations + +import copy +import json +import logging +import os + +from pathlib import Path +from typing import Any + +import yaml + +logger = logging.getLogger("edisgo.run.config") + + +def load_config(cfg_or_path) -> dict[str, Any]: + """ + Load, merge, adapt, and normalize a pipeline config. + + Accepts a path to a YAML/JSON file or a dict. The returned dict + always has the normalized shape expected by the runner: + + * top-level ``stages`` (list of ``{name, pipeline, ...}``) + * ``scenario`` (may be ``None``) + * optional ``grid``, ``database``, ``results`` sections + * no ``pipeline``, ``extends``, or ``external_config`` keys + (they have been consumed) + + Parameters + ---------- + cfg_or_path : str, pathlib.Path, or dict + Either a path to a YAML/JSON config file, or a dict already + holding the config. A dict is deep-copied so the caller's + dict is not mutated. + + Returns + ------- + dict + The fully resolved, normalized config. + + Raises + ------ + FileNotFoundError + If the given path (or an ``extends`` reference) does not + exist. + ValueError + If the config has both ``pipeline`` and ``stages``, missing + ``pipeline``/``stages``, duplicate stage names, or a stage + without ``name``/``pipeline``. + + """ + if isinstance(cfg_or_path, (dict,)): + cfg = copy.deepcopy(cfg_or_path) + base_dir = Path.cwd() + else: + path = Path(cfg_or_path).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f"Config file not found: {path}") + cfg = _read_file(path) + base_dir = path.parent + + cfg = _resolve_extends(cfg, base_dir) + cfg = _apply_external_config(cfg) + cfg = _adapt_ego_legacy(cfg) + cfg = _normalize_stages(cfg) + return cfg + + +def _read_file(path: Path) -> dict[str, Any]: + """ + Parse a YAML or JSON file into a dict. + + Parameters + ---------- + path : pathlib.Path + File path. Extension (``.json``, ``.yaml``, ``.yml``) selects + the parser. Unknown extensions fall back to JSON first, then + YAML. + + Returns + ------- + dict + Parsed config contents. + + """ + text = path.read_text() + suffix = path.suffix.lower() + if suffix == ".json": + return json.loads(text) + if suffix in (".yaml", ".yml"): + return yaml.safe_load(text) + try: + return json.loads(text) + except json.JSONDecodeError: + return yaml.safe_load(text) + + +def _resolve_extends(cfg: dict, base_dir: Path) -> dict: + """ + Resolve an ``extends:`` reference and deep-merge parent into child. + + The parent is loaded recursively, so a chain of ``extends:`` works. + References are looked up as (1) a bundled preset name under + :mod:`edisgo.run.presets`, (2) a path relative to ``base_dir``. + The child's keys override the parent's on conflicts. + + Parameters + ---------- + cfg : dict + Child config (may contain ``extends:``). + base_dir : pathlib.Path + Directory against which relative ``extends`` paths are + resolved (usually the directory of the child config). + + Returns + ------- + dict + Merged config with ``extends`` consumed. + + Raises + ------ + FileNotFoundError + If the referenced parent config file does not exist. + + """ + ext = cfg.pop("extends", None) + if ext is None: + return cfg + ext_path = Path(ext).expanduser() + if not ext_path.is_absolute(): + preset_path = _preset_path(str(ext_path)) + if preset_path is not None: + ext_path = preset_path + else: + ext_path = (base_dir / ext_path).resolve() + if not ext_path.is_file(): + raise FileNotFoundError(f"extends: file not found: {ext_path}") + parent = _read_file(ext_path) + parent = _resolve_extends(parent, ext_path.parent) + return _deep_merge(parent, cfg) + + +def _preset_path(name: str) -> Path | None: + """ + Look up a preset YAML/JSON by bare name. + + Searches the ``edisgo/run/presets/`` directory for a file matching + ``name``, ``name.yaml``, ``name.yml``, or ``name.json`` (in that + order). + + Parameters + ---------- + name : str + Preset identifier, e.g. ``"uc2_flex_opf"`` or + ``"presets/uc2_flex_opf.yaml"``. + + Returns + ------- + pathlib.Path or None + The resolved preset path, or ``None`` if no match is found. + + """ + presets_dir = Path(__file__).parent / "presets" + candidates = [ + presets_dir / name, + presets_dir / f"{name}.yaml", + presets_dir / f"{name}.yml", + presets_dir / f"{name}.json", + ] + for c in candidates: + if c.is_file(): + return c + return None + + +def _apply_external_config(cfg: dict) -> dict: + """ + Merge an ``external_config:`` file on top of the current config. + + Used to keep machine-specific secrets (DB credentials, result + directories) out of versioned scenario configs. If the referenced + file does not exist, a warning is logged but the config is used + as-is. + + Parameters + ---------- + cfg : dict + Config possibly containing an ``external_config:`` key. + + Returns + ------- + dict + Merged config with ``external_config`` consumed. + + """ + ext = cfg.pop("external_config", None) + if ext is None: + return cfg + path = Path(os.path.expanduser(ext)) + if not path.is_file(): + logger.warning(f"external_config file not found, skipping: {path}") + return cfg + override = _read_file(path) + return _deep_merge(cfg, override) + + +def _deep_merge(base: dict, override: dict) -> dict: + """ + Recursively merge two dicts, with ``override`` winning on conflicts. + + Nested dicts are merged key-by-key. Non-dict values (including + lists) are replaced wholesale — lists are NOT concatenated, to + keep the merge semantics predictable (otherwise a preset could + silently extend the child's pipeline). + + Parameters + ---------- + base : dict + Parent / lower-priority dict. + override : dict + Child / higher-priority dict. + + Returns + ------- + dict + A new dict holding the merge result. Inputs are not mutated. + + """ + out = copy.deepcopy(base) if base else {} + for key, val in (override or {}).items(): + if ( + key in out + and isinstance(out[key], dict) + and isinstance(val, dict) + ): + out[key] = _deep_merge(out[key], val) + else: + out[key] = copy.deepcopy(val) + return out + + +def _normalize_stages(cfg: dict) -> dict: + """ + Collapse a flat ``pipeline:`` into the canonical ``stages`` shape. + + After this step the runner only has to iterate ``cfg["stages"]``; + flat configs become a single stage named ``main``. + + Parameters + ---------- + cfg : dict + Config with either ``pipeline`` or ``stages`` at the top + level. + + Returns + ------- + dict + Config with ``stages`` guaranteed to be present and + ``pipeline`` removed. + + Raises + ------ + ValueError + If both ``pipeline`` and ``stages`` are present, if neither + is present, if any stage is missing ``name``/``pipeline``, or + if stage names are not unique. + + """ + if "stages" in cfg and "pipeline" in cfg: + raise ValueError( + "Config has both top-level 'pipeline' and 'stages'. " + "Use only one." + ) + if "stages" not in cfg: + pipeline = cfg.pop("pipeline", None) + if pipeline is None: + raise ValueError( + "Config must define either 'pipeline' or 'stages'." + ) + cfg["stages"] = [{"name": "main", "pipeline": pipeline}] + + seen = set() + for stage in cfg["stages"]: + if "name" not in stage: + raise ValueError("Every stage needs a 'name' key.") + if stage["name"] in seen: + raise ValueError( + f"Duplicate stage name: {stage['name']}" + ) + seen.add(stage["name"]) + if "pipeline" not in stage: + raise ValueError( + f"Stage '{stage['name']}' is missing 'pipeline'." + ) + return cfg + + +_EGO_TASK_MAP = { + "1_setup_grid": "setup_grid", + "5_grid_reinforcement": "reinforce", + "4_optimisation": "optimize", + "worst_case_ts": "worst_case_ts", + "base_reinforce": "base_reinforce", + "oedb_ts": "oedb_ts", + "import_heat_pumps_from_db": "import_heat_pumps", + "import_home_batteries_from_db": "import_home_batteries", + "import_dsm_from_db": "import_dsm", + "import_electromobility_from_db": "import_electromobility", + "load_charging_from_files": "load_charging_from_files", + "load_from_base": "load_from_base", +} +"""Mapping from eGo task names to edisgo.run task names. eGo-specific +tasks with no eDisGo equivalent (e.g. ``2_specs_overlying_grid``, +``3_temporal_complexity_reduction``) are intentionally missing — they +require eTraGo and are logged as "skipped" when adapted.""" + + +def _adapt_ego_legacy(cfg: dict) -> dict: + """ + Map an eGo-style ``scenario_setting_*.json`` onto the new schema. + + Recognizes an eGo config by the presence of an ``eDisGo.tasks`` + key at the top level together with the absence of + ``pipeline``/``stages``. Translates: + + * ``eDisGo.grid_path`` → ``grid.ding0_path`` + * ``eDisGo.results`` → ``results.directory`` + * ``eTraGo.scn_name`` → ``scenario`` + * ``eDisGo.tasks`` → ``pipeline`` (via :data:`_EGO_TASK_MAP`) + * top-level ``database``/``ssh`` kept under ``database`` + + eGo-only tasks (overlying grid / temporal reduction) are + dropped with a warning. Cosmetic keys (``eGo``, ``eTraGo``, + ``_comment``, ``_workflow``) are stripped. + + Parameters + ---------- + cfg : dict + Possibly-legacy config. + + Returns + ------- + dict + Adapted config. If the input is not an eGo-legacy config, it + is returned unchanged. + + """ + if "eDisGo" not in cfg or "pipeline" in cfg or "stages" in cfg: + return cfg + + edisgo_cfg = cfg["eDisGo"] + tasks = edisgo_cfg.get("tasks") + if tasks is None: + return cfg + + logger.info( + "Detected legacy eGo config schema — adapting to edisgo.run." + ) + mapped = [] + for t in tasks: + if t not in _EGO_TASK_MAP: + logger.warning( + f"eGo task '{t}' has no eDisGo equivalent — skipping " + "(likely eTraGo-specific)." + ) + continue + mapped.append(_EGO_TASK_MAP[t]) + + adapted: dict[str, Any] = { + "scenario": cfg.get("eTraGo", {}).get("scn_name", "eGon2035"), + "grid": {"ding0_path": edisgo_cfg.get("grid_path")}, + "results": {"directory": edisgo_cfg.get("results")}, + "pipeline": mapped, + } + if "database" in cfg: + adapted["database"] = cfg["database"] + if "ssh" in cfg: + adapted["database"]["ssh"] = cfg["ssh"] + for side_key in ("eGo", "eTraGo", "ssh", "_comment", "_workflow"): + cfg.pop(side_key, None) + cfg.pop("eDisGo", None) + return _deep_merge(adapted, cfg) diff --git a/edisgo/run/context.py b/edisgo/run/context.py new file mode 100644 index 000000000..c2fbce234 --- /dev/null +++ b/edisgo/run/context.py @@ -0,0 +1,115 @@ +""" +Runtime context passed to every task during pipeline execution. + +The context is a small mutable object that threads shared state between +tasks without polluting the :class:`~edisgo.EDisGo` instance itself. +Typical uses: + +* ``scenario`` — the active eGon scenario name (``eGon2035``, + ``eGon100RE``, …) so tasks don't have to re-read it from the config. +* ``engine`` — a SQLAlchemy engine, lazily created on first DB access + via :meth:`RunContext.ensure_engine`. Tasks that don't touch the + database never pay connection cost. +* ``results_dir`` — base directory for stage artifacts and ``save``. +* ``flags`` — free-form boolean/state flags tasks set to coordinate + with each other (``has_heat_pumps``, ``timeseries_set``, …). +* ``stage_artifacts`` — map ``stage_name -> path`` of zip/dir artifacts + emitted by ``save``, consumed by later stages via ``load_from``. + +Tasks should treat ``flags`` as advisory — they MAY short-circuit based +on a flag but MUST NOT assume a flag is present. +""" +from __future__ import annotations + +import logging + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass +class RunContext: + """ + Mutable per-run state shared across all tasks of a pipeline. + + Attributes + ---------- + scenario : str or None + Active scenario name from the top-level ``scenario:`` key. + engine : sqlalchemy.engine.Engine or None + Database engine for oedb-backed imports. Created lazily; + see :meth:`ensure_engine`. + results_dir : pathlib.Path or None + Base directory for stage outputs. Resolved from + ``results.directory`` in the config. + logger : logging.Logger + Logger instance used by tasks and the runner. Defaults to + the ``edisgo.run`` logger. + flags : dict + Free-form state flags that tasks use to communicate. Common + keys: ``grid_loaded``, ``timeseries_set``, + ``reactive_power_set``, ``has_heat_pumps``, ``has_dsm``, + ``has_home_batteries``, ``has_electromobility``, + ``base_reinforced``, ``last_saved``. + stage_artifacts : dict + Map ``stage_name -> Path`` of save-artifacts. Populated by the + ``save`` task when running inside a named stage, consumed by + subsequent stages that set ``load_from:``. + current_stage : str or None + Name of the stage currently executing. Set by the runner. + raw_config : dict + The fully resolved pipeline config (after ``extends``, + ``external_config``, and eGo-legacy adaptation). Tasks can + read supplementary keys like ``database.*`` from here. + + """ + + scenario: str | None = None + engine: Any = None + results_dir: Path | None = None + logger: logging.Logger = field( + default_factory=lambda: logging.getLogger("edisgo.run") + ) + flags: dict[str, Any] = field(default_factory=dict) + stage_artifacts: dict[str, Path] = field(default_factory=dict) + current_stage: str | None = None + raw_config: dict[str, Any] = field(default_factory=dict) + + def ensure_engine(self): + """ + Return a database engine, creating it on first call. + + Reads the ``database`` section of :attr:`raw_config` and calls + :func:`edisgo.io.db.engine`. Caches the engine on the context + so subsequent calls reuse the same connection. + + Returns + ------- + sqlalchemy.engine.Engine + The active database engine. + + Raises + ------ + RuntimeError + If the config has no ``database`` section — indicates the + pipeline wants to reach the database without configuring + it. + + """ + if self.engine is not None: + return self.engine + db_cfg = self.raw_config.get("database") + if not db_cfg: + raise RuntimeError( + "Task needs a database engine but no 'database' section " + "is configured." + ) + from edisgo.io.db import engine as egon_engine + + ssh_cfg = db_cfg.get("ssh") or {} + self.engine = egon_engine( + path=db_cfg.get("credentials_path"), + ssh=bool(ssh_cfg.get("enabled", False)), + ) + return self.engine diff --git a/edisgo/run/presets/basic.yaml b/edisgo/run/presets/basic.yaml new file mode 100644 index 000000000..136161855 --- /dev/null +++ b/edisgo/run/presets/basic.yaml @@ -0,0 +1,22 @@ +_comment: | + Basic preset: worst-case pre-reinforce → reinforce. + Minimal end-to-end example with no database dependency. + Reproduces the core of example_01 without flex imports. + +_workflow: + - setup_grid: load ding0 topology + - worst_case_ts: set worst-case time series (feed-in + load) + - reactive_power: fix reactive power control + - check_integrity: validate grid consistency + - reinforce: run grid reinforcement + - save: persist topology + timeseries + results + +scenario: eGon2035 + +pipeline: + - setup_grid + - worst_case_ts + - reactive_power + - check_integrity + - reinforce + - save diff --git a/edisgo/run/presets/r4mu_base_and_scenario.yaml b/edisgo/run/presets/r4mu_base_and_scenario.yaml new file mode 100644 index 000000000..6412f36ca --- /dev/null +++ b/edisgo/run/presets/r4mu_base_and_scenario.yaml @@ -0,0 +1,49 @@ +_comment: | + R4MU — two-stage base + scenario reinforcement: + Stage 1 produces a base-reinforced grid (generators + heat pumps) + and saves it as an artifact. Stage 2 loads that artifact, integrates + scenario-specific charging stations from a GeoPackage/CSV directory, + applies worst-case time series, and runs a scenario-specific + reinforce. Cost delta = extra reinforcement caused by the charging + scenario. + +_workflow: + - stage base: + - setup_grid: load ding0 topology + import generators + - import_heat_pumps: from egon_data + - worst_case_ts + - reactive_power + - reinforce + - save (artifact consumed by next stage) + - stage scenario: + - load_from: base + - load_charging_from_files: integrate scenario charging + - worst_case_ts + - reactive_power + - reinforce (delta only) + - save + +scenario: eGon2035 + +stages: + - name: base + pipeline: + - setup_grid: {import_generators: true} + - import_heat_pumps + - worst_case_ts + - reactive_power + - reinforce + - save + - name: scenario + load_from: base + params: + charging_dir: "./charging_scenario_1" + mv_threshold_kw: 100 + pipeline: + - load_charging_from_files: + charging_dir: "{{params.charging_dir}}" + mv_threshold_kw: "{{params.mv_threshold_kw}}" + - worst_case_ts + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc1_loads_worst_case.yaml b/edisgo/run/presets/uc1_loads_worst_case.yaml new file mode 100644 index 000000000..2204d3ce9 --- /dev/null +++ b/edisgo/run/presets/uc1_loads_worst_case.yaml @@ -0,0 +1,32 @@ +_comment: | + UC1 — worst-case flexibility loads: + load grid, base-reinforce (generators only), then import flex assets + (heat pumps, home batteries, DSM, electromobility) and apply worst-case + time series before a final reinforce. Cost delta = extra reinforcement + caused by the new assets under worst-case conditions. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging) + - worst_case_ts: synthetic worst case incl. new assets + - reactive_power: fix reactive power control + - reinforce: final reinforcement — delta only + - save: persist topology + results + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - worst_case_ts + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc2_flex_opf.yaml b/edisgo/run/presets/uc2_flex_opf.yaml new file mode 100644 index 000000000..c09cae87e --- /dev/null +++ b/edisgo/run/presets/uc2_flex_opf.yaml @@ -0,0 +1,43 @@ +_comment: | + UC2 — OPF with full flexibility: + Like UC1 but loads real egon_data time series (oedb) and runs a + powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) + before the final reinforce. Cost delta = extra reinforcement needed + under optimal flex dispatch. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (168 h, 2035) + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - reactive_power + - check_integrity + - optimize: pm_optimize with flex assets (SOC, opf v2) + - reinforce: final reinforcement + - save + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - oedb_ts: + timeindex: {start: "2035-01-01", periods: 168, freq: h} + dispatchable: {other: 0.7} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - check_integrity + - optimize: + flexible: [heat_pumps, storage] + method: soc + opf_version: 2 + - reinforce + - save diff --git a/edisgo/run/presets/uc3_oedb_ts.yaml b/edisgo/run/presets/uc3_oedb_ts.yaml new file mode 100644 index 000000000..59c184cdd --- /dev/null +++ b/edisgo/run/presets/uc3_oedb_ts.yaml @@ -0,0 +1,36 @@ +_comment: | + UC3 — real-world time series without OPF: + Like UC1 but uses real egon_data time series (oedb) instead of + synthetic worst cases. No optimization, no eTraGo. Difference to + UC1 is the data source for the final TS; difference to UC2 is no + OPF. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging) + - oedb_ts: real egon_data time series + - apply_heat_pump_strategy: uncontrolled + - reactive_power + - reinforce: final reinforcement + - save + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - oedb_ts: + timeindex: {start: "2035-01-01", periods: 168, freq: h} + dispatchable: {other: 0.7} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - reinforce + - save diff --git a/edisgo/run/registry.py b/edisgo/run/registry.py new file mode 100644 index 000000000..8aed4f3f5 --- /dev/null +++ b/edisgo/run/registry.py @@ -0,0 +1,113 @@ +""" +Task registry for the eDisGo pipeline runner. + +This module holds the global, process-wide mapping of task names to task +functions. Tasks are registered via the :func:`register_task` decorator +and looked up by name at pipeline execution time by the runner. Keeping +the registry separate from both the runner and the task implementations +lets external projects add their own tasks without patching eDisGo — +just import ``register_task`` and decorate a function. + +Registered tasks all share the signature ``(edisgo, ctx, **params)`` +where ``edisgo`` is the current :class:`~edisgo.EDisGo` instance (or +``None`` before it has been created by the first task), ``ctx`` is a +:class:`~edisgo.run.context.RunContext`, and ``**params`` are the +parameters passed from the YAML/JSON step definition. A task may return +an updated ``edisgo`` object (e.g. ``setup_grid`` creates it, ``load_*`` +replaces it); otherwise the runner keeps using the same instance. +""" +from __future__ import annotations + +from typing import Callable + +_TASKS: dict[str, Callable] = {} + + +def register_task(name: str) -> Callable[[Callable], Callable]: + """ + Decorator to register a task function under the given name. + + The decorated function becomes addressable from YAML/JSON pipelines + as either a plain string ``name`` or a single-key mapping + ``name: {param: value, ...}``. The name must be unique globally — + re-registering raises :class:`ValueError` to prevent silent + overrides across plugins. + + Parameters + ---------- + name : str + Unique task name used in pipeline definitions. + + Returns + ------- + Callable + A decorator that registers ``fn`` and returns it unchanged. + + Raises + ------ + ValueError + If ``name`` is already registered. + + Examples + -------- + >>> @register_task("set_timeindex_weekly") + ... def task_weekly(edisgo, ctx, *, start): + ... import pandas as pd + ... edisgo.set_timeindex(pd.date_range(start, periods=168, freq="h")) + + """ + def deco(fn: Callable) -> Callable: + if name in _TASKS: + raise ValueError( + f"Task '{name}' is already registered " + f"(existing={_TASKS[name].__qualname__}, " + f"new={fn.__qualname__})." + ) + _TASKS[name] = fn + return fn + + return deco + + +def get_task(name: str) -> Callable: + """ + Look up a registered task function by name. + + Parameters + ---------- + name : str + Task name as used in pipeline definitions. + + Returns + ------- + Callable + The task function registered under ``name``. + + Raises + ------ + KeyError + If ``name`` is not registered. The error message lists all + known task names to aid typo debugging. + + """ + if name not in _TASKS: + raise KeyError( + f"Unknown task: '{name}'. Known tasks: {sorted(_TASKS)}" + ) + return _TASKS[name] + + +def known_tasks() -> list[str]: + """ + Return a sorted list of all registered task names. + + Useful for error messages, CLI completion, and tests that assert + core tasks exist. + + Returns + ------- + list of str + All registered task names in alphabetical order. + + """ + return sorted(_TASKS) diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py new file mode 100644 index 000000000..63f30aa07 --- /dev/null +++ b/edisgo/run/runner.py @@ -0,0 +1,261 @@ +""" +Pipeline execution engine for the eDisGo runner. + +This module ties the other three pieces — :mod:`edisgo.run.config` +(loader), :mod:`edisgo.run.validator` (static checks), and +:mod:`edisgo.run.registry` (task lookup) — together into a linear +stage-by-stage executor. + +The execution model: + +1. Load and validate the config. +2. Build a :class:`~edisgo.run.context.RunContext`. +3. For each stage, if the stage declares ``load_from: X``, reload + the EDisGo object from stage ``X``'s save-artifact (topology + + results only; time series are dropped to let the new stage set + fresh ones). +4. For each step in the stage's pipeline, look up the task function + in the registry and call it with the current EDisGo object and + the context. A task may return a new EDisGo object (``setup_grid``, + ``load_from_base``) which then replaces the current one. +5. Repeat for all stages, finally return the EDisGo object. + +Two entry points are exposed: + +* :func:`run_edisgo` — starts from no EDisGo object; the first task + must create one (usually ``setup_grid``). +* :func:`_run_pipeline_on` — starts from an existing EDisGo instance; + used by :meth:`edisgo.EDisGo.run_pipeline`. +""" +from __future__ import annotations + +import logging + +from pathlib import Path +from typing import Any + +from edisgo.run import tasks as _tasks # noqa: F401 — triggers registration +from edisgo.run.config import load_config +from edisgo.run.context import RunContext +from edisgo.run.registry import get_task +from edisgo.run.validator import _split_step, validate + +logger = logging.getLogger("edisgo.run.runner") + + +def run_edisgo(config) -> Any: + """ + Run an eDisGo pipeline from a YAML/JSON config or dict. + + This is the standalone entry point. The pipeline's first task is + typically ``setup_grid`` or ``load_from_base`` to bootstrap the + :class:`~edisgo.EDisGo` instance. If you already have one, + prefer :meth:`edisgo.EDisGo.run_pipeline` instead. + + Parameters + ---------- + config : str, pathlib.Path, or dict + Path to a YAML/JSON pipeline config, or an in-memory dict of + the same shape. + + Returns + ------- + :class:`~edisgo.EDisGo` + The EDisGo instance after the last stage has run. For + multi-stage configs this is the object produced by the final + stage. + + """ + return _run_pipeline_on(None, config) + + +def _run_pipeline_on(edisgo, config): + """ + Internal runner shared by :func:`run_edisgo` and the EDisGo method. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Existing EDisGo instance to operate on, or ``None`` to have + the first task create one. + config : str, pathlib.Path, or dict + Config to execute. Passed through to + :func:`edisgo.run.config.load_config`. + + Returns + ------- + edisgo.EDisGo + The final EDisGo instance. + + Raises + ------ + RuntimeError + If a stage declares ``load_from: X`` but ``X`` produced no + artifact (typically because validate() was skipped). + + """ + cfg = load_config(config) + validate(cfg) + ctx = _build_context(cfg) + + for stage in cfg["stages"]: + ctx.current_stage = stage["name"] + ctx.logger.info(f"=== stage '{stage['name']}' ===") + + load_from = stage.get("load_from") + if load_from is not None: + artifact = ctx.stage_artifacts.get(load_from) + if artifact is None: + raise RuntimeError( + f"Stage '{stage['name']}' wants to load from " + f"'{load_from}' but no artifact is registered." + ) + edisgo = _load_artifact(str(artifact)) + + params = stage.get("params", {}) or {} + for step in stage["pipeline"]: + name, step_params = _split_step(step) + step_params = _resolve_templating(step_params, params) + ctx.logger.info(f" -> task '{name}'") + task_fn = get_task(name) + result = task_fn(edisgo, ctx, **step_params) + if result is not None: + edisgo = result + + return edisgo + + +def _build_context(cfg: dict) -> RunContext: + """ + Build a :class:`~edisgo.run.context.RunContext` from a config. + + Wires ``scenario`` and ``results.directory`` into the context and + stores the full config under :attr:`RunContext.raw_config` so + tasks can read supplementary sections. + + Parameters + ---------- + cfg : dict + Normalized config. + + Returns + ------- + RunContext + Initialized context with no engine, no artifacts, empty flags. + + """ + results_cfg = cfg.get("results") or {} + results_dir = results_cfg.get("directory") + return RunContext( + scenario=cfg.get("scenario"), + results_dir=Path(results_dir) if results_dir else None, + raw_config=cfg, + ) + + +def _load_artifact(path: str): + """ + Reload an EDisGo instance from a save-artifact for a ``load_from``. + + Loads topology + results only; time series and flex data are + dropped so the consuming stage can set them fresh. Equipment + changes are reset so the next stage's reinforce accounts only + for its own scenario. + + Parameters + ---------- + path : str + Path to a directory or ``.zip`` produced by the ``save`` + task. + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + from_zip = path.endswith(".zip") + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=False, + import_results=True, + import_electromobility=False, + import_heat_pump=False, + import_dsm=False, + import_overlying_grid=False, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + edisgo.results.equipment_changes = pd.DataFrame() + return edisgo + + +def _resolve_templating(step_params: dict, stage_params: dict) -> dict: + """ + Substitute ``{{params.x}}`` placeholders in step parameters. + + Stage-level ``params:`` allows a preset to expose a few knobs that + individual step parameters can reference. Only simple + ``{{params.KEY}}`` expansions inside string values are supported + (no filters, no conditionals, no nested expressions) — deliberately + kept trivial to avoid a Jinja dependency. + + Parameters + ---------- + step_params : dict + Keyword arguments for a single step. + stage_params : dict + Stage-level ``params:`` dict. + + Returns + ------- + dict + ``step_params`` with template strings resolved. + + """ + if not stage_params or not step_params: + return step_params + out = {} + for k, v in step_params.items(): + if isinstance(v, str) and "{{" in v: + out[k] = _render_template(v, stage_params) + else: + out[k] = v + return out + + +def _render_template(s: str, stage_params: dict) -> str: + """ + Expand ``{{params.KEY}}`` references in a single string. + + Parameters + ---------- + s : str + Source string. + stage_params : dict + Mapping of stage-level parameters. + + Returns + ------- + str + Rendered string. Unknown keys are left in place (the original + placeholder remains) so downstream errors point at the + typo-ed key rather than silently turning into an empty + string. + + """ + import re + + def repl(match): + expr = match.group(1).strip() + if expr.startswith("params."): + key = expr.split(".", 1)[1] + return str(stage_params.get(key, match.group(0))) + return match.group(0) + + return re.sub(r"\{\{\s*([^}]+)\s*\}\}", repl, s) diff --git a/edisgo/run/tasks/__init__.py b/edisgo/run/tasks/__init__.py new file mode 100644 index 000000000..0d59ea02a --- /dev/null +++ b/edisgo/run/tasks/__init__.py @@ -0,0 +1,25 @@ +""" +Task implementations for the eDisGo pipeline runner. + +Importing this package as a side effect registers every task defined +in its submodules with :func:`edisgo.run.registry.register_task`, so +that the runner sees them at execution time. The submodules are: + +* :mod:`.grid` — ``setup_grid``, ``load_from_base`` +* :mod:`.timeseries` — ``worst_case_ts``, ``oedb_ts``, ``manual_ts``, + ``set_timeindex``, ``reactive_power`` +* :mod:`.flex` — flex imports + (``import_heat_pumps``, ``import_home_batteries``, ``import_dsm``, + ``import_electromobility``, ``import_generators``) and operating + strategies (``apply_charging_strategy``, + ``apply_heat_pump_strategy``) +* :mod:`.analysis` — ``check_integrity``, ``analyze``, ``reinforce``, + ``base_reinforce``, ``optimize`` +* :mod:`.io` — ``save``, ``load_charging_from_files`` + +Task signature convention: ``(edisgo, ctx, **params)``. A task may +mutate ``edisgo`` in place and/or return a new EDisGo instance (the +returned value, if non-None, replaces the current one in the runner's +loop). +""" +from edisgo.run.tasks import analysis, flex, grid, io, timeseries # noqa: F401 diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py new file mode 100644 index 000000000..3f604914e --- /dev/null +++ b/edisgo/run/tasks/io.py @@ -0,0 +1,179 @@ +""" +Input/output tasks — persisting results and ingesting external files. + +* :func:`task_save` (``save``) — persist topology, time series, and + results to disk (directory or zip). Also publishes the artifact + path into ``ctx.stage_artifacts`` so a later stage can + ``load_from:``. +* :func:`task_load_charging_from_files` + (``load_charging_from_files``) — R4MU-specific placeholder for + integrating scenario charging stations from a directory of CSV / + GeoPackage files; implementation is deferred until needed. +""" +from __future__ import annotations + +import os + +from edisgo.run.registry import register_task + + +@register_task("save") +def task_save(edisgo, ctx, *, directory=None, save_topology=True, + save_timeseries=True, save_results=True, + save_electromobility=None, save_opf_results=False, + save_heatpump=None, save_overlying_grid=False, + save_dsm=None, archive=False, archive_type="zip", + reduce_memory=False, parameters=None): + """ + Save the current EDisGo state to disk. + + If ``directory`` is not given, the artifact is written under + ``ctx.results_dir / `` so every stage gets its own + subdirectory. When ``archive=True`` the result is a single zip; + the artifact path (including ``.zip``) is recorded in + ``ctx.stage_artifacts[]`` so a downstream stage can + declare ``load_from: ``. + + Flags drive smart defaults for the optional ``save_*`` switches: + if flex data is absent (per ``ctx.flags``), saving it is skipped. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to persist. + ctx : RunContext + Run context. Uses ``ctx.results_dir``, ``ctx.current_stage``, + and reads ``has_heat_pumps`` / ``has_dsm`` / + ``has_electromobility`` flags. + directory : str, optional + Absolute target directory. If omitted, derived from + ``ctx.results_dir / ctx.current_stage``. + save_topology : bool, optional + Write the topology CSVs. Default ``True``. + save_timeseries : bool, optional + Write time-series CSVs. Default ``True``. + save_results : bool, optional + Write the results CSVs (equipment changes, expansion costs, + etc.). Default ``True``. + save_electromobility : bool or None, optional + If ``None``, auto-enabled iff + ``ctx.flags['has_electromobility']`` is truthy. + save_opf_results : bool, optional + Write OPF results if present. + save_heatpump : bool or None, optional + If ``None``, auto-enabled iff ``ctx.flags['has_heat_pumps']`` + is truthy. + save_overlying_grid : bool, optional + Write overlying-grid (eTraGo) specs if present. + save_dsm : bool or None, optional + If ``None``, auto-enabled iff ``ctx.flags['has_dsm']`` is + truthy. + archive : bool, optional + Pack the directory into a single ``.zip`` archive. + archive_type : str, optional + Archive format (currently only ``"zip"``). + reduce_memory : bool, optional + Downcast float time-series to ``float32`` to save disk. + parameters : dict, optional + Fine-grained selection of which results fields to write, + e.g. ``{"grid_expansion_results": ["equipment_changes"]}``. + + Returns + ------- + edisgo.EDisGo + The unchanged EDisGo instance. + + Raises + ------ + ValueError + If no ``directory`` is given and ``ctx.results_dir`` is also + unset. + + """ + if directory is None: + if ctx.results_dir is None: + raise ValueError( + "Task 'save' needs a 'directory' parameter or " + "config.results.directory." + ) + stage = ctx.current_stage or "main" + directory = os.path.join(str(ctx.results_dir), stage) + + if save_heatpump is None: + save_heatpump = ctx.flags.get("has_heat_pumps", False) + if save_dsm is None: + save_dsm = ctx.flags.get("has_dsm", False) + if save_electromobility is None: + save_electromobility = ctx.flags.get("has_electromobility", False) + + kwargs = dict( + directory=directory, + save_topology=save_topology, + save_timeseries=save_timeseries, + save_results=save_results, + save_electromobility=save_electromobility, + save_opf_results=save_opf_results, + save_heatpump=save_heatpump, + save_overlying_grid=save_overlying_grid, + save_dsm=save_dsm, + ) + if archive: + kwargs["archive"] = True + kwargs["archive_type"] = archive_type + if reduce_memory: + kwargs["reduce_memory"] = True + if parameters is not None: + kwargs["parameters"] = parameters + + edisgo.save(**kwargs) + + saved_path = directory + (".zip" if archive else "") + if ctx.current_stage: + ctx.stage_artifacts[ctx.current_stage] = saved_path + ctx.flags["last_saved"] = saved_path + return edisgo + + +@register_task("load_charging_from_files") +def task_load_charging_from_files(edisgo, ctx, *, charging_dir, + use_case_to_sector=None, + mv_threshold_kw=100.0): + """ + Integrate scenario charging stations from files (R4MU workflow). + + PLACEHOLDER — the full implementation lives in eGo's + ``_run_edisgo_task_load_charging_from_files`` and needs to be + ported when R4MU is prioritised. The eGo version reads a + GeoPackage / CSV of charging locations, filters by the MV grid + district geometry, and integrates them into the topology via + :func:`find_nearest_bus` / ``integrate_component_based_on_geolocation`` + with a use-case-to-sector mapping and an MV/LV connection + threshold. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + charging_dir : str + Directory containing the charging-station source files. + use_case_to_sector : dict, optional + Maps raw use-case labels (``"home_detached"`` etc.) to + eDisGo sector names (``"home"``, ``"work"``, …). + mv_threshold_kw : float, optional + Capacity threshold above which stations connect to an MV + bus; below connect to LV. + + Raises + ------ + NotImplementedError + Always — port the eGo implementation before using. + + """ + raise NotImplementedError( + "Task 'load_charging_from_files' is a placeholder port from " + "eGo R4MU. Port the logic from eGo's " + "_run_edisgo_task_load_charging_from_files when R4MU is " + "needed." + ) diff --git a/edisgo/run/validator.py b/edisgo/run/validator.py new file mode 100644 index 000000000..3989b8398 --- /dev/null +++ b/edisgo/run/validator.py @@ -0,0 +1,201 @@ +""" +Static validator for pipeline configs. + +The validator enforces structural and ordering rules that the runner +would otherwise hit at execution time — often after 20 minutes of work. +Running these checks up-front turns "cryptic AttributeError after half +the pipeline" into a clear ``ValueError`` at startup. + +Checked rules: + +* every step maps to a known, registered task name; +* ``reactive_power`` comes after every time-series task in a stage, + never before — ``set_time_series_reactive_power_control`` overwrites + reactive power on the currently set active-power time series; +* ``analyze`` and ``reinforce`` require a time-series task earlier in + the stage (or a ``load_from:`` that brings a prepared grid); +* ``optimize`` requires both a time-series task and at least one flex + import earlier in the stage — OPF without flexibility is meaningless; +* flex imports (``import_heat_pumps``, …) require a loaded grid, i.e. + an earlier ``setup_grid`` / ``load_from_base`` / a stage-level + ``load_from:``; +* ``base_reinforce`` likewise requires a loaded grid; +* a stage that declares ``load_from: X`` can only run if stage ``X`` + ran earlier AND contains a ``save`` step. +""" +from __future__ import annotations + +from typing import Any + +from edisgo.run.registry import known_tasks + +_TS_TASKS = {"worst_case_ts", "oedb_ts", "manual_ts", "set_timeindex"} +_GRID_CREATING_TASKS = {"setup_grid", "load_from_base"} +_FLEX_IMPORTS = { + "import_heat_pumps", + "import_home_batteries", + "import_dsm", + "import_electromobility", +} + + +def validate(cfg: dict) -> None: + """ + Validate a normalized pipeline config against the ordering rules. + + This function does not return a value. On success it simply + returns; on any rule violation it raises :class:`ValueError` with + a message identifying the offending stage and task. + + Parameters + ---------- + cfg : dict + Normalized config as returned by + :func:`edisgo.run.config.load_config`. Must have a ``stages`` + list at the top level. + + Raises + ------ + ValueError + If the config has no stages, an unknown task name, a + structural problem (reactive before TS, reinforce without TS, + optimize without flex, flex import without grid, …), or a + stage references a ``load_from`` source that doesn't exist or + has no ``save`` step. + + """ + stages = cfg.get("stages") or [] + if not stages: + raise ValueError("Config has no stages to run.") + + available_artifacts: set[str] = set() + + for stage in stages: + name = stage["name"] + pipeline = stage.get("pipeline") or [] + load_from = stage.get("load_from") + + if load_from is not None and load_from not in available_artifacts: + raise ValueError( + f"Stage '{name}' requires 'load_from: {load_from}' but " + f"that stage has not run or did not save. Available: " + f"{sorted(available_artifacts)}" + ) + + grid_available = load_from is not None + ts_set = False + reactive_set = False + flex_imported = False + has_save = False + + for step in pipeline: + task_name, _params = _split_step(step) + if task_name not in known_tasks(): + raise ValueError( + f"Unknown task '{task_name}' in stage '{name}'. " + f"Known: {known_tasks()}" + ) + + if task_name in _GRID_CREATING_TASKS: + grid_available = True + if task_name in _TS_TASKS: + if reactive_set: + raise ValueError( + f"Stage '{name}': time-series task " + f"'{task_name}' comes after 'reactive_power' " + f"— reactive_power must be the last " + f"time-series-altering step." + ) + ts_set = True + if task_name == "reactive_power": + reactive_set = True + if task_name in _FLEX_IMPORTS: + flex_imported = True + if not grid_available: + raise ValueError( + f"Stage '{name}': task '{task_name}' requires " + f"a loaded grid (setup_grid or " + f"load_from_base) before it." + ) + if task_name in {"analyze", "reinforce"} and not ( + ts_set or load_from + ): + raise ValueError( + f"Stage '{name}': task '{task_name}' requires time " + f"series to be set (e.g. worst_case_ts or " + f"oedb_ts) before it." + ) + if task_name == "optimize": + if not ts_set and not load_from: + raise ValueError( + f"Stage '{name}': 'optimize' requires time " + f"series." + ) + if not flex_imported and not load_from: + raise ValueError( + f"Stage '{name}': 'optimize' requires at least " + f"one flex asset to be imported." + ) + if task_name == "base_reinforce" and not grid_available: + raise ValueError( + f"Stage '{name}': 'base_reinforce' requires a " + f"loaded grid before it." + ) + if task_name == "save": + has_save = True + + if has_save: + available_artifacts.add(name) + + +def _split_step(step: Any) -> tuple[str, dict]: + """ + Normalize a pipeline step into ``(task_name, params)``. + + Steps are allowed in two forms in YAML/JSON: + + * bare string — ``worst_case_ts`` → ``("worst_case_ts", {})`` + * single-key mapping — + ``import_electromobility: {charging_strategy: dumb}`` + → ``("import_electromobility", {"charging_strategy": "dumb"})`` + + ``None`` as the parameter value is treated as an empty dict so + that YAML's ``task:`` (with nothing after the colon) works. + + Parameters + ---------- + step : str or dict + Raw step as it appears in the pipeline list. + + Returns + ------- + tuple of (str, dict) + The task name and its keyword arguments. + + Raises + ------ + ValueError + If ``step`` is not a string or a single-key mapping, or if + the parameter value is not a mapping. + + """ + if isinstance(step, str): + return step, {} + if isinstance(step, dict): + if len(step) != 1: + raise ValueError( + f"Task step must be a string or single-key mapping, " + f"got: {step}" + ) + (name, params), = step.items() + if params is None: + params = {} + if not isinstance(params, dict): + raise ValueError( + f"Parameters for task '{name}' must be a mapping, " + f"got: {type(params).__name__}" + ) + return name, params + raise ValueError( + f"Task step must be string or mapping, got: {step!r}" + ) diff --git a/setup.py b/setup.py index a07f355c3..706b047b8 100644 --- a/setup.py +++ b/setup.py @@ -100,6 +100,7 @@ def read(fname): "edisgo": [ os.path.join("config", "*.cfg"), os.path.join("equipment", "*.csv"), + os.path.join("run", "presets", "*.yaml"), ] }, ) diff --git a/tests/run/__init__.py b/tests/run/__init__.py new file mode 100644 index 000000000..baf15e08b --- /dev/null +++ b/tests/run/__init__.py @@ -0,0 +1 @@ +"""Tests for the :mod:`edisgo.run` pipeline runner.""" diff --git a/tests/run/test_config.py b/tests/run/test_config.py new file mode 100644 index 000000000..10b4ec474 --- /dev/null +++ b/tests/run/test_config.py @@ -0,0 +1,154 @@ +""" +Unit tests for :mod:`edisgo.run.config` — loader, merger, adapter. + +Covers YAML/JSON parity, ``extends`` resolution (preset-by-name and +relative paths), deep-merge semantics, stage normalization, and the +eGo-legacy adapter. +""" +import json + +import pytest +import yaml + +from edisgo.run.config import _deep_merge, load_config + + +def _write(tmp_path, name, data): + """ + Helper: write ``data`` to ``tmp_path/name`` as YAML or JSON. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest-provided temporary directory. + name : str + File name with extension (``.yaml``/``.yml``/``.json``). + data : dict + Payload. + + Returns + ------- + pathlib.Path + Path to the written file. + + """ + path = tmp_path / name + if name.endswith(".json"): + path.write_text(json.dumps(data)) + else: + path.write_text(yaml.safe_dump(data)) + return path + + +def test_load_flat_pipeline_normalized_to_stages(tmp_path): + """A flat ``pipeline:`` must normalize to a single 'main' stage.""" + p = _write(tmp_path, "cfg.yaml", { + "scenario": "eGon2035", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + }) + cfg = load_config(str(p)) + assert "pipeline" not in cfg + assert cfg["stages"] == [ + {"name": "main", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"]} + ] + + +def test_yaml_and_json_equivalent(tmp_path): + """YAML and JSON payloads with identical content must load equal.""" + data = { + "scenario": "eGon2035", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + } + yaml_path = _write(tmp_path, "cfg.yaml", data) + json_path = _write(tmp_path, "cfg.json", data) + assert load_config(str(yaml_path)) == load_config(str(json_path)) + + +def test_extends_merges_parent(tmp_path): + """Child config must deep-merge with its ``extends:`` parent.""" + parent = _write(tmp_path, "parent.yaml", { + "scenario": "eGon2035", + "grid": {"legacy_ding0_grids": False}, + "pipeline": ["setup_grid", "reinforce"], + }) + child = _write(tmp_path, "child.yaml", { + "extends": str(parent), + "grid": {"ding0_path": "/tmp/xyz"}, + }) + cfg = load_config(str(child)) + assert cfg["scenario"] == "eGon2035" + assert cfg["grid"] == { + "legacy_ding0_grids": False, "ding0_path": "/tmp/xyz" + } + assert cfg["stages"][0]["pipeline"] == ["setup_grid", "reinforce"] + + +def test_extends_preset_by_name(tmp_path): + """``extends: basic`` must resolve to the bundled basic preset.""" + child = _write(tmp_path, "child.yaml", { + "extends": "basic", + "grid": {"ding0_path": "/tmp/xyz"}, + }) + cfg = load_config(str(child)) + assert "stages" in cfg + assert cfg["grid"]["ding0_path"] == "/tmp/xyz" + + +def test_deep_merge_nested(): + """Nested dicts must be merged key-by-key, child wins on conflict.""" + base = {"a": {"b": 1, "c": 2}, "d": 4} + over = {"a": {"b": 99, "e": 5}} + merged = _deep_merge(base, over) + assert merged == {"a": {"b": 99, "c": 2, "e": 5}, "d": 4} + + +def test_both_pipeline_and_stages_rejected(tmp_path): + """Top-level ``pipeline`` and ``stages`` are mutually exclusive.""" + p = _write(tmp_path, "cfg.yaml", { + "pipeline": ["setup_grid"], + "stages": [{"name": "x", "pipeline": ["setup_grid"]}], + }) + with pytest.raises(ValueError, match="both"): + load_config(str(p)) + + +def test_duplicate_stage_names_rejected(tmp_path): + """Stage names must be unique; duplicates raise ValueError.""" + p = _write(tmp_path, "cfg.yaml", { + "stages": [ + {"name": "x", "pipeline": ["setup_grid"]}, + {"name": "x", "pipeline": ["reinforce"]}, + ], + }) + with pytest.raises(ValueError, match="Duplicate stage"): + load_config(str(p)) + + +def test_ego_legacy_adapter(tmp_path): + """An eGo ``scenario_setting_*.json`` must adapt to the new schema.""" + ego_cfg = { + "eGo": {"eDisGo": True}, + "eTraGo": {"scn_name": "eGon2035"}, + "eDisGo": { + "grid_path": "/some/path", + "results": "/tmp/results", + "tasks": [ + "1_setup_grid", + "base_reinforce", + "import_heat_pumps_from_db", + "worst_case_ts", + "5_grid_reinforcement", + ], + }, + "database": {"host": "localhost"}, + } + p = _write(tmp_path, "legacy.json", ego_cfg) + cfg = load_config(str(p)) + assert cfg["scenario"] == "eGon2035" + assert cfg["grid"]["ding0_path"] == "/some/path" + assert cfg["stages"][0]["pipeline"] == [ + "setup_grid", "base_reinforce", "import_heat_pumps", + "worst_case_ts", "reinforce", + ] + assert cfg["database"]["host"] == "localhost" diff --git a/tests/run/test_registry.py b/tests/run/test_registry.py new file mode 100644 index 000000000..a56070bf6 --- /dev/null +++ b/tests/run/test_registry.py @@ -0,0 +1,35 @@ +""" +Unit tests for :mod:`edisgo.run.registry`. + +Verifies that core tasks are discoverable, that ``get_task`` raises a +useful error on typos, and that duplicate registrations are rejected. +""" +import pytest + +from edisgo.run.registry import get_task, known_tasks, register_task + + +def test_known_tasks_contains_core(): + """All core task names must be registered on import.""" + tasks = known_tasks() + for core in ["setup_grid", "worst_case_ts", "reactive_power", + "reinforce", "analyze", "save"]: + assert core in tasks + + +def test_get_task_unknown_raises(): + """Unknown task names must surface as a descriptive KeyError.""" + with pytest.raises(KeyError, match="Unknown task"): + get_task("does_not_exist") + + +def test_register_task_duplicate_raises(): + """Registering the same task name twice is a bug — must raise.""" + @register_task("_test_task_for_dup_check") + def _a(edisgo, ctx): + """Marker task #1 — test fixture only.""" + + with pytest.raises(ValueError, match="already registered"): + @register_task("_test_task_for_dup_check") + def _b(edisgo, ctx): + """Marker task #2 — test fixture only, must not register.""" diff --git a/tests/run/test_runner.py b/tests/run/test_runner.py new file mode 100644 index 000000000..0ecb6d08e --- /dev/null +++ b/tests/run/test_runner.py @@ -0,0 +1,118 @@ +""" +End-to-end tests for the eDisGo pipeline runner. + +Uses the small test grid under ``tests/data/ding0_test_network_2`` +(exposed by :mod:`tests.conftest` as +``pytest.ding0_test_network_2_path``) to run full pipelines without +touching the database. Covers: + +* the standalone ``run_edisgo`` entry point with a flat pipeline, +* the instance method ``EDisGo.run_pipeline``, +* the stage mechanism with ``save`` + ``load_from``. +""" +import os + +import pytest + +from edisgo.run import run_edisgo + + +@pytest.fixture +def basic_cfg(tmp_path): + """ + Minimal end-to-end config fixture. + + Produces a config that loads the small ding0 test grid, sets + worst-case time series, fixes reactive power, checks integrity, + runs reinforcement, and saves — no database needed. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest-provided temp directory for the run's artifacts. + + Returns + ------- + dict + The config dict. + + """ + return { + "scenario": "eGon2035", + "grid": { + "ding0_path": pytest.ding0_test_network_2_path, + "legacy_ding0_grids": True, + }, + "results": {"directory": str(tmp_path)}, + "pipeline": [ + "setup_grid", + "worst_case_ts", + "reactive_power", + "check_integrity", + "reinforce", + "save", + ], + } + + +def test_runner_basic_end_to_end(basic_cfg): + """A flat-pipeline run must execute and persist the expected artifact.""" + edisgo = run_edisgo(basic_cfg) + assert edisgo is not None + assert edisgo.topology is not None + assert os.path.isdir(os.path.join(basic_cfg["results"]["directory"], + "main")) + + +def test_runner_method_on_edisgo(basic_cfg): + """``EDisGo.run_pipeline`` must operate on the existing instance.""" + from edisgo import EDisGo + + basic_cfg["pipeline"] = basic_cfg["pipeline"][1:] # skip setup_grid + edisgo = EDisGo( + ding0_grid=basic_cfg["grid"]["ding0_path"], + legacy_ding0_grids=True, + ) + edisgo = edisgo.run_pipeline(basic_cfg) + assert edisgo.topology is not None + + +def test_runner_two_stages_with_load_from(tmp_path): + """ + A two-stage run must save the first stage and reload it via + ``load_from`` in the second stage, producing both artifacts. + """ + cfg = { + "scenario": "eGon2035", + "grid": { + "ding0_path": pytest.ding0_test_network_2_path, + "legacy_ding0_grids": True, + }, + "results": {"directory": str(tmp_path)}, + "stages": [ + { + "name": "base", + "pipeline": [ + "setup_grid", + "worst_case_ts", + "reactive_power", + "reinforce", + {"save": {"archive": True}}, + ], + }, + { + "name": "scenario", + "load_from": "base", + "pipeline": [ + "worst_case_ts", + "reactive_power", + "reinforce", + "save", + ], + }, + ], + } + edisgo = run_edisgo(cfg) + assert edisgo.topology is not None + assert os.path.exists(os.path.join(str(tmp_path), "base.zip")) + assert os.path.isdir(os.path.join(str(tmp_path), "scenario")) diff --git a/tests/run/test_validator.py b/tests/run/test_validator.py new file mode 100644 index 000000000..4b86f40cf --- /dev/null +++ b/tests/run/test_validator.py @@ -0,0 +1,97 @@ +""" +Unit tests for :mod:`edisgo.run.validator`. + +Each test pins one ordering rule: reactive-before-TS, reinforce +without TS, optimize without flex, flex import without grid, and the +stage-level ``load_from`` constraints. +""" +import pytest + +from edisgo.run.validator import validate + + +def _wrap(pipeline): + """ + Wrap a flat pipeline into a single-stage config dict. + + Parameters + ---------- + pipeline : list + Ordered list of task names / single-key mappings. + + Returns + ------- + dict + Minimal config in the shape expected by :func:`validate`. + + """ + return {"stages": [{"name": "main", "pipeline": pipeline}]} + + +def test_valid_pipeline(): + """A well-formed pipeline must pass validation without raising.""" + validate(_wrap(["setup_grid", "worst_case_ts", "reactive_power", + "reinforce", "save"])) + + +def test_unknown_task_rejected(): + """Typo'd task names must be rejected.""" + with pytest.raises(ValueError, match="Unknown task"): + validate(_wrap(["setup_grid", "nonexistent_task"])) + + +def test_reactive_before_ts_rejected(): + """reactive_power before a TS task violates the ordering rule.""" + with pytest.raises(ValueError, match="reactive_power"): + validate(_wrap(["setup_grid", "reactive_power", "worst_case_ts"])) + + +def test_reinforce_without_ts_rejected(): + """reinforce without any prior time-series step must fail.""" + with pytest.raises(ValueError, match="time series"): + validate(_wrap(["setup_grid", "reinforce"])) + + +def test_optimize_without_flex_rejected(): + """optimize requires at least one flex asset to be imported.""" + with pytest.raises(ValueError, match="flex asset"): + validate(_wrap(["setup_grid", "worst_case_ts", "optimize"])) + + +def test_flex_import_before_grid_rejected(): + """Flex imports require a loaded grid — pre-loading is not enough.""" + with pytest.raises(ValueError, match="loaded grid"): + validate(_wrap(["import_heat_pumps", "worst_case_ts", "reinforce"])) + + +def test_stage_load_from_missing_rejected(): + """``load_from: X`` where X has not run must fail.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce"]}, + {"name": "b", "load_from": "nonexistent", + "pipeline": ["reinforce"]}, + ]} + with pytest.raises(ValueError, match="load_from"): + validate(cfg) + + +def test_stage_load_from_requires_save_in_source(): + """A stage consumed by ``load_from`` must itself end with ``save``.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce"]}, # no save + {"name": "b", "load_from": "a", "pipeline": ["reinforce"]}, + ]} + with pytest.raises(ValueError, match="load_from"): + validate(cfg) + + +def test_stage_load_from_with_save_ok(): + """Stage chain with a save in the source must validate successfully.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce", "save"]}, + {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, + ]} + validate(cfg) From 78d07bceab9ede3b0557f327c5daf87cfd44357f Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:18:51 +0200 Subject: [PATCH 02/66] Add example yaml for full example --- edisgo/run/presets/uc4_example_MS.yaml | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 edisgo/run/presets/uc4_example_MS.yaml diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example_MS.yaml new file mode 100644 index 000000000..a72573496 --- /dev/null +++ b/edisgo/run/presets/uc4_example_MS.yaml @@ -0,0 +1,56 @@ +_comment: | + UC3 — OPF with full flexibility: + Like UC1 but loads real egon_data time series (oedb) and runs a + powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) + before the final reinforce. Cost delta = extra reinforcement needed + under optimal flex dispatch. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_generators: from edon-data + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - reactive_power + - check_integrity + - optimize: pm_optimize with flex assets (SOC, opf v2) + - reinforce: final reinforcement + - save + +scenario: eGon2035 +grid: + ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + legacy_ding0_grids: false + +database: + ssh: + enabled: false + +timeindex: {start: "2035-01-01", periods: 24, freq: h} + +results: + directory: results/uc4_example + +pipeline: + - setup_grid + - base_reinforce + - import_generators + - import_home_batteries + - import_heat_pumps + - import_dsm + - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - oedb_ts: + dispatchable: {other: 0.7} + - reactive_power + - check_integrity + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - reinforce + - save From de69b53812ce865cb765cba942e795fdd46d30cc Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:20:22 +0200 Subject: [PATCH 03/66] Add file for analysis-tasks, Add short cut for DSM --- edisgo/run/tasks/analysis.py | 321 +++++++++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 edisgo/run/tasks/analysis.py diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py new file mode 100644 index 000000000..f028bb31c --- /dev/null +++ b/edisgo/run/tasks/analysis.py @@ -0,0 +1,321 @@ +""" +Power-flow, reinforcement, and optimization tasks. + +The three analysis layers: + +* :func:`task_analyze` (``analyze``) — non-linear AC load flow over + the active time series; does not modify the topology. +* :func:`task_reinforce` (``reinforce``) — iterative reinforcement + that adds/upgrades equipment until all technical constraints are + met. Populates ``results.equipment_changes``. +* :func:`task_optimize` (``optimize``) — powermodels OPF over + flexibilities (heat pumps, EV, DSM, storage) to minimize + reinforcement need. + +In addition: + +* :func:`task_check_integrity` (``check_integrity``) — a cheap + sanity check before the expensive steps. +* :func:`task_base_reinforce` (``base_reinforce``) — two-phase helper: + worst-case TS → reinforce → reset ``equipment_changes``. Used to + produce a "base" grid whose subsequent reinforce costs reflect + only a scenario overlay. +""" +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + + +@register_task("check_integrity") +def task_check_integrity(edisgo, ctx): + """ + Run EDisGo's integrity checks on the topology and time series. + + Catches bus mismatches, missing time series for components, and + similar structural problems. Raises if something is off — do not + swallow it silently. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to check. + ctx : RunContext + Run context (unused). + + Returns + ------- + edisgo.EDisGo + The unchanged EDisGo instance. + + """ + edisgo.check_integrity() + return edisgo + + +@register_task("analyze") +def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, + raise_not_converged=False, troubleshooting_mode=None): + """ + Run AC power flow over the active time series. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to analyze. + ctx : RunContext + Run context. Stores the number of non-converged time steps + under ``ctx.flags['not_converged_steps']`` and warns if any. + mode : str, optional + ``None`` (default) runs the full grid; ``"mv"`` runs only the + medium-voltage level; ``"lv"`` runs only LV. + timesteps : pandas.DatetimeIndex, optional + Restrict the analysis to these time steps. + raise_not_converged : bool, optional + If ``True``, raise on non-convergence. Default ``False`` so + the pipeline can continue and ``reinforce`` can attempt to + resolve the issue. + troubleshooting_mode : str, optional + Extra diagnostic mode passed through to + :meth:`EDisGo.analyze`. + + Returns + ------- + edisgo.EDisGo + The analyzed EDisGo instance. + + """ + result = edisgo.analyze( + mode=mode, + timesteps=timesteps, + raise_not_converged=raise_not_converged, + troubleshooting_mode=troubleshooting_mode, + ) + if isinstance(result, tuple) and len(result) == 2: + converged, not_converged = result + ctx.flags["not_converged_steps"] = len(not_converged) + if len(not_converged) > 0: + ctx.logger.warning( + f"Power flow did not converge for {len(not_converged)} " + f"time steps." + ) + return edisgo + + +@register_task("reinforce") +def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, + copy_grid=False, max_while_iterations=20, + split_voltage_band=True, mode=None, + without_generator_import=False, n_minus_one=False, + catch_convergence_problems=False): + """ + Run iterative grid reinforcement. + + Adds/upgrades lines and transformers until voltage and loading + constraints are met for all time steps. Results accumulate in + :attr:`EDisGo.results.equipment_changes` and + :attr:`~EDisGo.results.grid_expansion_costs`. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to reinforce. + ctx : RunContext + Run context (unused beyond logging). + timesteps_pfa : pandas.DatetimeIndex, optional + Restrict the reinforcement's analysis to these time steps. + reduced_analysis : bool, optional + If ``True``, use a cheaper convergence check during + reinforcement. + copy_grid : bool, optional + If ``True``, operate on a copy and return it as a new + instance (default ``False``). + max_while_iterations : int, optional + Cap on the outer iteration loop. + split_voltage_band : bool, optional + Split the allowed voltage deviation between MV and LV + (typical MV/LV coupling rule). + mode : str, optional + ``None``, ``"mv"``, ``"lv"``, or ``"mvlv"``. Restricts + reinforcement to a voltage level. + without_generator_import : bool, optional + Skip the implicit generator import step. + n_minus_one : bool, optional + Enable (N-1) contingency reinforcement. Expensive. + catch_convergence_problems : bool, optional + Wrap in the catch-convergence helper for troublesome grids. + + Returns + ------- + edisgo.EDisGo + The reinforced EDisGo instance. + + """ + edisgo.reinforce( + timesteps_pfa=timesteps_pfa, + reduced_analysis=reduced_analysis, + copy_grid=copy_grid, + max_while_iterations=max_while_iterations, + split_voltage_band=split_voltage_band, + mode=mode, + without_generator_import=without_generator_import, + n_minus_one=n_minus_one, + catch_convergence_problems=catch_convergence_problems, + ) + return edisgo + + +@register_task("base_reinforce") +def task_base_reinforce(edisgo, ctx, *, cases=None, + reset_equipment_changes=True, save_artifact=True): + """ + Produce a base-reinforced grid and reset the cost accumulator. + + This is the composite step ported from eGo's two-phase reinforce + workflow: + + 1. Set synthetic worst-case time series (``feed-in_case`` + + ``load_case``). + 2. Run :meth:`EDisGo.reinforce` to bring the grid to a neutral + baseline. + 3. Optionally save the resulting grid so downstream stages can + ``load_from: ...``. + 4. Clear :attr:`Results.equipment_changes` so the next reinforce + captures only scenario-specific deltas. + 5. Restore the prior time index so the next TS-setting task + starts from a clean state. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to base-reinforce. + ctx : RunContext + Run context. ``ctx.results_dir`` is the artifact destination. + Sets ``ctx.flags['base_reinforced'] = True`` and + ``ctx.stage_artifacts['__base_reinforce__']`` on save. + cases : list of str, optional + Which worst cases to set (subset of + ``{"load_case", "feed-in_case"}``). Default is both. + reset_equipment_changes : bool, optional + Clear the equipment-changes DataFrame after reinforcement. + save_artifact : bool, optional + Write a ``grid_data_base_reinforcement.zip`` next to the + other results. + + Returns + ------- + edisgo.EDisGo + The base-reinforced EDisGo instance. + + """ + import os + + prev_timeindex = edisgo.timeseries.timeindex + + edisgo.set_time_series_worst_case_analysis(cases=cases) + edisgo.reinforce() + + if save_artifact and ctx.results_dir is not None: + artifact_dir = os.path.join( + str(ctx.results_dir), "grid_data_base_reinforcement" + ) + edisgo.save( + directory=artifact_dir, + save_topology=True, + save_timeseries=False, + save_results=True, + archive=True, + archive_type="zip", + parameters={"grid_expansion_results": ["equipment_changes"]}, + ) + ctx.stage_artifacts["__base_reinforce__"] = artifact_dir + ".zip" + + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + + if len(prev_timeindex) > 0: + edisgo.set_timeindex(prev_timeindex) + + ctx.flags["base_reinforced"] = True + return edisgo + + +@register_task("optimize") +def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, + flexible_hps=None, flexible_loads=None, + flexible_storage_units=None, opf_version=2, method="soc", + warm_start=False, s_base=1): + """ + Run a powermodels optimal-power-flow (OPF) over flexibilities. + + If ``flexible`` is given (high-level shortcut), it expands to the + lower-level ``flexible_*`` lists automatically: + + * ``"heat_pumps"`` → all loads of type ``heat_pump`` + * ``"charging_points"`` → all loads of type ``charging_point`` + * ``"storage"`` → all storage-unit indices + * ``"loads"`` → all DSM-ready load indices + + Explicit ``flexible_*`` kwargs override the shortcut. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to optimize. + ctx : RunContext + Run context (unused). + flexible : list of str, optional + High-level selector, subset of ``{"heat_pumps", + "charging_points", "storage"}``. If ``None``, nothing is + auto-populated. + flexible_cps : list of str, optional + Explicit list of flexible charging-point names. + flexible_hps : list of str, optional + Explicit list of flexible heat-pump load names. + flexible_loads : list of str, optional + Explicit list of flexible DSM load names. + flexible_storage_units : list of str, optional + Explicit list of flexible storage-unit names. + opf_version : int, optional + Powermodels OPF formulation version (1 or 2, default 2). + method : str, optional + OPF relaxation method, e.g. ``"soc"`` (second-order cone). + warm_start : bool, optional + Reuse a previous solution as the starting point. + s_base : float, optional + Per-unit base power for normalization. + + Returns + ------- + edisgo.EDisGo + The optimized EDisGo instance. + + """ + flexible = flexible or [] + + if flexible_hps is None and "heat_pumps" in flexible: + flexible_hps = edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "heat_pump" + ].index.tolist() + if flexible_cps is None and "charging_points" in flexible: + flexible_cps = edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "charging_point" + ].index.tolist() + if flexible_storage_units is None and "storage" in flexible: + flexible_storage_units = edisgo.topology.storage_units_df.index.tolist() + if flexible_loads is not None and "dsm" in flexbile: + flexible_loads = edisgo.dsm.p_min.columns.values + + + edisgo.pm_optimize( + flexible_cps=flexible_cps or [], + flexible_hps=flexible_hps or [], + flexible_loads=flexible_loads or [], + flexible_storage_units=flexible_storage_units or [], + opf_version=opf_version, + method=method, + warm_start=warm_start, + s_base=s_base, + ) + return edisgo From fb5d32b533d7ec9a7d0206c84f13f8e8b10a30c9 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:21:10 +0200 Subject: [PATCH 04/66] Add file for flex-tasks, Add flexibility band generation, --- edisgo/run/tasks/flex.py | 282 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 edisgo/run/tasks/flex.py diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py new file mode 100644 index 000000000..fd914a2a0 --- /dev/null +++ b/edisgo/run/tasks/flex.py @@ -0,0 +1,282 @@ +""" +Flex-asset import and operation-strategy tasks. + +These tasks either pull flex assets (heat pumps, home batteries, DSM, +electromobility, generators) from egon_data / OEP into the topology, +or apply an operating strategy on assets already present. They must +run AFTER the grid is loaded (``setup_grid`` or ``load_from_base``) +and typically BEFORE the time-series step, so the time series can +cover the new assets. +""" +from __future__ import annotations + +from edisgo.run.registry import register_task + + +@register_task("import_heat_pumps") +def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): + """ + Import heat pumps from egon_data into the topology. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets + ``ctx.flags['has_heat_pumps']`` to the observed count. + import_types : list of str, optional + Subset of ``["individual_heat_pumps", "central_heat_pumps"]``; + default imports both. + timeindex : pandas.DatetimeIndex, optional + Restrict COP / heat-demand time series to this index. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_heat_pumps( + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + timeindex=timeindex, + import_types=import_types, + ) + ctx.flags["has_heat_pumps"] = len( + edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "heat_pump" + ] + ) > 0 + return edisgo + + +@register_task("import_home_batteries") +def task_import_home_batteries(edisgo, ctx): + """ + Import home batteries from egon_data into the topology. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets + ``ctx.flags['has_home_batteries']``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_home_batteries( + scenario=ctx.scenario, engine=ctx.ensure_engine() + ) + ctx.flags["has_home_batteries"] = ( + not edisgo.topology.storage_units_df.empty + ) + return edisgo + + +@register_task("import_dsm") +def task_import_dsm(edisgo, ctx, *, timeindex=None): + """ + Import demand-side-management potential from egon_data. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets ``ctx.flags['has_dsm']``. + timeindex : pandas.DatetimeIndex, optional + Restrict DSM availability time series to this index. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_dsm( + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + timeindex=timeindex, + ) + ctx.flags["has_dsm"] = ( + edisgo.dsm.p_max is not None and not edisgo.dsm.p_max.empty + ) + return edisgo + + +@register_task("import_electromobility") +def task_import_electromobility(edisgo, ctx, *, data_source="oedb", + charging_strategy="dumb", + flexibility_bands_ucs = None, + import_electromobility_data_kwds=None, + allocate_charging_demand_kwds=None): + """ + Import electromobility data (charging processes + parks). + + Optionally applies a charging strategy directly after import to + turn the raw charging processes into active-power time series on + the charging points. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()`` (for ``data_source='oedb'``). Sets + ``ctx.flags['has_electromobility'] = True``. + data_source : str, optional + ``"oedb"`` (egon_data) or ``"directory"`` (requires + ``import_electromobility_data_kwds={"charging_processes_dir": + ..., "potential_charging_points_dir": ...}``). + charging_strategy : str or None, optional + Charging strategy applied right after import. ``"dumb"`` + (uncontrolled, default), ``"reduced"``, ``"residual"``, or + ``None`` to skip. + flexibility_bands_ucs : str or list of str, optional + Charging-point use case(s) to compute flexibility bands for + via :meth:`Electromobility.get_flexibility_bands` after import + and charging-strategy application. Valid entries: + ``"home"``, ``"work"``, ``"public"``, ``"hpc"``. Pass a single + string for one use case or a list for multiple. ``None`` + (default) skips flexibility-band computation. + import_electromobility_data_kwds : dict, optional + Extra kwargs passed through to the underlying importer. + allocate_charging_demand_kwds : dict, optional + Extra kwargs for charging-demand allocation. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_electromobility( + data_source=data_source, + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + import_electromobility_data_kwds=import_electromobility_data_kwds, + allocate_charging_demand_kwds=allocate_charging_demand_kwds, + ) + if charging_strategy: + edisgo.apply_charging_strategy(strategy=charging_strategy) + if flexibility_bands_ucs is not None: + edisgo.electromobility.get_flexibility_bands( + edisgo, + use_case=flexibility_bands_ucs, + ) + ctx.flags["has_electromobility"] = True + return edisgo + + +@register_task("apply_charging_strategy") +def task_apply_charging_strategy(edisgo, ctx, *, strategy="dumb", + charging_park_ids=None): + """ + Apply a charging strategy to the already-imported EV fleet. + + Standalone variant of the step that ``import_electromobility`` + does inline. Useful when you want to import once and then try + multiple strategies in different runs. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + strategy : str, optional + Strategy name (``"dumb"`` / ``"reduced"`` / ``"residual"``). + charging_park_ids : list of int, optional + Restrict the strategy to these charging-park IDs. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.apply_charging_strategy( + strategy=strategy, charging_park_ids=charging_park_ids + ) + return edisgo + + +@register_task("apply_heat_pump_strategy") +def task_apply_heat_pump_strategy(edisgo, ctx, *, strategy="uncontrolled", + heat_pump_names=None): + """ + Apply a heat-pump operating strategy. + + Skipped with an info-log if no heat pumps are present + (``ctx.flags['has_heat_pumps']`` is falsy), so pipelines can + safely include this step without a conditional guard. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + strategy : str, optional + Operating strategy (``"uncontrolled"``, ``"flexible"``, …). + heat_pump_names : list of str, optional + Restrict to specific heat-pump load names; default is all. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if not ctx.flags.get("has_heat_pumps"): + ctx.logger.info( + "Skipping 'apply_heat_pump_strategy': no heat pumps " + "present." + ) + return edisgo + edisgo.apply_heat_pump_operating_strategy( + strategy=strategy, heat_pump_names=heat_pump_names + ) + return edisgo + + +@register_task("import_generators") +def task_import_generators(edisgo, ctx, *, generator_scenario=None): + """ + Import future generators for the active scenario. + + Thin wrapper around :meth:`EDisGo.import_generators`. Mostly + useful when you want to split grid loading and generator import + into two separate pipeline steps. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. ``ctx.scenario`` is used if + ``generator_scenario`` is not given. + generator_scenario : str, optional + Scenario name, e.g. ``"nep2035"`` or ``"ego100"``. Defaults + to ``ctx.scenario``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_generators( + generator_scenario=generator_scenario or ctx.scenario + ) + return edisgo From bd5eaa8d5f214e206549f0837ac591706c77dbdf Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:22:18 +0200 Subject: [PATCH 05/66] Add file for grid-tasks, Add timeindex in setup task --- edisgo/run/tasks/grid.py | 172 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 edisgo/run/tasks/grid.py diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py new file mode 100644 index 000000000..b472f3244 --- /dev/null +++ b/edisgo/run/tasks/grid.py @@ -0,0 +1,172 @@ +""" +Grid loading tasks — bring an EDisGo instance into existence. + +Two ways to start a pipeline: + +* :func:`task_setup_grid` (``setup_grid``) — read a ding0 topology + from disk. This is the typical first step of every pipeline. +* :func:`task_load_from_base` (``load_from_base``) — reload a + previously saved EDisGo instance. Used to split a computation into + a slow "base" phase and one or more fast "scenario" phases that + reuse the base-reinforced grid. +""" +from __future__ import annotations + +from edisgo.run.registry import register_task + + +@register_task("setup_grid") +def task_setup_grid(edisgo, ctx, *, timeindex = None, ding0_path=None, legacy_ding0_grids=None, + import_generators=False, generator_scenario=None): + """ + Load a ding0 grid into an EDisGo instance. + + If the runner was started without an EDisGo object (via + :func:`edisgo.run.run_edisgo`) this task creates one from the + ding0 CSV directory. If an EDisGo object is already present (via + :meth:`edisgo.EDisGo.run_pipeline`), it imports the topology into + that existing instance. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Current EDisGo instance, or ``None`` to create a fresh one. + ctx : RunContext + Run context. ``ctx.raw_config['grid']`` is consulted when + parameters are not passed explicitly. + ding0_path : str, optional + Path to the ding0 grid directory. Falls back to + ``ctx.raw_config['grid']['ding0_path']``. + legacy_ding0_grids : bool, optional + Whether to treat the ding0 directory as the legacy format. + Falls back to ``ctx.raw_config['grid']['legacy_ding0_grids']`` + and ultimately to ``False``. + import_generators : bool, optional + If ``True``, call :meth:`EDisGo.import_generators` after + loading the grid. + generator_scenario : str, optional + Generator scenario name passed to + :meth:`EDisGo.import_generators` (only if + ``import_generators=True``). + + Returns + ------- + edisgo.EDisGo + The EDisGo instance with the ding0 topology loaded. + + Raises + ------ + ValueError + If no ``ding0_path`` is given either as a task parameter or + under ``config.grid.ding0_path``. + + """ + from edisgo import EDisGo + + grid_cfg = ctx.raw_config.get("grid", {}) + ding0_path = ding0_path or grid_cfg.get("ding0_path") + if ding0_path is None: + raise ValueError( + "Task 'setup_grid' requires 'ding0_path' either as task " + "parameter or under config.grid.ding0_path." + ) + if legacy_ding0_grids is None: + legacy_ding0_grids = grid_cfg.get("legacy_ding0_grids", False) + + if edisgo is None: + edisgo = EDisGo( + ding0_grid=str(ding0_path), + legacy_ding0_grids=legacy_ding0_grids, + ) + else: + edisgo.import_ding0_grid( + path=str(ding0_path), legacy_ding0_grids=legacy_ding0_grids + ) + + if import_generators: + edisgo.import_generators(generator_scenario=generator_scenario) + + if timeindex is not None: + ti_df = pd.date_range( + start=timeindex["start"], + periods=timeindex["periods"], + freq=timeindex.get("freq", "h"), + ) + edisgo.set_timeindex(ti_df) + + ctx.flags["grid_loaded"] = True + return edisgo + + +@register_task("load_from_base") +def task_load_from_base(edisgo, ctx, *, path, reset_equipment_changes=True, + import_timeseries=False, import_results=False, + import_electromobility=False, import_heat_pump=False, + import_dsm=False, import_overlying_grid=False): + """ + Reload an EDisGo instance from a previously saved directory/zip. + + This is the two-phase R4MU workflow's entry point: stage 1 + produces a base-reinforced grid and saves it, stage 2 (or N) + starts from ``load_from_base`` to pick up that grid and apply + scenario-specific modifications. The cost of the scenario then + shows up cleanly in ``equipment_changes`` because we reset it on + load. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Unused — the task always replaces whatever was there. + ctx : RunContext + Run context (logger only). + path : str + Directory or ``.zip`` produced by :func:`task_save`. + reset_equipment_changes : bool, optional + If ``True`` (default), clear + :attr:`Results.equipment_changes` so only the scenario's + reinforce is tracked. + import_timeseries : bool, optional + Whether to import the saved time series. Default: ``False`` + so the next stage sets its own. + import_results : bool, optional + Whether to import saved results. Default: ``False``. + import_electromobility : bool, optional + Whether to import saved electromobility data. + import_heat_pump : bool, optional + Whether to import saved heat-pump data. + import_dsm : bool, optional + Whether to import saved DSM data. + import_overlying_grid : bool, optional + Whether to import saved overlying-grid data (eTraGo + specifications). + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import os + + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + path = str(path) + from_zip = path.endswith(".zip") or not os.path.isdir(path) + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=import_timeseries, + import_results=import_results, + import_electromobility=import_electromobility, + import_heat_pump=import_heat_pump, + import_dsm=import_dsm, + import_overlying_grid=import_overlying_grid, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + ctx.flags["grid_loaded"] = True + return edisgo From aa7800482446dca7df914250b5f6b9c3fd5d2ca2 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:23:27 +0200 Subject: [PATCH 06/66] Add file for timeseries-tasks --- edisgo/run/tasks/timeseries.py | 298 +++++++++++++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 edisgo/run/tasks/timeseries.py diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py new file mode 100644 index 000000000..f738aa056 --- /dev/null +++ b/edisgo/run/tasks/timeseries.py @@ -0,0 +1,298 @@ +""" +Time-series tasks — set active/reactive power profiles on EDisGo. + +Time series drive every downstream step: ``analyze``, ``reinforce`` +and ``optimize`` all operate on the time index and power time series +attached to the EDisGo object. The order inside a stage matters: + +1. Set the time index and active-power profiles with one of + :func:`task_worst_case_ts`, :func:`task_oedb_ts`, + :func:`task_manual_ts`, possibly :func:`task_set_timeindex`. +2. Finally call :func:`task_reactive_power` to fix reactive power + control — this MUST come last because it overwrites whatever + reactive power was set by the earlier steps. +""" +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + + +@register_task("worst_case_ts") +def task_worst_case_ts(edisgo, ctx, *, cases=None, + generators_names=None, loads_names=None, + storage_units_names=None): + """ + Set synthetic worst-case active-power time series. + + Produces two snapshots (load case and feed-in case) that + represent the network's extremes. Useful for a coarse first + reinforce that does not require real load/generation data. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['timeseries_set'] = True``. + cases : list of str, optional + Subset of ``{"load_case", "feed-in_case"}``. Default is both. + generators_names : list of str, optional + Restrict to these generator names; default is all. + loads_names : list of str, optional + Restrict to these load names; default is all. + storage_units_names : list of str, optional + Restrict to these storage units; default is all. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.set_time_series_worst_case_analysis( + cases=cases, + generators_names=generators_names, + loads_names=loads_names, + storage_units_names=storage_units_names, + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("set_timeindex") +def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, + freq="h"): + """ + Set the time index on the EDisGo object. + + Useful as a stand-alone step when you want a specific hourly + range without immediately attaching time-series data (the + ``oedb_ts`` task already accepts a ``timeindex`` argument and + does this internally). + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + start : str or pandas.Timestamp + First timestamp of the range. + periods : int, optional + Number of periods; mutually exclusive with ``end``. + end : str or pandas.Timestamp, optional + Last timestamp; mutually exclusive with ``periods``. + freq : str, optional + pandas frequency string, default hourly (``"h"``). + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + Raises + ------ + ValueError + If neither ``periods`` nor ``end`` is provided. + + """ + if end is not None: + timeindex = pd.date_range(start=start, end=end, freq=freq) + else: + if periods is None: + raise ValueError( + "set_timeindex needs either 'periods' or 'end'." + ) + timeindex = pd.date_range(start=start, periods=periods, freq=freq) + edisgo.set_timeindex(timeindex) + return edisgo + + +@register_task("oedb_ts") +def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, + fluctuating="oedb", conventional_loads="oedb", + charging_points_ts=None): + """ + Set active-power time series from egon_data (OEP) plus overrides. + + This is the "real data" path: wind and solar profiles come from + ``egon_era5_renewable_feedin``, conventional loads come from the + egon demand tables. Dispatchable generators (conventional, + etc.) are set via a per-technology-type profile since egon_data + does not dispatch them. Storage units default to zero if not + already set. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()`` when any source is ``"oedb"``. Sets + ``ctx.flags['timeseries_set'] = True``. + timeindex : dict, optional + ``{"start": ..., "periods": N, "freq": "h"}``. If present, a + matching :class:`~pandas.DatetimeIndex` is set before + importing data. + dispatchable : dict, optional + Per-technology scaling factors, e.g. ``{"other": 0.7}`` → + constant profile of 0.7 p.u. for all non-fluctuating + generators of type "other". + fluctuating : str or pandas.DataFrame, optional + How to populate wind/solar. ``"oedb"`` pulls egon_data, + ``"default"`` uses bundled standard profiles, or a DataFrame + with columns "solar" / "wind" is passed through. + conventional_loads : str, optional + Source for conventional loads (not heat pumps / charging + points). ``"oedb"`` or ``"demandlib"``. + charging_points_ts : pandas.DataFrame, optional + Explicit active-power profile for charging points; default + ``None`` leaves them untouched so + :func:`task_apply_charging_strategy` can set them. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if timeindex is not None: + ti_df = pd.date_range( + start=timeindex["start"], + periods=timeindex["periods"], + freq=timeindex.get("freq", "h"), + ) + edisgo.set_timeindex(ti_df) + + dispatchable_df = None + if dispatchable is not None: + ti = edisgo.timeseries.timeindex + dispatchable_df = pd.DataFrame(dispatchable, index=ti) + + conv_loads_names = None + if conventional_loads == "oedb": + conv_loads_names = edisgo.topology.loads_df.loc[ + ~edisgo.topology.loads_df.type.isin( + ["heat_pump", "charging_point"] + ) + ].index.tolist() + + edisgo.set_time_series_active_power_predefined( + fluctuating_generators_ts=fluctuating, + conventional_loads_ts=conventional_loads, + conventional_loads_names=conv_loads_names, + dispatchable_generators_ts=dispatchable_df, + charging_points_ts=charging_points_ts, + scenario=ctx.scenario, + engine=ctx.ensure_engine() if fluctuating == "oedb" + or conventional_loads == "oedb" else None, + ) + + su_names = edisgo.topology.storage_units_df.index + if len(su_names) > 0 and edisgo.timeseries.storage_units_active_power.empty: + edisgo.timeseries.storage_units_active_power = pd.DataFrame( + 0.0, index=edisgo.timeseries.timeindex, columns=su_names, + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("manual_ts") +def task_manual_ts(edisgo, ctx, *, + generators_active_power=None, + generators_reactive_power=None, + loads_active_power=None, + loads_reactive_power=None, + storage_units_active_power=None, + storage_units_reactive_power=None): + """ + Set active/reactive power time series from explicit DataFrames. + + Used when the caller already has the raw profiles (e.g. from a + coupled run) and wants to inject them directly. Any argument left + at ``None`` is not touched. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['timeseries_set'] = True``. + generators_active_power : dict or pandas.DataFrame, optional + Generator active-power profile(s). Converted via + :class:`pandas.DataFrame`. + generators_reactive_power : dict or pandas.DataFrame, optional + Generator reactive-power profile(s). + loads_active_power : dict or pandas.DataFrame, optional + Load active-power profile(s). + loads_reactive_power : dict or pandas.DataFrame, optional + Load reactive-power profile(s). + storage_units_active_power : dict or pandas.DataFrame, optional + Storage-unit active-power profile(s). + storage_units_reactive_power : dict or pandas.DataFrame, optional + Storage-unit reactive-power profile(s). + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + def _as_df(obj): + return pd.DataFrame(obj) if obj is not None else None + + edisgo.set_time_series_manual( + generators_active_power=_as_df(generators_active_power), + generators_reactive_power=_as_df(generators_reactive_power), + loads_active_power=_as_df(loads_active_power), + loads_reactive_power=_as_df(loads_reactive_power), + storage_units_active_power=_as_df(storage_units_active_power), + storage_units_reactive_power=_as_df(storage_units_reactive_power), + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("reactive_power") +def task_reactive_power(edisgo, ctx, *, control="fixed_cosphi", + generators_parametrisation="default", + loads_parametrisation="default", + storage_units_parametrisation="default"): + """ + Apply reactive-power control on top of the active-power time series. + + This MUST be the last time-series-altering step before + ``analyze`` / ``reinforce`` / ``optimize``. The validator + enforces this ordering rule statically. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['reactive_power_set'] = True``. + control : str, optional + Reactive-power control strategy; typically ``"fixed_cosphi"``. + generators_parametrisation : str or dict, optional + Per-generator parametrisation, ``"default"`` uses the config. + loads_parametrisation : str or dict, optional + Per-load parametrisation. + storage_units_parametrisation : str or dict, optional + Per-storage-unit parametrisation. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.set_time_series_reactive_power_control( + control=control, + generators_parametrisation=generators_parametrisation, + loads_parametrisation=loads_parametrisation, + storage_units_parametrisation=storage_units_parametrisation, + ) + ctx.flags["reactive_power_set"] = True + return edisgo From 3a288806d17c6fed76dd4aa8f6f115119ad1775d Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Tue, 19 May 2026 16:40:05 +0200 Subject: [PATCH 07/66] Edit uc4 example --- edisgo/run/presets/uc4_example_MS.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example_MS.yaml index a72573496..13e08b258 100644 --- a/edisgo/run/presets/uc4_example_MS.yaml +++ b/edisgo/run/presets/uc4_example_MS.yaml @@ -24,16 +24,21 @@ _workflow: scenario: eGon2035 grid: ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + # grid path should be set in run file legacy_ding0_grids: false + # legacy parameter too database: ssh: enabled: false timeindex: {start: "2035-01-01", periods: 24, freq: h} +# only set in preset, not set in run-functions (eDisGo and eGo) results: directory: results/uc4_example +# actually all parameters + pipeline: - setup_grid @@ -42,6 +47,7 @@ pipeline: - import_home_batteries - import_heat_pumps - import_dsm + # where is decided, which electromobility use cases are used? - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - apply_heat_pump_strategy: {strategy: uncontrolled} - oedb_ts: @@ -49,6 +55,7 @@ pipeline: - reactive_power - check_integrity - optimize: + # where is decided which flexibilities are used in the OPF? flexible: [heat_pumps, storage, charging_points, dsm] method: soc opf_version: 2 From b7e741531adc7bb57c625aec1dcca78b5c81b1de Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Wed, 20 May 2026 18:17:15 +0200 Subject: [PATCH 08/66] Wire overlying-grid data through pipeline runner Adds optional overlying_grid_data kwarg to run_edisgo() that is stashed on RunContext for downstream tasks instead of being passed as a keyword to every task (which broke task signatures). task_import_overlying_grid_data now: - accepts the standard (edisgo, ctx, *, ...) signature - reads overlying_grid_data from ctx, falls back to overlying_grid.path in the runner config - loads dispatchable + renewables_potential CSVs and applies them via set_time_series_active_power_predefined - shifts the year and reindexes overlying-grid attributes to the active edisgo timeindex so CSV-based input lines up with OEDB time series task_set_timeindex now reduces existing time-series data to the new index (via reduce_timeseries_data_to_given_timeindex) instead of silently leaving stale data behind. Renames the uc4_example_MS preset to uc4_example. --- .../{uc4_example_MS.yaml => uc4_example.yaml} | 9 +- edisgo/run/runner.py | 8 +- edisgo/run/tasks/io.py | 158 ++++++++++++++++-- edisgo/run/tasks/timeseries.py | 85 ++++++---- 4 files changed, 214 insertions(+), 46 deletions(-) rename edisgo/run/presets/{uc4_example_MS.yaml => uc4_example.yaml} (88%) diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example.yaml similarity index 88% rename from edisgo/run/presets/uc4_example_MS.yaml rename to edisgo/run/presets/uc4_example.yaml index 13e08b258..2d2650279 100644 --- a/edisgo/run/presets/uc4_example_MS.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -23,7 +23,7 @@ _workflow: scenario: eGon2035 grid: - ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + ding0_path: "/path/to/ding0_grid" # grid path should be set in run file legacy_ding0_grids: false # legacy parameter too @@ -48,10 +48,13 @@ pipeline: - import_heat_pumps - import_dsm # where is decided, which electromobility use cases are used? - - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - - apply_heat_pump_strategy: {strategy: uncontrolled} + - import_electromobility: {charging_strategy: null, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - oedb_ts: dispatchable: {other: 0.7} + timeindex: {start: "2035-01-01", periods: 24, freq: h} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - import_overlying_grid_data - reactive_power - check_integrity - optimize: diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py index 63f30aa07..2d08fd3bc 100644 --- a/edisgo/run/runner.py +++ b/edisgo/run/runner.py @@ -27,6 +27,7 @@ * :func:`_run_pipeline_on` — starts from an existing EDisGo instance; used by :meth:`edisgo.EDisGo.run_pipeline`. """ + from __future__ import annotations import logging @@ -43,7 +44,7 @@ logger = logging.getLogger("edisgo.run.runner") -def run_edisgo(config) -> Any: +def run_edisgo(config, overlying_grid_data=None) -> Any: """ Run an eDisGo pipeline from a YAML/JSON config or dict. @@ -66,10 +67,10 @@ def run_edisgo(config) -> Any: stage. """ - return _run_pipeline_on(None, config) + return _run_pipeline_on(None, config, overlying_grid_data=overlying_grid_data) -def _run_pipeline_on(edisgo, config): +def _run_pipeline_on(edisgo, config, overlying_grid_data=None): """ Internal runner shared by :func:`run_edisgo` and the EDisGo method. @@ -97,6 +98,7 @@ def _run_pipeline_on(edisgo, config): cfg = load_config(config) validate(cfg) ctx = _build_context(cfg) + ctx.overlying_grid_data = overlying_grid_data for stage in cfg["stages"]: ctx.current_stage = stage["name"] diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 3f604914e..4e2e84d15 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -10,6 +10,7 @@ integrating scenario charging stations from a directory of CSV / GeoPackage files; implementation is deferred until needed. """ + from __future__ import annotations import os @@ -18,12 +19,24 @@ @register_task("save") -def task_save(edisgo, ctx, *, directory=None, save_topology=True, - save_timeseries=True, save_results=True, - save_electromobility=None, save_opf_results=False, - save_heatpump=None, save_overlying_grid=False, - save_dsm=None, archive=False, archive_type="zip", - reduce_memory=False, parameters=None): +def task_save( + edisgo, + ctx, + *, + directory=None, + save_topology=True, + save_timeseries=True, + save_results=True, + save_electromobility=None, + save_opf_results=False, + save_heatpump=None, + save_overlying_grid=False, + save_dsm=None, + archive=False, + archive_type="zip", + reduce_memory=False, + parameters=None, +): """ Save the current EDisGo state to disk. @@ -93,8 +106,7 @@ def task_save(edisgo, ctx, *, directory=None, save_topology=True, if directory is None: if ctx.results_dir is None: raise ValueError( - "Task 'save' needs a 'directory' parameter or " - "config.results.directory." + "Task 'save' needs a 'directory' parameter or config.results.directory." ) stage = ctx.current_stage or "main" directory = os.path.join(str(ctx.results_dir), stage) @@ -135,9 +147,9 @@ def task_save(edisgo, ctx, *, directory=None, save_topology=True, @register_task("load_charging_from_files") -def task_load_charging_from_files(edisgo, ctx, *, charging_dir, - use_case_to_sector=None, - mv_threshold_kw=100.0): +def task_load_charging_from_files( + edisgo, ctx, *, charging_dir, use_case_to_sector=None, mv_threshold_kw=100.0 +): """ Integrate scenario charging stations from files (R4MU workflow). @@ -177,3 +189,127 @@ def task_load_charging_from_files(edisgo, ctx, *, charging_dir, "_run_edisgo_task_load_charging_from_files when R4MU is " "needed." ) + + +@register_task("import_overlying_grid_data") +def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): + """ + Import overlying grid data into the EDisGo instance. + + When ``overlying_grid_data`` is a dict of DataFrames (as returned by + ``get_etrago_results_per_bus``), the overlying-grid attributes and + dispatchable/fluctuating generator time series are set from it. + + When ``overlying_grid_path`` is a directory path, the overlying-grid + attributes are loaded from CSV files in that directory, and + ``dispatchable_generators_active_power.csv`` / + ``renewables_potential.csv`` are applied as generator time series + if present. + + Falls back to ``ctx.raw_config['eDisGo']['overlying_grid_source']`` + as the directory path when neither argument is given. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + overlying_grid_path : str, optional + Directory containing overlying-grid CSV files. + overlying_grid_data : dict, optional + Dict of DataFrames as returned by ``get_etrago_results_per_bus``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + import pandas as pd + + overlying_grid_data = getattr(ctx, "overlying_grid_data", None) + + if overlying_grid_data is not None: + # eTraGo results dict — set standard overlying-grid attributes + for attr in edisgo.overlying_grid._attributes: + if attr in overlying_grid_data: + setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + # set generator time series + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=overlying_grid_data.get( + "dispatchable_generators_active_power" + ), + fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), + ) + return edisgo + + # resolve path: explicit arg → runner config overlying_grid.path → skip + if overlying_grid_path is None: + overlying_grid_path = (ctx.raw_config.get("overlying_grid") or {}).get("path") + + if overlying_grid_path is None: + ctx.logger.warning( + "task 'import_overlying_grid_data': no overlying_grid_data or " + "overlying_grid_path provided — skipping." + ) + return edisgo + + # load overlying-grid attributes from CSV directory + edisgo.overlying_grid.from_csv(overlying_grid_path) + + # reindex overlying-grid attributes to match edisgo timeindex + # CSVs may use a different year — shift year then reindex + edisgo_ti = edisgo.timeseries.timeindex + if not edisgo_ti.empty: + for attr in edisgo.overlying_grid._attributes: + ts = getattr(edisgo.overlying_grid, attr) + if ts.empty: + continue + csv_year = ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) + if isinstance(ts, pd.Series): + setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + else: + setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + + # load dispatchable generator and renewables time series from the same dir + disp_path = os.path.join( + overlying_grid_path, "dispatchable_generators_active_power.csv" + ) + if os.path.isfile(disp_path): + disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = disp_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + disp_ts.index = disp_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + disp_ts = disp_ts.reindex(edisgo_ti) + else: + disp_ts = None + + pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") + if os.path.isfile(pot_path): + pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = pot_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + pot_ts.index = pot_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + pot_ts = pot_ts.reindex(edisgo_ti) + else: + pot_ts = None + + if disp_ts is not None or pot_ts is not None: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + fluctuating_generators_ts=pot_ts, + ) + + return edisgo diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py index f738aa056..cf918c235 100644 --- a/edisgo/run/tasks/timeseries.py +++ b/edisgo/run/tasks/timeseries.py @@ -12,6 +12,7 @@ control — this MUST come last because it overwrites whatever reactive power was set by the earlier steps. """ + from __future__ import annotations import pandas as pd @@ -20,9 +21,15 @@ @register_task("worst_case_ts") -def task_worst_case_ts(edisgo, ctx, *, cases=None, - generators_names=None, loads_names=None, - storage_units_names=None): +def task_worst_case_ts( + edisgo, + ctx, + *, + cases=None, + generators_names=None, + loads_names=None, + storage_units_names=None, +): """ Set synthetic worst-case active-power time series. @@ -62,8 +69,7 @@ def task_worst_case_ts(edisgo, ctx, *, cases=None, @register_task("set_timeindex") -def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, - freq="h"): +def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, freq="h"): """ Set the time index on the EDisGo object. @@ -98,22 +104,32 @@ def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, If neither ``periods`` nor ``end`` is provided. """ + from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex + if end is not None: timeindex = pd.date_range(start=start, end=end, freq=freq) else: if periods is None: - raise ValueError( - "set_timeindex needs either 'periods' or 'end'." - ) + raise ValueError("set_timeindex needs either 'periods' or 'end'.") timeindex = pd.date_range(start=start, periods=periods, freq=freq) - edisgo.set_timeindex(timeindex) + if edisgo.timeseries.timeindex.empty: + edisgo.set_timeindex(timeindex) + else: + reduce_timeseries_data_to_given_timeindex(edisgo, timeindex) return edisgo @register_task("oedb_ts") -def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, - fluctuating="oedb", conventional_loads="oedb", - charging_points_ts=None): +def task_oedb_ts( + edisgo, + ctx, + *, + timeindex=None, + dispatchable=None, + fluctuating="oedb", + conventional_loads="oedb", + charging_points_ts=None, +): """ Set active-power time series from egon_data (OEP) plus overrides. @@ -174,9 +190,7 @@ def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, conv_loads_names = None if conventional_loads == "oedb": conv_loads_names = edisgo.topology.loads_df.loc[ - ~edisgo.topology.loads_df.type.isin( - ["heat_pump", "charging_point"] - ) + ~edisgo.topology.loads_df.type.isin(["heat_pump", "charging_point"]) ].index.tolist() edisgo.set_time_series_active_power_predefined( @@ -186,27 +200,34 @@ def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, dispatchable_generators_ts=dispatchable_df, charging_points_ts=charging_points_ts, scenario=ctx.scenario, - engine=ctx.ensure_engine() if fluctuating == "oedb" - or conventional_loads == "oedb" else None, + engine=ctx.ensure_engine() + if fluctuating == "oedb" or conventional_loads == "oedb" + else None, ) su_names = edisgo.topology.storage_units_df.index if len(su_names) > 0 and edisgo.timeseries.storage_units_active_power.empty: edisgo.timeseries.storage_units_active_power = pd.DataFrame( - 0.0, index=edisgo.timeseries.timeindex, columns=su_names, + 0.0, + index=edisgo.timeseries.timeindex, + columns=su_names, ) ctx.flags["timeseries_set"] = True return edisgo @register_task("manual_ts") -def task_manual_ts(edisgo, ctx, *, - generators_active_power=None, - generators_reactive_power=None, - loads_active_power=None, - loads_reactive_power=None, - storage_units_active_power=None, - storage_units_reactive_power=None): +def task_manual_ts( + edisgo, + ctx, + *, + generators_active_power=None, + generators_reactive_power=None, + loads_active_power=None, + loads_reactive_power=None, + storage_units_active_power=None, + storage_units_reactive_power=None, +): """ Set active/reactive power time series from explicit DataFrames. @@ -240,6 +261,7 @@ def task_manual_ts(edisgo, ctx, *, The modified EDisGo instance. """ + def _as_df(obj): return pd.DataFrame(obj) if obj is not None else None @@ -256,10 +278,15 @@ def _as_df(obj): @register_task("reactive_power") -def task_reactive_power(edisgo, ctx, *, control="fixed_cosphi", - generators_parametrisation="default", - loads_parametrisation="default", - storage_units_parametrisation="default"): +def task_reactive_power( + edisgo, + ctx, + *, + control="fixed_cosphi", + generators_parametrisation="default", + loads_parametrisation="default", + storage_units_parametrisation="default", +): """ Apply reactive-power control on top of the active-power time series. From b55e5a03f841cdec2366d8f2177158ea3174f2b5 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Thu, 21 May 2026 16:26:21 +0200 Subject: [PATCH 09/66] Fix OPF result write-back, flex resolution and overlying-grid SOC reindex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit powermodels_io.from_powermodels now slices the destination time index explicitly when writing OPF flex results back to the EDisGo timeseries (gen_nd, heatpumps, electromobility, dsm, storage). Using `loc[:, names]` overwrites every row of the underlying DataFrame even when the OPF only covered a subset of timesteps; restricting to `timeseries.timeindex` keeps untouched rows intact. task_optimize: - fixes typo `flexbile` → `flexible` in the dsm shortcut - corrects the dsm condition (was checking `is not None`, should populate when `None`) - materializes empty flex lists once instead of repeating `or []` at every call site task_import_overlying_grid_data reindexes the three SOC attributes (storage_units_soc, thermal_storage_units_{central,decentral}_soc) to timeindex + 1 extra step, because PowerModels expects the end-of-period SOC value; non-SOC overlying-grid attributes still reindex to the plain timeindex. uc4_example preset reworked: switches to opf_version 3 (HV constraints from overlying grid), drops base_reinforce + check_integrity from the pipeline, adds an explicit `overlying_grid.path` config slot, splits import_electromobility kwargs onto separate lines, and enables archive + save_opf_results on the final save step. --- edisgo/io/powermodels_io.py | 33 +++++++----- edisgo/run/presets/uc4_example.yaml | 47 ++++++++--------- edisgo/run/tasks/analysis.py | 79 +++++++++++++++++++++-------- edisgo/run/tasks/io.py | 14 +++-- 4 files changed, 108 insertions(+), 65 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index e82cc4734..6bf21affb 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -250,10 +250,10 @@ def from_powermodels( Base value of apparent power for per unit system. Default: 1 MVA. """ - if type(pm_results) == str: + if isinstance(pm_results, str): with open(pm_results) as f: pm = json.loads(json.load(f)) - elif type(pm_results) == dict: + elif isinstance(pm_results, dict): pm = pm_results else: raise ValueError( @@ -306,17 +306,20 @@ def from_powermodels( ] results = pd.DataFrame(index=timesteps, columns=names, data=data) if (flex == "gen_nd") & (pm["nw"]["1"]["opf_version"] in [3, 4]): - edisgo_object.timeseries._generators_active_power.loc[:, names] = ( + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._generators_active_power.loc[ti, names] = ( edisgo_object.timeseries.generators_active_power.loc[:, names].values - results[names].values ) elif flex in ["heatpumps", "electromobility"]: - edisgo_object.timeseries._loads_active_power.loc[:, names] = results[ + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._loads_active_power.loc[ti, names] = results[ names ].values elif flex == "dsm": - edisgo_object.timeseries._loads_active_power.loc[:, names] = ( - edisgo_object.timeseries._loads_active_power.loc[:, names].values + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._loads_active_power.loc[ti, names] = ( + edisgo_object.timeseries._loads_active_power.loc[ti, names].values + results[names].values ) elif flex == "storage": @@ -328,8 +331,9 @@ def from_powermodels( data=results[names].values, ) else: + ti = edisgo_object.timeseries.timeindex edisgo_object.timeseries._storage_units_active_power.loc[ - :, names + ti, names ] = results[names].values except AttributeError: setattr( @@ -787,8 +791,8 @@ def _build_branch(edisgo_obj, psa_net, pm, flexible_storage_units, s_base): # only modify r, x and l values if min value is too small branches[par] = val.clip(lower=min_value) logger.warning( - f"Min value of {text} is too small. Lowest {100 * quant}% of {text} values will be set " - f"to {min_value} {unit}" + f"Min value of {text} is too small. Lowest {100 * quant}% of " + f"{text} values will be set to {min_value} {unit}" ) for branch_i in np.arange(len(branches.index)): @@ -933,8 +937,8 @@ def _build_load( pf, sign = _get_pf(edisgo_obj, pm, idx_bus, "charging_point") else: logger.warning( - f"No type specified for load {loads_df.index[load_i]}. Power factor and sign will" - "be set for conventional load." + f"No type specified for load {loads_df.index[load_i]}. " + "Power factor and sign will be set for conventional load." ) pf, sign = _get_pf(edisgo_obj, pm, idx_bus, "conventional_load") p_d = psa_net.loads_t.p_set[loads_df.index[load_i]] @@ -1219,9 +1223,10 @@ def _build_heatpump(psa_net, pm, edisgo_obj, s_base, flexible_hps): comparison = (heat_df2[hp_p_nom.index] > hp_cop * hp_p_nom.squeeze()).any() if comparison.any(): logger.warning( - "Heat demand is higher than rated heatpump power" - f" of heatpumps: {comparison.index[comparison.values].values}. Demand can not be covered if no sufficient" - " heat storage capacities are available." + "Heat demand is higher than rated heatpump power of heatpumps: " + f"{comparison.index[comparison.values].values}. " + "Demand can not be covered if no sufficient heat storage " + "capacities are available." ) for hp_i in np.arange(len(heat_df.index)): idx_bus = _mapping(psa_net, edisgo_obj, heat_df.bus.iloc[hp_i]) diff --git a/edisgo/run/presets/uc4_example.yaml b/edisgo/run/presets/uc4_example.yaml index 2d2650279..d3f9efd90 100644 --- a/edisgo/run/presets/uc4_example.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -1,54 +1,51 @@ _comment: | - UC3 — OPF with full flexibility: - Like UC1 but loads real egon_data time series (oedb) and runs a - powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) - before the final reinforce. Cost delta = extra reinforcement needed - under optimal flex dispatch. + UC4 — OPF with full flexibility: + Loads real egon_data time series (oedb) and runs a powermodels OPF + over flexibilities (heat pumps, EV, DSM, storage) with HV requirements + from overlying grid. opf_version 3 activates HV-constraints from + overlying_grid CSV directory. _workflow: - - setup_grid: load ding0 topology, import generators - - base_reinforce: worst-case TS + reinforce + reset equipment_changes - - import_generators: from edon-data - - import_heat_pumps: from egon_data + - setup_grid: load ding0 topology + - import_generators: from egon_data - import_home_batteries: from egon_data + - import_heat_pumps: from egon_data - import_dsm: from egon_data - import_electromobility: from egon_data (dumb charging, flex bands) - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_charging_strategy: dumb - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) - - reactive_power - - check_integrity - - optimize: pm_optimize with flex assets (SOC, opf v2) - - reinforce: final reinforcement - - save + - import_overlying_grid_data: HV constraints from CSV dir + - optimize: pm_optimize with flex assets (SOC, opf v3 = HV constraints) scenario: eGon2035 + grid: ding0_path: "/path/to/ding0_grid" - # grid path should be set in run file legacy_ding0_grids: false - # legacy parameter too database: ssh: enabled: false timeindex: {start: "2035-01-01", periods: 24, freq: h} -# only set in preset, not set in run-functions (eDisGo and eGo) + +overlying_grid: + path: "/path/to/overlying_grid_csv_dir" results: directory: results/uc4_example -# actually all parameters pipeline: - setup_grid - - base_reinforce - import_generators - import_home_batteries - import_heat_pumps - import_dsm - # where is decided, which electromobility use cases are used? - - import_electromobility: {charging_strategy: null, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} + - import_electromobility: + charging_strategy: null + flexibility_bands_ucs: ["home", "work", "public", "hpc"] - oedb_ts: dispatchable: {other: 0.7} timeindex: {start: "2035-01-01", periods: 24, freq: h} @@ -56,11 +53,11 @@ pipeline: - apply_heat_pump_strategy: {strategy: uncontrolled} - import_overlying_grid_data - reactive_power - - check_integrity - optimize: - # where is decided which flexibilities are used in the OPF? flexible: [heat_pumps, storage, charging_points, dsm] method: soc - opf_version: 2 + opf_version: 3 - reinforce - - save + - save: + archive: true + save_opf_results: true diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py index f028bb31c..becff93b9 100644 --- a/edisgo/run/tasks/analysis.py +++ b/edisgo/run/tasks/analysis.py @@ -21,6 +21,7 @@ produce a "base" grid whose subsequent reinforce costs reflect only a scenario overlay. """ + from __future__ import annotations import pandas as pd @@ -55,8 +56,15 @@ def task_check_integrity(edisgo, ctx): @register_task("analyze") -def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, - raise_not_converged=False, troubleshooting_mode=None): +def task_analyze( + edisgo, + ctx, + *, + mode=None, + timesteps=None, + raise_not_converged=False, + troubleshooting_mode=None, +): """ Run AC power flow over the active time series. @@ -97,18 +105,26 @@ def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, ctx.flags["not_converged_steps"] = len(not_converged) if len(not_converged) > 0: ctx.logger.warning( - f"Power flow did not converge for {len(not_converged)} " - f"time steps." + f"Power flow did not converge for {len(not_converged)} time steps." ) return edisgo @register_task("reinforce") -def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, - copy_grid=False, max_while_iterations=20, - split_voltage_band=True, mode=None, - without_generator_import=False, n_minus_one=False, - catch_convergence_problems=False): +def task_reinforce( + edisgo, + ctx, + *, + timesteps_pfa=None, + reduced_analysis=False, + copy_grid=False, + max_while_iterations=20, + split_voltage_band=True, + mode=None, + without_generator_import=False, + n_minus_one=False, + catch_convergence_problems=False, +): """ Run iterative grid reinforcement. @@ -167,8 +183,9 @@ def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, @register_task("base_reinforce") -def task_base_reinforce(edisgo, ctx, *, cases=None, - reset_equipment_changes=True, save_artifact=True): +def task_base_reinforce( + edisgo, ctx, *, cases=None, reset_equipment_changes=True, save_artifact=True +): """ Produce a base-reinforced grid and reset the cost accumulator. @@ -242,10 +259,20 @@ def task_base_reinforce(edisgo, ctx, *, cases=None, @register_task("optimize") -def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, - flexible_hps=None, flexible_loads=None, - flexible_storage_units=None, opf_version=2, method="soc", - warm_start=False, s_base=1): +def task_optimize( + edisgo, + ctx, + *, + flexible=None, + flexible_cps=None, + flexible_hps=None, + flexible_loads=None, + flexible_storage_units=None, + opf_version=2, + method="soc", + warm_start=False, + s_base=1, +): """ Run a powermodels optimal-power-flow (OPF) over flexibilities. @@ -304,15 +331,23 @@ def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, ].index.tolist() if flexible_storage_units is None and "storage" in flexible: flexible_storage_units = edisgo.topology.storage_units_df.index.tolist() - if flexible_loads is not None and "dsm" in flexbile: - flexible_loads = edisgo.dsm.p_min.columns.values - + if flexible_loads is None and "dsm" in flexible: + flexible_loads = edisgo.dsm.p_min.columns.values + + if flexible_cps is None: + flexible_cps = [] + if flexible_hps is None: + flexible_hps = [] + if flexible_loads is None: + flexible_loads = [] + if flexible_storage_units is None: + flexible_storage_units = [] edisgo.pm_optimize( - flexible_cps=flexible_cps or [], - flexible_hps=flexible_hps or [], - flexible_loads=flexible_loads or [], - flexible_storage_units=flexible_storage_units or [], + flexible_cps=flexible_cps, + flexible_hps=flexible_hps, + flexible_loads=flexible_loads, + flexible_storage_units=flexible_storage_units, opf_version=opf_version, method=method, warm_start=warm_start, diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 4e2e84d15..c2b748cf0 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -262,6 +262,14 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): # CSVs may use a different year — shift year then reindex edisgo_ti = edisgo.timeseries.timeindex if not edisgo_ti.empty: + # SOC needs one extra step at the end (end-of-period state) + ti_freq = edisgo_ti.freq or (edisgo_ti[1] - edisgo_ti[0]) + edisgo_ti_plus1 = edisgo_ti.union([edisgo_ti[-1] + ti_freq]) + soc_attrs = { + "storage_units_soc", + "thermal_storage_units_decentral_soc", + "thermal_storage_units_central_soc", + } for attr in edisgo.overlying_grid._attributes: ts = getattr(edisgo.overlying_grid, attr) if ts.empty: @@ -270,10 +278,8 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): edisgo_year = edisgo_ti[0].year if csv_year != edisgo_year: ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) - if isinstance(ts, pd.Series): - setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) - else: - setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti + setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) # load dispatchable generator and renewables time series from the same dir disp_path = os.path.join( From bb3a720a09e78db8693bac72322e816a2b6f8694 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:15:42 +0200 Subject: [PATCH 10/66] Changes by JD before edits of MS --- edisgo/edisgo.py | 21 + edisgo/run/__init__.py | 31 ++ edisgo/run/config.py | 410 ++++++++++++++++++ edisgo/run/context.py | 115 +++++ edisgo/run/presets/basic.yaml | 22 + .../run/presets/r4mu_base_and_scenario.yaml | 49 +++ edisgo/run/presets/uc1_loads_worst_case.yaml | 32 ++ edisgo/run/presets/uc2_flex_opf.yaml | 43 ++ edisgo/run/presets/uc3_oedb_ts.yaml | 36 ++ edisgo/run/registry.py | 113 +++++ edisgo/run/runner.py | 261 +++++++++++ edisgo/run/tasks/__init__.py | 25 ++ edisgo/run/tasks/io.py | 179 ++++++++ edisgo/run/validator.py | 201 +++++++++ setup.py | 1 + tests/run/__init__.py | 1 + tests/run/test_config.py | 154 +++++++ tests/run/test_registry.py | 35 ++ tests/run/test_runner.py | 118 +++++ tests/run/test_validator.py | 97 +++++ 20 files changed, 1944 insertions(+) create mode 100644 edisgo/run/__init__.py create mode 100644 edisgo/run/config.py create mode 100644 edisgo/run/context.py create mode 100644 edisgo/run/presets/basic.yaml create mode 100644 edisgo/run/presets/r4mu_base_and_scenario.yaml create mode 100644 edisgo/run/presets/uc1_loads_worst_case.yaml create mode 100644 edisgo/run/presets/uc2_flex_opf.yaml create mode 100644 edisgo/run/presets/uc3_oedb_ts.yaml create mode 100644 edisgo/run/registry.py create mode 100644 edisgo/run/runner.py create mode 100644 edisgo/run/tasks/__init__.py create mode 100644 edisgo/run/tasks/io.py create mode 100644 edisgo/run/validator.py create mode 100644 tests/run/__init__.py create mode 100644 tests/run/test_config.py create mode 100644 tests/run/test_registry.py create mode 100644 tests/run/test_runner.py create mode 100644 tests/run/test_validator.py diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index ed6ed376d..80579bf48 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -232,6 +232,27 @@ def config(self): def config(self, kwargs): self._config = Config(**kwargs) + def run_pipeline(self, config): + """ + Run a YAML/JSON task pipeline on this EDisGo instance. + + See :mod:`edisgo.run` for the config schema and task list. + + Parameters + ---------- + config : str, :class:`pathlib.Path`, or dict + Pipeline config as path to a YAML/JSON file or as a dict. + + Returns + ------- + :class:`~.EDisGo` + The EDisGo instance after the pipeline has run. + + """ + from edisgo.run import _run_pipeline_on + + return _run_pipeline_on(self, config) + def import_ding0_grid(self, path, legacy_ding0_grids=True): """ Import ding0 topology data from csv files in the format as diff --git a/edisgo/run/__init__.py b/edisgo/run/__init__.py new file mode 100644 index 000000000..cd202ddef --- /dev/null +++ b/edisgo/run/__init__.py @@ -0,0 +1,31 @@ +""" +YAML/JSON-driven pipeline runner for eDisGo. + +Two entry points share the same core: + + from edisgo.run import run_edisgo + edisgo = run_edisgo("presets/uc2_flex_opf.yaml") + + # or, on an existing EDisGo instance: + edisgo = EDisGo(ding0_grid="30879") + edisgo.run_pipeline("my_run.yaml") + +Pipelines are lists of named tasks from :mod:`edisgo.run.tasks`. Each step +is either a string (``worst_case_ts``) or a single-key mapping with +parameters (``import_electromobility: {charging_strategy: dumb}``). Tasks +can be grouped into ordered ``stages`` that can save artifacts and reload +them with ``load_from``, enabling two-phase workflows (base reinforce + +per-scenario reinforce). +""" + +from edisgo.run.context import RunContext +from edisgo.run.registry import known_tasks, register_task +from edisgo.run.runner import _run_pipeline_on, run_edisgo + +__all__ = [ + "RunContext", + "_run_pipeline_on", + "known_tasks", + "register_task", + "run_edisgo", +] diff --git a/edisgo/run/config.py b/edisgo/run/config.py new file mode 100644 index 000000000..5c4ee8573 --- /dev/null +++ b/edisgo/run/config.py @@ -0,0 +1,410 @@ +""" +Config loader and schema normalizer for the eDisGo pipeline runner. + +The loader turns a YAML file, JSON file, or Python dict into the +canonical internal schema consumed by :mod:`edisgo.run.runner`. It +handles four concerns in a fixed order: + +1. **Read** — parse YAML/JSON (auto-detected by extension; unknown + extensions are tried as JSON first, then YAML). +2. **extends** — resolve a ``extends:`` key recursively into the + parent config and deep-merge; the child overrides parent keys. The + ``extends:`` value may be a path (relative to the including file) + or a bare preset name (resolved against + :mod:`edisgo.run.presets`). +3. **external_config** — merge machine-specific overrides from an + ``external_config:`` path (typically ``~/.edisgo/secrets.json`` + with DB credentials). Keys in the external file override keys in + the main config. +4. **eGo-legacy adaptation** — if the config looks like an eGo + ``scenario_setting_*.json`` (has top-level ``eDisGo.tasks``), map + it onto the new schema so old eGo configs run unchanged. +5. **Stage normalization** — collapse a flat ``pipeline:`` into a + single-stage ``stages: [{name: main, pipeline: [...]}]`` so the + runner only ever deals with the stage form. + +Only :func:`load_config` is public. Everything else is implementation +detail. +""" +from __future__ import annotations + +import copy +import json +import logging +import os + +from pathlib import Path +from typing import Any + +import yaml + +logger = logging.getLogger("edisgo.run.config") + + +def load_config(cfg_or_path) -> dict[str, Any]: + """ + Load, merge, adapt, and normalize a pipeline config. + + Accepts a path to a YAML/JSON file or a dict. The returned dict + always has the normalized shape expected by the runner: + + * top-level ``stages`` (list of ``{name, pipeline, ...}``) + * ``scenario`` (may be ``None``) + * optional ``grid``, ``database``, ``results`` sections + * no ``pipeline``, ``extends``, or ``external_config`` keys + (they have been consumed) + + Parameters + ---------- + cfg_or_path : str, pathlib.Path, or dict + Either a path to a YAML/JSON config file, or a dict already + holding the config. A dict is deep-copied so the caller's + dict is not mutated. + + Returns + ------- + dict + The fully resolved, normalized config. + + Raises + ------ + FileNotFoundError + If the given path (or an ``extends`` reference) does not + exist. + ValueError + If the config has both ``pipeline`` and ``stages``, missing + ``pipeline``/``stages``, duplicate stage names, or a stage + without ``name``/``pipeline``. + + """ + if isinstance(cfg_or_path, (dict,)): + cfg = copy.deepcopy(cfg_or_path) + base_dir = Path.cwd() + else: + path = Path(cfg_or_path).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f"Config file not found: {path}") + cfg = _read_file(path) + base_dir = path.parent + + cfg = _resolve_extends(cfg, base_dir) + cfg = _apply_external_config(cfg) + cfg = _adapt_ego_legacy(cfg) + cfg = _normalize_stages(cfg) + return cfg + + +def _read_file(path: Path) -> dict[str, Any]: + """ + Parse a YAML or JSON file into a dict. + + Parameters + ---------- + path : pathlib.Path + File path. Extension (``.json``, ``.yaml``, ``.yml``) selects + the parser. Unknown extensions fall back to JSON first, then + YAML. + + Returns + ------- + dict + Parsed config contents. + + """ + text = path.read_text() + suffix = path.suffix.lower() + if suffix == ".json": + return json.loads(text) + if suffix in (".yaml", ".yml"): + return yaml.safe_load(text) + try: + return json.loads(text) + except json.JSONDecodeError: + return yaml.safe_load(text) + + +def _resolve_extends(cfg: dict, base_dir: Path) -> dict: + """ + Resolve an ``extends:`` reference and deep-merge parent into child. + + The parent is loaded recursively, so a chain of ``extends:`` works. + References are looked up as (1) a bundled preset name under + :mod:`edisgo.run.presets`, (2) a path relative to ``base_dir``. + The child's keys override the parent's on conflicts. + + Parameters + ---------- + cfg : dict + Child config (may contain ``extends:``). + base_dir : pathlib.Path + Directory against which relative ``extends`` paths are + resolved (usually the directory of the child config). + + Returns + ------- + dict + Merged config with ``extends`` consumed. + + Raises + ------ + FileNotFoundError + If the referenced parent config file does not exist. + + """ + ext = cfg.pop("extends", None) + if ext is None: + return cfg + ext_path = Path(ext).expanduser() + if not ext_path.is_absolute(): + preset_path = _preset_path(str(ext_path)) + if preset_path is not None: + ext_path = preset_path + else: + ext_path = (base_dir / ext_path).resolve() + if not ext_path.is_file(): + raise FileNotFoundError(f"extends: file not found: {ext_path}") + parent = _read_file(ext_path) + parent = _resolve_extends(parent, ext_path.parent) + return _deep_merge(parent, cfg) + + +def _preset_path(name: str) -> Path | None: + """ + Look up a preset YAML/JSON by bare name. + + Searches the ``edisgo/run/presets/`` directory for a file matching + ``name``, ``name.yaml``, ``name.yml``, or ``name.json`` (in that + order). + + Parameters + ---------- + name : str + Preset identifier, e.g. ``"uc2_flex_opf"`` or + ``"presets/uc2_flex_opf.yaml"``. + + Returns + ------- + pathlib.Path or None + The resolved preset path, or ``None`` if no match is found. + + """ + presets_dir = Path(__file__).parent / "presets" + candidates = [ + presets_dir / name, + presets_dir / f"{name}.yaml", + presets_dir / f"{name}.yml", + presets_dir / f"{name}.json", + ] + for c in candidates: + if c.is_file(): + return c + return None + + +def _apply_external_config(cfg: dict) -> dict: + """ + Merge an ``external_config:`` file on top of the current config. + + Used to keep machine-specific secrets (DB credentials, result + directories) out of versioned scenario configs. If the referenced + file does not exist, a warning is logged but the config is used + as-is. + + Parameters + ---------- + cfg : dict + Config possibly containing an ``external_config:`` key. + + Returns + ------- + dict + Merged config with ``external_config`` consumed. + + """ + ext = cfg.pop("external_config", None) + if ext is None: + return cfg + path = Path(os.path.expanduser(ext)) + if not path.is_file(): + logger.warning(f"external_config file not found, skipping: {path}") + return cfg + override = _read_file(path) + return _deep_merge(cfg, override) + + +def _deep_merge(base: dict, override: dict) -> dict: + """ + Recursively merge two dicts, with ``override`` winning on conflicts. + + Nested dicts are merged key-by-key. Non-dict values (including + lists) are replaced wholesale — lists are NOT concatenated, to + keep the merge semantics predictable (otherwise a preset could + silently extend the child's pipeline). + + Parameters + ---------- + base : dict + Parent / lower-priority dict. + override : dict + Child / higher-priority dict. + + Returns + ------- + dict + A new dict holding the merge result. Inputs are not mutated. + + """ + out = copy.deepcopy(base) if base else {} + for key, val in (override or {}).items(): + if ( + key in out + and isinstance(out[key], dict) + and isinstance(val, dict) + ): + out[key] = _deep_merge(out[key], val) + else: + out[key] = copy.deepcopy(val) + return out + + +def _normalize_stages(cfg: dict) -> dict: + """ + Collapse a flat ``pipeline:`` into the canonical ``stages`` shape. + + After this step the runner only has to iterate ``cfg["stages"]``; + flat configs become a single stage named ``main``. + + Parameters + ---------- + cfg : dict + Config with either ``pipeline`` or ``stages`` at the top + level. + + Returns + ------- + dict + Config with ``stages`` guaranteed to be present and + ``pipeline`` removed. + + Raises + ------ + ValueError + If both ``pipeline`` and ``stages`` are present, if neither + is present, if any stage is missing ``name``/``pipeline``, or + if stage names are not unique. + + """ + if "stages" in cfg and "pipeline" in cfg: + raise ValueError( + "Config has both top-level 'pipeline' and 'stages'. " + "Use only one." + ) + if "stages" not in cfg: + pipeline = cfg.pop("pipeline", None) + if pipeline is None: + raise ValueError( + "Config must define either 'pipeline' or 'stages'." + ) + cfg["stages"] = [{"name": "main", "pipeline": pipeline}] + + seen = set() + for stage in cfg["stages"]: + if "name" not in stage: + raise ValueError("Every stage needs a 'name' key.") + if stage["name"] in seen: + raise ValueError( + f"Duplicate stage name: {stage['name']}" + ) + seen.add(stage["name"]) + if "pipeline" not in stage: + raise ValueError( + f"Stage '{stage['name']}' is missing 'pipeline'." + ) + return cfg + + +_EGO_TASK_MAP = { + "1_setup_grid": "setup_grid", + "5_grid_reinforcement": "reinforce", + "4_optimisation": "optimize", + "worst_case_ts": "worst_case_ts", + "base_reinforce": "base_reinforce", + "oedb_ts": "oedb_ts", + "import_heat_pumps_from_db": "import_heat_pumps", + "import_home_batteries_from_db": "import_home_batteries", + "import_dsm_from_db": "import_dsm", + "import_electromobility_from_db": "import_electromobility", + "load_charging_from_files": "load_charging_from_files", + "load_from_base": "load_from_base", +} +"""Mapping from eGo task names to edisgo.run task names. eGo-specific +tasks with no eDisGo equivalent (e.g. ``2_specs_overlying_grid``, +``3_temporal_complexity_reduction``) are intentionally missing — they +require eTraGo and are logged as "skipped" when adapted.""" + + +def _adapt_ego_legacy(cfg: dict) -> dict: + """ + Map an eGo-style ``scenario_setting_*.json`` onto the new schema. + + Recognizes an eGo config by the presence of an ``eDisGo.tasks`` + key at the top level together with the absence of + ``pipeline``/``stages``. Translates: + + * ``eDisGo.grid_path`` → ``grid.ding0_path`` + * ``eDisGo.results`` → ``results.directory`` + * ``eTraGo.scn_name`` → ``scenario`` + * ``eDisGo.tasks`` → ``pipeline`` (via :data:`_EGO_TASK_MAP`) + * top-level ``database``/``ssh`` kept under ``database`` + + eGo-only tasks (overlying grid / temporal reduction) are + dropped with a warning. Cosmetic keys (``eGo``, ``eTraGo``, + ``_comment``, ``_workflow``) are stripped. + + Parameters + ---------- + cfg : dict + Possibly-legacy config. + + Returns + ------- + dict + Adapted config. If the input is not an eGo-legacy config, it + is returned unchanged. + + """ + if "eDisGo" not in cfg or "pipeline" in cfg or "stages" in cfg: + return cfg + + edisgo_cfg = cfg["eDisGo"] + tasks = edisgo_cfg.get("tasks") + if tasks is None: + return cfg + + logger.info( + "Detected legacy eGo config schema — adapting to edisgo.run." + ) + mapped = [] + for t in tasks: + if t not in _EGO_TASK_MAP: + logger.warning( + f"eGo task '{t}' has no eDisGo equivalent — skipping " + "(likely eTraGo-specific)." + ) + continue + mapped.append(_EGO_TASK_MAP[t]) + + adapted: dict[str, Any] = { + "scenario": cfg.get("eTraGo", {}).get("scn_name", "eGon2035"), + "grid": {"ding0_path": edisgo_cfg.get("grid_path")}, + "results": {"directory": edisgo_cfg.get("results")}, + "pipeline": mapped, + } + if "database" in cfg: + adapted["database"] = cfg["database"] + if "ssh" in cfg: + adapted["database"]["ssh"] = cfg["ssh"] + for side_key in ("eGo", "eTraGo", "ssh", "_comment", "_workflow"): + cfg.pop(side_key, None) + cfg.pop("eDisGo", None) + return _deep_merge(adapted, cfg) diff --git a/edisgo/run/context.py b/edisgo/run/context.py new file mode 100644 index 000000000..c2fbce234 --- /dev/null +++ b/edisgo/run/context.py @@ -0,0 +1,115 @@ +""" +Runtime context passed to every task during pipeline execution. + +The context is a small mutable object that threads shared state between +tasks without polluting the :class:`~edisgo.EDisGo` instance itself. +Typical uses: + +* ``scenario`` — the active eGon scenario name (``eGon2035``, + ``eGon100RE``, …) so tasks don't have to re-read it from the config. +* ``engine`` — a SQLAlchemy engine, lazily created on first DB access + via :meth:`RunContext.ensure_engine`. Tasks that don't touch the + database never pay connection cost. +* ``results_dir`` — base directory for stage artifacts and ``save``. +* ``flags`` — free-form boolean/state flags tasks set to coordinate + with each other (``has_heat_pumps``, ``timeseries_set``, …). +* ``stage_artifacts`` — map ``stage_name -> path`` of zip/dir artifacts + emitted by ``save``, consumed by later stages via ``load_from``. + +Tasks should treat ``flags`` as advisory — they MAY short-circuit based +on a flag but MUST NOT assume a flag is present. +""" +from __future__ import annotations + +import logging + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass +class RunContext: + """ + Mutable per-run state shared across all tasks of a pipeline. + + Attributes + ---------- + scenario : str or None + Active scenario name from the top-level ``scenario:`` key. + engine : sqlalchemy.engine.Engine or None + Database engine for oedb-backed imports. Created lazily; + see :meth:`ensure_engine`. + results_dir : pathlib.Path or None + Base directory for stage outputs. Resolved from + ``results.directory`` in the config. + logger : logging.Logger + Logger instance used by tasks and the runner. Defaults to + the ``edisgo.run`` logger. + flags : dict + Free-form state flags that tasks use to communicate. Common + keys: ``grid_loaded``, ``timeseries_set``, + ``reactive_power_set``, ``has_heat_pumps``, ``has_dsm``, + ``has_home_batteries``, ``has_electromobility``, + ``base_reinforced``, ``last_saved``. + stage_artifacts : dict + Map ``stage_name -> Path`` of save-artifacts. Populated by the + ``save`` task when running inside a named stage, consumed by + subsequent stages that set ``load_from:``. + current_stage : str or None + Name of the stage currently executing. Set by the runner. + raw_config : dict + The fully resolved pipeline config (after ``extends``, + ``external_config``, and eGo-legacy adaptation). Tasks can + read supplementary keys like ``database.*`` from here. + + """ + + scenario: str | None = None + engine: Any = None + results_dir: Path | None = None + logger: logging.Logger = field( + default_factory=lambda: logging.getLogger("edisgo.run") + ) + flags: dict[str, Any] = field(default_factory=dict) + stage_artifacts: dict[str, Path] = field(default_factory=dict) + current_stage: str | None = None + raw_config: dict[str, Any] = field(default_factory=dict) + + def ensure_engine(self): + """ + Return a database engine, creating it on first call. + + Reads the ``database`` section of :attr:`raw_config` and calls + :func:`edisgo.io.db.engine`. Caches the engine on the context + so subsequent calls reuse the same connection. + + Returns + ------- + sqlalchemy.engine.Engine + The active database engine. + + Raises + ------ + RuntimeError + If the config has no ``database`` section — indicates the + pipeline wants to reach the database without configuring + it. + + """ + if self.engine is not None: + return self.engine + db_cfg = self.raw_config.get("database") + if not db_cfg: + raise RuntimeError( + "Task needs a database engine but no 'database' section " + "is configured." + ) + from edisgo.io.db import engine as egon_engine + + ssh_cfg = db_cfg.get("ssh") or {} + self.engine = egon_engine( + path=db_cfg.get("credentials_path"), + ssh=bool(ssh_cfg.get("enabled", False)), + ) + return self.engine diff --git a/edisgo/run/presets/basic.yaml b/edisgo/run/presets/basic.yaml new file mode 100644 index 000000000..136161855 --- /dev/null +++ b/edisgo/run/presets/basic.yaml @@ -0,0 +1,22 @@ +_comment: | + Basic preset: worst-case pre-reinforce → reinforce. + Minimal end-to-end example with no database dependency. + Reproduces the core of example_01 without flex imports. + +_workflow: + - setup_grid: load ding0 topology + - worst_case_ts: set worst-case time series (feed-in + load) + - reactive_power: fix reactive power control + - check_integrity: validate grid consistency + - reinforce: run grid reinforcement + - save: persist topology + timeseries + results + +scenario: eGon2035 + +pipeline: + - setup_grid + - worst_case_ts + - reactive_power + - check_integrity + - reinforce + - save diff --git a/edisgo/run/presets/r4mu_base_and_scenario.yaml b/edisgo/run/presets/r4mu_base_and_scenario.yaml new file mode 100644 index 000000000..6412f36ca --- /dev/null +++ b/edisgo/run/presets/r4mu_base_and_scenario.yaml @@ -0,0 +1,49 @@ +_comment: | + R4MU — two-stage base + scenario reinforcement: + Stage 1 produces a base-reinforced grid (generators + heat pumps) + and saves it as an artifact. Stage 2 loads that artifact, integrates + scenario-specific charging stations from a GeoPackage/CSV directory, + applies worst-case time series, and runs a scenario-specific + reinforce. Cost delta = extra reinforcement caused by the charging + scenario. + +_workflow: + - stage base: + - setup_grid: load ding0 topology + import generators + - import_heat_pumps: from egon_data + - worst_case_ts + - reactive_power + - reinforce + - save (artifact consumed by next stage) + - stage scenario: + - load_from: base + - load_charging_from_files: integrate scenario charging + - worst_case_ts + - reactive_power + - reinforce (delta only) + - save + +scenario: eGon2035 + +stages: + - name: base + pipeline: + - setup_grid: {import_generators: true} + - import_heat_pumps + - worst_case_ts + - reactive_power + - reinforce + - save + - name: scenario + load_from: base + params: + charging_dir: "./charging_scenario_1" + mv_threshold_kw: 100 + pipeline: + - load_charging_from_files: + charging_dir: "{{params.charging_dir}}" + mv_threshold_kw: "{{params.mv_threshold_kw}}" + - worst_case_ts + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc1_loads_worst_case.yaml b/edisgo/run/presets/uc1_loads_worst_case.yaml new file mode 100644 index 000000000..2204d3ce9 --- /dev/null +++ b/edisgo/run/presets/uc1_loads_worst_case.yaml @@ -0,0 +1,32 @@ +_comment: | + UC1 — worst-case flexibility loads: + load grid, base-reinforce (generators only), then import flex assets + (heat pumps, home batteries, DSM, electromobility) and apply worst-case + time series before a final reinforce. Cost delta = extra reinforcement + caused by the new assets under worst-case conditions. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging) + - worst_case_ts: synthetic worst case incl. new assets + - reactive_power: fix reactive power control + - reinforce: final reinforcement — delta only + - save: persist topology + results + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - worst_case_ts + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc2_flex_opf.yaml b/edisgo/run/presets/uc2_flex_opf.yaml new file mode 100644 index 000000000..c09cae87e --- /dev/null +++ b/edisgo/run/presets/uc2_flex_opf.yaml @@ -0,0 +1,43 @@ +_comment: | + UC2 — OPF with full flexibility: + Like UC1 but loads real egon_data time series (oedb) and runs a + powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) + before the final reinforce. Cost delta = extra reinforcement needed + under optimal flex dispatch. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (168 h, 2035) + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - reactive_power + - check_integrity + - optimize: pm_optimize with flex assets (SOC, opf v2) + - reinforce: final reinforcement + - save + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - oedb_ts: + timeindex: {start: "2035-01-01", periods: 168, freq: h} + dispatchable: {other: 0.7} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - check_integrity + - optimize: + flexible: [heat_pumps, storage] + method: soc + opf_version: 2 + - reinforce + - save diff --git a/edisgo/run/presets/uc3_oedb_ts.yaml b/edisgo/run/presets/uc3_oedb_ts.yaml new file mode 100644 index 000000000..59c184cdd --- /dev/null +++ b/edisgo/run/presets/uc3_oedb_ts.yaml @@ -0,0 +1,36 @@ +_comment: | + UC3 — real-world time series without OPF: + Like UC1 but uses real egon_data time series (oedb) instead of + synthetic worst cases. No optimization, no eTraGo. Difference to + UC1 is the data source for the final TS; difference to UC2 is no + OPF. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging) + - oedb_ts: real egon_data time series + - apply_heat_pump_strategy: uncontrolled + - reactive_power + - reinforce: final reinforcement + - save + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - oedb_ts: + timeindex: {start: "2035-01-01", periods: 168, freq: h} + dispatchable: {other: 0.7} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - reinforce + - save diff --git a/edisgo/run/registry.py b/edisgo/run/registry.py new file mode 100644 index 000000000..8aed4f3f5 --- /dev/null +++ b/edisgo/run/registry.py @@ -0,0 +1,113 @@ +""" +Task registry for the eDisGo pipeline runner. + +This module holds the global, process-wide mapping of task names to task +functions. Tasks are registered via the :func:`register_task` decorator +and looked up by name at pipeline execution time by the runner. Keeping +the registry separate from both the runner and the task implementations +lets external projects add their own tasks without patching eDisGo — +just import ``register_task`` and decorate a function. + +Registered tasks all share the signature ``(edisgo, ctx, **params)`` +where ``edisgo`` is the current :class:`~edisgo.EDisGo` instance (or +``None`` before it has been created by the first task), ``ctx`` is a +:class:`~edisgo.run.context.RunContext`, and ``**params`` are the +parameters passed from the YAML/JSON step definition. A task may return +an updated ``edisgo`` object (e.g. ``setup_grid`` creates it, ``load_*`` +replaces it); otherwise the runner keeps using the same instance. +""" +from __future__ import annotations + +from typing import Callable + +_TASKS: dict[str, Callable] = {} + + +def register_task(name: str) -> Callable[[Callable], Callable]: + """ + Decorator to register a task function under the given name. + + The decorated function becomes addressable from YAML/JSON pipelines + as either a plain string ``name`` or a single-key mapping + ``name: {param: value, ...}``. The name must be unique globally — + re-registering raises :class:`ValueError` to prevent silent + overrides across plugins. + + Parameters + ---------- + name : str + Unique task name used in pipeline definitions. + + Returns + ------- + Callable + A decorator that registers ``fn`` and returns it unchanged. + + Raises + ------ + ValueError + If ``name`` is already registered. + + Examples + -------- + >>> @register_task("set_timeindex_weekly") + ... def task_weekly(edisgo, ctx, *, start): + ... import pandas as pd + ... edisgo.set_timeindex(pd.date_range(start, periods=168, freq="h")) + + """ + def deco(fn: Callable) -> Callable: + if name in _TASKS: + raise ValueError( + f"Task '{name}' is already registered " + f"(existing={_TASKS[name].__qualname__}, " + f"new={fn.__qualname__})." + ) + _TASKS[name] = fn + return fn + + return deco + + +def get_task(name: str) -> Callable: + """ + Look up a registered task function by name. + + Parameters + ---------- + name : str + Task name as used in pipeline definitions. + + Returns + ------- + Callable + The task function registered under ``name``. + + Raises + ------ + KeyError + If ``name`` is not registered. The error message lists all + known task names to aid typo debugging. + + """ + if name not in _TASKS: + raise KeyError( + f"Unknown task: '{name}'. Known tasks: {sorted(_TASKS)}" + ) + return _TASKS[name] + + +def known_tasks() -> list[str]: + """ + Return a sorted list of all registered task names. + + Useful for error messages, CLI completion, and tests that assert + core tasks exist. + + Returns + ------- + list of str + All registered task names in alphabetical order. + + """ + return sorted(_TASKS) diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py new file mode 100644 index 000000000..63f30aa07 --- /dev/null +++ b/edisgo/run/runner.py @@ -0,0 +1,261 @@ +""" +Pipeline execution engine for the eDisGo runner. + +This module ties the other three pieces — :mod:`edisgo.run.config` +(loader), :mod:`edisgo.run.validator` (static checks), and +:mod:`edisgo.run.registry` (task lookup) — together into a linear +stage-by-stage executor. + +The execution model: + +1. Load and validate the config. +2. Build a :class:`~edisgo.run.context.RunContext`. +3. For each stage, if the stage declares ``load_from: X``, reload + the EDisGo object from stage ``X``'s save-artifact (topology + + results only; time series are dropped to let the new stage set + fresh ones). +4. For each step in the stage's pipeline, look up the task function + in the registry and call it with the current EDisGo object and + the context. A task may return a new EDisGo object (``setup_grid``, + ``load_from_base``) which then replaces the current one. +5. Repeat for all stages, finally return the EDisGo object. + +Two entry points are exposed: + +* :func:`run_edisgo` — starts from no EDisGo object; the first task + must create one (usually ``setup_grid``). +* :func:`_run_pipeline_on` — starts from an existing EDisGo instance; + used by :meth:`edisgo.EDisGo.run_pipeline`. +""" +from __future__ import annotations + +import logging + +from pathlib import Path +from typing import Any + +from edisgo.run import tasks as _tasks # noqa: F401 — triggers registration +from edisgo.run.config import load_config +from edisgo.run.context import RunContext +from edisgo.run.registry import get_task +from edisgo.run.validator import _split_step, validate + +logger = logging.getLogger("edisgo.run.runner") + + +def run_edisgo(config) -> Any: + """ + Run an eDisGo pipeline from a YAML/JSON config or dict. + + This is the standalone entry point. The pipeline's first task is + typically ``setup_grid`` or ``load_from_base`` to bootstrap the + :class:`~edisgo.EDisGo` instance. If you already have one, + prefer :meth:`edisgo.EDisGo.run_pipeline` instead. + + Parameters + ---------- + config : str, pathlib.Path, or dict + Path to a YAML/JSON pipeline config, or an in-memory dict of + the same shape. + + Returns + ------- + :class:`~edisgo.EDisGo` + The EDisGo instance after the last stage has run. For + multi-stage configs this is the object produced by the final + stage. + + """ + return _run_pipeline_on(None, config) + + +def _run_pipeline_on(edisgo, config): + """ + Internal runner shared by :func:`run_edisgo` and the EDisGo method. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Existing EDisGo instance to operate on, or ``None`` to have + the first task create one. + config : str, pathlib.Path, or dict + Config to execute. Passed through to + :func:`edisgo.run.config.load_config`. + + Returns + ------- + edisgo.EDisGo + The final EDisGo instance. + + Raises + ------ + RuntimeError + If a stage declares ``load_from: X`` but ``X`` produced no + artifact (typically because validate() was skipped). + + """ + cfg = load_config(config) + validate(cfg) + ctx = _build_context(cfg) + + for stage in cfg["stages"]: + ctx.current_stage = stage["name"] + ctx.logger.info(f"=== stage '{stage['name']}' ===") + + load_from = stage.get("load_from") + if load_from is not None: + artifact = ctx.stage_artifacts.get(load_from) + if artifact is None: + raise RuntimeError( + f"Stage '{stage['name']}' wants to load from " + f"'{load_from}' but no artifact is registered." + ) + edisgo = _load_artifact(str(artifact)) + + params = stage.get("params", {}) or {} + for step in stage["pipeline"]: + name, step_params = _split_step(step) + step_params = _resolve_templating(step_params, params) + ctx.logger.info(f" -> task '{name}'") + task_fn = get_task(name) + result = task_fn(edisgo, ctx, **step_params) + if result is not None: + edisgo = result + + return edisgo + + +def _build_context(cfg: dict) -> RunContext: + """ + Build a :class:`~edisgo.run.context.RunContext` from a config. + + Wires ``scenario`` and ``results.directory`` into the context and + stores the full config under :attr:`RunContext.raw_config` so + tasks can read supplementary sections. + + Parameters + ---------- + cfg : dict + Normalized config. + + Returns + ------- + RunContext + Initialized context with no engine, no artifacts, empty flags. + + """ + results_cfg = cfg.get("results") or {} + results_dir = results_cfg.get("directory") + return RunContext( + scenario=cfg.get("scenario"), + results_dir=Path(results_dir) if results_dir else None, + raw_config=cfg, + ) + + +def _load_artifact(path: str): + """ + Reload an EDisGo instance from a save-artifact for a ``load_from``. + + Loads topology + results only; time series and flex data are + dropped so the consuming stage can set them fresh. Equipment + changes are reset so the next stage's reinforce accounts only + for its own scenario. + + Parameters + ---------- + path : str + Path to a directory or ``.zip`` produced by the ``save`` + task. + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + from_zip = path.endswith(".zip") + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=False, + import_results=True, + import_electromobility=False, + import_heat_pump=False, + import_dsm=False, + import_overlying_grid=False, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + edisgo.results.equipment_changes = pd.DataFrame() + return edisgo + + +def _resolve_templating(step_params: dict, stage_params: dict) -> dict: + """ + Substitute ``{{params.x}}`` placeholders in step parameters. + + Stage-level ``params:`` allows a preset to expose a few knobs that + individual step parameters can reference. Only simple + ``{{params.KEY}}`` expansions inside string values are supported + (no filters, no conditionals, no nested expressions) — deliberately + kept trivial to avoid a Jinja dependency. + + Parameters + ---------- + step_params : dict + Keyword arguments for a single step. + stage_params : dict + Stage-level ``params:`` dict. + + Returns + ------- + dict + ``step_params`` with template strings resolved. + + """ + if not stage_params or not step_params: + return step_params + out = {} + for k, v in step_params.items(): + if isinstance(v, str) and "{{" in v: + out[k] = _render_template(v, stage_params) + else: + out[k] = v + return out + + +def _render_template(s: str, stage_params: dict) -> str: + """ + Expand ``{{params.KEY}}`` references in a single string. + + Parameters + ---------- + s : str + Source string. + stage_params : dict + Mapping of stage-level parameters. + + Returns + ------- + str + Rendered string. Unknown keys are left in place (the original + placeholder remains) so downstream errors point at the + typo-ed key rather than silently turning into an empty + string. + + """ + import re + + def repl(match): + expr = match.group(1).strip() + if expr.startswith("params."): + key = expr.split(".", 1)[1] + return str(stage_params.get(key, match.group(0))) + return match.group(0) + + return re.sub(r"\{\{\s*([^}]+)\s*\}\}", repl, s) diff --git a/edisgo/run/tasks/__init__.py b/edisgo/run/tasks/__init__.py new file mode 100644 index 000000000..0d59ea02a --- /dev/null +++ b/edisgo/run/tasks/__init__.py @@ -0,0 +1,25 @@ +""" +Task implementations for the eDisGo pipeline runner. + +Importing this package as a side effect registers every task defined +in its submodules with :func:`edisgo.run.registry.register_task`, so +that the runner sees them at execution time. The submodules are: + +* :mod:`.grid` — ``setup_grid``, ``load_from_base`` +* :mod:`.timeseries` — ``worst_case_ts``, ``oedb_ts``, ``manual_ts``, + ``set_timeindex``, ``reactive_power`` +* :mod:`.flex` — flex imports + (``import_heat_pumps``, ``import_home_batteries``, ``import_dsm``, + ``import_electromobility``, ``import_generators``) and operating + strategies (``apply_charging_strategy``, + ``apply_heat_pump_strategy``) +* :mod:`.analysis` — ``check_integrity``, ``analyze``, ``reinforce``, + ``base_reinforce``, ``optimize`` +* :mod:`.io` — ``save``, ``load_charging_from_files`` + +Task signature convention: ``(edisgo, ctx, **params)``. A task may +mutate ``edisgo`` in place and/or return a new EDisGo instance (the +returned value, if non-None, replaces the current one in the runner's +loop). +""" +from edisgo.run.tasks import analysis, flex, grid, io, timeseries # noqa: F401 diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py new file mode 100644 index 000000000..3f604914e --- /dev/null +++ b/edisgo/run/tasks/io.py @@ -0,0 +1,179 @@ +""" +Input/output tasks — persisting results and ingesting external files. + +* :func:`task_save` (``save``) — persist topology, time series, and + results to disk (directory or zip). Also publishes the artifact + path into ``ctx.stage_artifacts`` so a later stage can + ``load_from:``. +* :func:`task_load_charging_from_files` + (``load_charging_from_files``) — R4MU-specific placeholder for + integrating scenario charging stations from a directory of CSV / + GeoPackage files; implementation is deferred until needed. +""" +from __future__ import annotations + +import os + +from edisgo.run.registry import register_task + + +@register_task("save") +def task_save(edisgo, ctx, *, directory=None, save_topology=True, + save_timeseries=True, save_results=True, + save_electromobility=None, save_opf_results=False, + save_heatpump=None, save_overlying_grid=False, + save_dsm=None, archive=False, archive_type="zip", + reduce_memory=False, parameters=None): + """ + Save the current EDisGo state to disk. + + If ``directory`` is not given, the artifact is written under + ``ctx.results_dir / `` so every stage gets its own + subdirectory. When ``archive=True`` the result is a single zip; + the artifact path (including ``.zip``) is recorded in + ``ctx.stage_artifacts[]`` so a downstream stage can + declare ``load_from: ``. + + Flags drive smart defaults for the optional ``save_*`` switches: + if flex data is absent (per ``ctx.flags``), saving it is skipped. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to persist. + ctx : RunContext + Run context. Uses ``ctx.results_dir``, ``ctx.current_stage``, + and reads ``has_heat_pumps`` / ``has_dsm`` / + ``has_electromobility`` flags. + directory : str, optional + Absolute target directory. If omitted, derived from + ``ctx.results_dir / ctx.current_stage``. + save_topology : bool, optional + Write the topology CSVs. Default ``True``. + save_timeseries : bool, optional + Write time-series CSVs. Default ``True``. + save_results : bool, optional + Write the results CSVs (equipment changes, expansion costs, + etc.). Default ``True``. + save_electromobility : bool or None, optional + If ``None``, auto-enabled iff + ``ctx.flags['has_electromobility']`` is truthy. + save_opf_results : bool, optional + Write OPF results if present. + save_heatpump : bool or None, optional + If ``None``, auto-enabled iff ``ctx.flags['has_heat_pumps']`` + is truthy. + save_overlying_grid : bool, optional + Write overlying-grid (eTraGo) specs if present. + save_dsm : bool or None, optional + If ``None``, auto-enabled iff ``ctx.flags['has_dsm']`` is + truthy. + archive : bool, optional + Pack the directory into a single ``.zip`` archive. + archive_type : str, optional + Archive format (currently only ``"zip"``). + reduce_memory : bool, optional + Downcast float time-series to ``float32`` to save disk. + parameters : dict, optional + Fine-grained selection of which results fields to write, + e.g. ``{"grid_expansion_results": ["equipment_changes"]}``. + + Returns + ------- + edisgo.EDisGo + The unchanged EDisGo instance. + + Raises + ------ + ValueError + If no ``directory`` is given and ``ctx.results_dir`` is also + unset. + + """ + if directory is None: + if ctx.results_dir is None: + raise ValueError( + "Task 'save' needs a 'directory' parameter or " + "config.results.directory." + ) + stage = ctx.current_stage or "main" + directory = os.path.join(str(ctx.results_dir), stage) + + if save_heatpump is None: + save_heatpump = ctx.flags.get("has_heat_pumps", False) + if save_dsm is None: + save_dsm = ctx.flags.get("has_dsm", False) + if save_electromobility is None: + save_electromobility = ctx.flags.get("has_electromobility", False) + + kwargs = dict( + directory=directory, + save_topology=save_topology, + save_timeseries=save_timeseries, + save_results=save_results, + save_electromobility=save_electromobility, + save_opf_results=save_opf_results, + save_heatpump=save_heatpump, + save_overlying_grid=save_overlying_grid, + save_dsm=save_dsm, + ) + if archive: + kwargs["archive"] = True + kwargs["archive_type"] = archive_type + if reduce_memory: + kwargs["reduce_memory"] = True + if parameters is not None: + kwargs["parameters"] = parameters + + edisgo.save(**kwargs) + + saved_path = directory + (".zip" if archive else "") + if ctx.current_stage: + ctx.stage_artifacts[ctx.current_stage] = saved_path + ctx.flags["last_saved"] = saved_path + return edisgo + + +@register_task("load_charging_from_files") +def task_load_charging_from_files(edisgo, ctx, *, charging_dir, + use_case_to_sector=None, + mv_threshold_kw=100.0): + """ + Integrate scenario charging stations from files (R4MU workflow). + + PLACEHOLDER — the full implementation lives in eGo's + ``_run_edisgo_task_load_charging_from_files`` and needs to be + ported when R4MU is prioritised. The eGo version reads a + GeoPackage / CSV of charging locations, filters by the MV grid + district geometry, and integrates them into the topology via + :func:`find_nearest_bus` / ``integrate_component_based_on_geolocation`` + with a use-case-to-sector mapping and an MV/LV connection + threshold. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + charging_dir : str + Directory containing the charging-station source files. + use_case_to_sector : dict, optional + Maps raw use-case labels (``"home_detached"`` etc.) to + eDisGo sector names (``"home"``, ``"work"``, …). + mv_threshold_kw : float, optional + Capacity threshold above which stations connect to an MV + bus; below connect to LV. + + Raises + ------ + NotImplementedError + Always — port the eGo implementation before using. + + """ + raise NotImplementedError( + "Task 'load_charging_from_files' is a placeholder port from " + "eGo R4MU. Port the logic from eGo's " + "_run_edisgo_task_load_charging_from_files when R4MU is " + "needed." + ) diff --git a/edisgo/run/validator.py b/edisgo/run/validator.py new file mode 100644 index 000000000..3989b8398 --- /dev/null +++ b/edisgo/run/validator.py @@ -0,0 +1,201 @@ +""" +Static validator for pipeline configs. + +The validator enforces structural and ordering rules that the runner +would otherwise hit at execution time — often after 20 minutes of work. +Running these checks up-front turns "cryptic AttributeError after half +the pipeline" into a clear ``ValueError`` at startup. + +Checked rules: + +* every step maps to a known, registered task name; +* ``reactive_power`` comes after every time-series task in a stage, + never before — ``set_time_series_reactive_power_control`` overwrites + reactive power on the currently set active-power time series; +* ``analyze`` and ``reinforce`` require a time-series task earlier in + the stage (or a ``load_from:`` that brings a prepared grid); +* ``optimize`` requires both a time-series task and at least one flex + import earlier in the stage — OPF without flexibility is meaningless; +* flex imports (``import_heat_pumps``, …) require a loaded grid, i.e. + an earlier ``setup_grid`` / ``load_from_base`` / a stage-level + ``load_from:``; +* ``base_reinforce`` likewise requires a loaded grid; +* a stage that declares ``load_from: X`` can only run if stage ``X`` + ran earlier AND contains a ``save`` step. +""" +from __future__ import annotations + +from typing import Any + +from edisgo.run.registry import known_tasks + +_TS_TASKS = {"worst_case_ts", "oedb_ts", "manual_ts", "set_timeindex"} +_GRID_CREATING_TASKS = {"setup_grid", "load_from_base"} +_FLEX_IMPORTS = { + "import_heat_pumps", + "import_home_batteries", + "import_dsm", + "import_electromobility", +} + + +def validate(cfg: dict) -> None: + """ + Validate a normalized pipeline config against the ordering rules. + + This function does not return a value. On success it simply + returns; on any rule violation it raises :class:`ValueError` with + a message identifying the offending stage and task. + + Parameters + ---------- + cfg : dict + Normalized config as returned by + :func:`edisgo.run.config.load_config`. Must have a ``stages`` + list at the top level. + + Raises + ------ + ValueError + If the config has no stages, an unknown task name, a + structural problem (reactive before TS, reinforce without TS, + optimize without flex, flex import without grid, …), or a + stage references a ``load_from`` source that doesn't exist or + has no ``save`` step. + + """ + stages = cfg.get("stages") or [] + if not stages: + raise ValueError("Config has no stages to run.") + + available_artifacts: set[str] = set() + + for stage in stages: + name = stage["name"] + pipeline = stage.get("pipeline") or [] + load_from = stage.get("load_from") + + if load_from is not None and load_from not in available_artifacts: + raise ValueError( + f"Stage '{name}' requires 'load_from: {load_from}' but " + f"that stage has not run or did not save. Available: " + f"{sorted(available_artifacts)}" + ) + + grid_available = load_from is not None + ts_set = False + reactive_set = False + flex_imported = False + has_save = False + + for step in pipeline: + task_name, _params = _split_step(step) + if task_name not in known_tasks(): + raise ValueError( + f"Unknown task '{task_name}' in stage '{name}'. " + f"Known: {known_tasks()}" + ) + + if task_name in _GRID_CREATING_TASKS: + grid_available = True + if task_name in _TS_TASKS: + if reactive_set: + raise ValueError( + f"Stage '{name}': time-series task " + f"'{task_name}' comes after 'reactive_power' " + f"— reactive_power must be the last " + f"time-series-altering step." + ) + ts_set = True + if task_name == "reactive_power": + reactive_set = True + if task_name in _FLEX_IMPORTS: + flex_imported = True + if not grid_available: + raise ValueError( + f"Stage '{name}': task '{task_name}' requires " + f"a loaded grid (setup_grid or " + f"load_from_base) before it." + ) + if task_name in {"analyze", "reinforce"} and not ( + ts_set or load_from + ): + raise ValueError( + f"Stage '{name}': task '{task_name}' requires time " + f"series to be set (e.g. worst_case_ts or " + f"oedb_ts) before it." + ) + if task_name == "optimize": + if not ts_set and not load_from: + raise ValueError( + f"Stage '{name}': 'optimize' requires time " + f"series." + ) + if not flex_imported and not load_from: + raise ValueError( + f"Stage '{name}': 'optimize' requires at least " + f"one flex asset to be imported." + ) + if task_name == "base_reinforce" and not grid_available: + raise ValueError( + f"Stage '{name}': 'base_reinforce' requires a " + f"loaded grid before it." + ) + if task_name == "save": + has_save = True + + if has_save: + available_artifacts.add(name) + + +def _split_step(step: Any) -> tuple[str, dict]: + """ + Normalize a pipeline step into ``(task_name, params)``. + + Steps are allowed in two forms in YAML/JSON: + + * bare string — ``worst_case_ts`` → ``("worst_case_ts", {})`` + * single-key mapping — + ``import_electromobility: {charging_strategy: dumb}`` + → ``("import_electromobility", {"charging_strategy": "dumb"})`` + + ``None`` as the parameter value is treated as an empty dict so + that YAML's ``task:`` (with nothing after the colon) works. + + Parameters + ---------- + step : str or dict + Raw step as it appears in the pipeline list. + + Returns + ------- + tuple of (str, dict) + The task name and its keyword arguments. + + Raises + ------ + ValueError + If ``step`` is not a string or a single-key mapping, or if + the parameter value is not a mapping. + + """ + if isinstance(step, str): + return step, {} + if isinstance(step, dict): + if len(step) != 1: + raise ValueError( + f"Task step must be a string or single-key mapping, " + f"got: {step}" + ) + (name, params), = step.items() + if params is None: + params = {} + if not isinstance(params, dict): + raise ValueError( + f"Parameters for task '{name}' must be a mapping, " + f"got: {type(params).__name__}" + ) + return name, params + raise ValueError( + f"Task step must be string or mapping, got: {step!r}" + ) diff --git a/setup.py b/setup.py index 0ee46d6c9..605bbe700 100644 --- a/setup.py +++ b/setup.py @@ -100,6 +100,7 @@ def read(fname): "edisgo": [ os.path.join("config", "*.cfg"), os.path.join("equipment", "*.csv"), + os.path.join("run", "presets", "*.yaml"), ] }, ) diff --git a/tests/run/__init__.py b/tests/run/__init__.py new file mode 100644 index 000000000..baf15e08b --- /dev/null +++ b/tests/run/__init__.py @@ -0,0 +1 @@ +"""Tests for the :mod:`edisgo.run` pipeline runner.""" diff --git a/tests/run/test_config.py b/tests/run/test_config.py new file mode 100644 index 000000000..10b4ec474 --- /dev/null +++ b/tests/run/test_config.py @@ -0,0 +1,154 @@ +""" +Unit tests for :mod:`edisgo.run.config` — loader, merger, adapter. + +Covers YAML/JSON parity, ``extends`` resolution (preset-by-name and +relative paths), deep-merge semantics, stage normalization, and the +eGo-legacy adapter. +""" +import json + +import pytest +import yaml + +from edisgo.run.config import _deep_merge, load_config + + +def _write(tmp_path, name, data): + """ + Helper: write ``data`` to ``tmp_path/name`` as YAML or JSON. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest-provided temporary directory. + name : str + File name with extension (``.yaml``/``.yml``/``.json``). + data : dict + Payload. + + Returns + ------- + pathlib.Path + Path to the written file. + + """ + path = tmp_path / name + if name.endswith(".json"): + path.write_text(json.dumps(data)) + else: + path.write_text(yaml.safe_dump(data)) + return path + + +def test_load_flat_pipeline_normalized_to_stages(tmp_path): + """A flat ``pipeline:`` must normalize to a single 'main' stage.""" + p = _write(tmp_path, "cfg.yaml", { + "scenario": "eGon2035", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + }) + cfg = load_config(str(p)) + assert "pipeline" not in cfg + assert cfg["stages"] == [ + {"name": "main", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"]} + ] + + +def test_yaml_and_json_equivalent(tmp_path): + """YAML and JSON payloads with identical content must load equal.""" + data = { + "scenario": "eGon2035", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + } + yaml_path = _write(tmp_path, "cfg.yaml", data) + json_path = _write(tmp_path, "cfg.json", data) + assert load_config(str(yaml_path)) == load_config(str(json_path)) + + +def test_extends_merges_parent(tmp_path): + """Child config must deep-merge with its ``extends:`` parent.""" + parent = _write(tmp_path, "parent.yaml", { + "scenario": "eGon2035", + "grid": {"legacy_ding0_grids": False}, + "pipeline": ["setup_grid", "reinforce"], + }) + child = _write(tmp_path, "child.yaml", { + "extends": str(parent), + "grid": {"ding0_path": "/tmp/xyz"}, + }) + cfg = load_config(str(child)) + assert cfg["scenario"] == "eGon2035" + assert cfg["grid"] == { + "legacy_ding0_grids": False, "ding0_path": "/tmp/xyz" + } + assert cfg["stages"][0]["pipeline"] == ["setup_grid", "reinforce"] + + +def test_extends_preset_by_name(tmp_path): + """``extends: basic`` must resolve to the bundled basic preset.""" + child = _write(tmp_path, "child.yaml", { + "extends": "basic", + "grid": {"ding0_path": "/tmp/xyz"}, + }) + cfg = load_config(str(child)) + assert "stages" in cfg + assert cfg["grid"]["ding0_path"] == "/tmp/xyz" + + +def test_deep_merge_nested(): + """Nested dicts must be merged key-by-key, child wins on conflict.""" + base = {"a": {"b": 1, "c": 2}, "d": 4} + over = {"a": {"b": 99, "e": 5}} + merged = _deep_merge(base, over) + assert merged == {"a": {"b": 99, "c": 2, "e": 5}, "d": 4} + + +def test_both_pipeline_and_stages_rejected(tmp_path): + """Top-level ``pipeline`` and ``stages`` are mutually exclusive.""" + p = _write(tmp_path, "cfg.yaml", { + "pipeline": ["setup_grid"], + "stages": [{"name": "x", "pipeline": ["setup_grid"]}], + }) + with pytest.raises(ValueError, match="both"): + load_config(str(p)) + + +def test_duplicate_stage_names_rejected(tmp_path): + """Stage names must be unique; duplicates raise ValueError.""" + p = _write(tmp_path, "cfg.yaml", { + "stages": [ + {"name": "x", "pipeline": ["setup_grid"]}, + {"name": "x", "pipeline": ["reinforce"]}, + ], + }) + with pytest.raises(ValueError, match="Duplicate stage"): + load_config(str(p)) + + +def test_ego_legacy_adapter(tmp_path): + """An eGo ``scenario_setting_*.json`` must adapt to the new schema.""" + ego_cfg = { + "eGo": {"eDisGo": True}, + "eTraGo": {"scn_name": "eGon2035"}, + "eDisGo": { + "grid_path": "/some/path", + "results": "/tmp/results", + "tasks": [ + "1_setup_grid", + "base_reinforce", + "import_heat_pumps_from_db", + "worst_case_ts", + "5_grid_reinforcement", + ], + }, + "database": {"host": "localhost"}, + } + p = _write(tmp_path, "legacy.json", ego_cfg) + cfg = load_config(str(p)) + assert cfg["scenario"] == "eGon2035" + assert cfg["grid"]["ding0_path"] == "/some/path" + assert cfg["stages"][0]["pipeline"] == [ + "setup_grid", "base_reinforce", "import_heat_pumps", + "worst_case_ts", "reinforce", + ] + assert cfg["database"]["host"] == "localhost" diff --git a/tests/run/test_registry.py b/tests/run/test_registry.py new file mode 100644 index 000000000..a56070bf6 --- /dev/null +++ b/tests/run/test_registry.py @@ -0,0 +1,35 @@ +""" +Unit tests for :mod:`edisgo.run.registry`. + +Verifies that core tasks are discoverable, that ``get_task`` raises a +useful error on typos, and that duplicate registrations are rejected. +""" +import pytest + +from edisgo.run.registry import get_task, known_tasks, register_task + + +def test_known_tasks_contains_core(): + """All core task names must be registered on import.""" + tasks = known_tasks() + for core in ["setup_grid", "worst_case_ts", "reactive_power", + "reinforce", "analyze", "save"]: + assert core in tasks + + +def test_get_task_unknown_raises(): + """Unknown task names must surface as a descriptive KeyError.""" + with pytest.raises(KeyError, match="Unknown task"): + get_task("does_not_exist") + + +def test_register_task_duplicate_raises(): + """Registering the same task name twice is a bug — must raise.""" + @register_task("_test_task_for_dup_check") + def _a(edisgo, ctx): + """Marker task #1 — test fixture only.""" + + with pytest.raises(ValueError, match="already registered"): + @register_task("_test_task_for_dup_check") + def _b(edisgo, ctx): + """Marker task #2 — test fixture only, must not register.""" diff --git a/tests/run/test_runner.py b/tests/run/test_runner.py new file mode 100644 index 000000000..0ecb6d08e --- /dev/null +++ b/tests/run/test_runner.py @@ -0,0 +1,118 @@ +""" +End-to-end tests for the eDisGo pipeline runner. + +Uses the small test grid under ``tests/data/ding0_test_network_2`` +(exposed by :mod:`tests.conftest` as +``pytest.ding0_test_network_2_path``) to run full pipelines without +touching the database. Covers: + +* the standalone ``run_edisgo`` entry point with a flat pipeline, +* the instance method ``EDisGo.run_pipeline``, +* the stage mechanism with ``save`` + ``load_from``. +""" +import os + +import pytest + +from edisgo.run import run_edisgo + + +@pytest.fixture +def basic_cfg(tmp_path): + """ + Minimal end-to-end config fixture. + + Produces a config that loads the small ding0 test grid, sets + worst-case time series, fixes reactive power, checks integrity, + runs reinforcement, and saves — no database needed. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest-provided temp directory for the run's artifacts. + + Returns + ------- + dict + The config dict. + + """ + return { + "scenario": "eGon2035", + "grid": { + "ding0_path": pytest.ding0_test_network_2_path, + "legacy_ding0_grids": True, + }, + "results": {"directory": str(tmp_path)}, + "pipeline": [ + "setup_grid", + "worst_case_ts", + "reactive_power", + "check_integrity", + "reinforce", + "save", + ], + } + + +def test_runner_basic_end_to_end(basic_cfg): + """A flat-pipeline run must execute and persist the expected artifact.""" + edisgo = run_edisgo(basic_cfg) + assert edisgo is not None + assert edisgo.topology is not None + assert os.path.isdir(os.path.join(basic_cfg["results"]["directory"], + "main")) + + +def test_runner_method_on_edisgo(basic_cfg): + """``EDisGo.run_pipeline`` must operate on the existing instance.""" + from edisgo import EDisGo + + basic_cfg["pipeline"] = basic_cfg["pipeline"][1:] # skip setup_grid + edisgo = EDisGo( + ding0_grid=basic_cfg["grid"]["ding0_path"], + legacy_ding0_grids=True, + ) + edisgo = edisgo.run_pipeline(basic_cfg) + assert edisgo.topology is not None + + +def test_runner_two_stages_with_load_from(tmp_path): + """ + A two-stage run must save the first stage and reload it via + ``load_from`` in the second stage, producing both artifacts. + """ + cfg = { + "scenario": "eGon2035", + "grid": { + "ding0_path": pytest.ding0_test_network_2_path, + "legacy_ding0_grids": True, + }, + "results": {"directory": str(tmp_path)}, + "stages": [ + { + "name": "base", + "pipeline": [ + "setup_grid", + "worst_case_ts", + "reactive_power", + "reinforce", + {"save": {"archive": True}}, + ], + }, + { + "name": "scenario", + "load_from": "base", + "pipeline": [ + "worst_case_ts", + "reactive_power", + "reinforce", + "save", + ], + }, + ], + } + edisgo = run_edisgo(cfg) + assert edisgo.topology is not None + assert os.path.exists(os.path.join(str(tmp_path), "base.zip")) + assert os.path.isdir(os.path.join(str(tmp_path), "scenario")) diff --git a/tests/run/test_validator.py b/tests/run/test_validator.py new file mode 100644 index 000000000..4b86f40cf --- /dev/null +++ b/tests/run/test_validator.py @@ -0,0 +1,97 @@ +""" +Unit tests for :mod:`edisgo.run.validator`. + +Each test pins one ordering rule: reactive-before-TS, reinforce +without TS, optimize without flex, flex import without grid, and the +stage-level ``load_from`` constraints. +""" +import pytest + +from edisgo.run.validator import validate + + +def _wrap(pipeline): + """ + Wrap a flat pipeline into a single-stage config dict. + + Parameters + ---------- + pipeline : list + Ordered list of task names / single-key mappings. + + Returns + ------- + dict + Minimal config in the shape expected by :func:`validate`. + + """ + return {"stages": [{"name": "main", "pipeline": pipeline}]} + + +def test_valid_pipeline(): + """A well-formed pipeline must pass validation without raising.""" + validate(_wrap(["setup_grid", "worst_case_ts", "reactive_power", + "reinforce", "save"])) + + +def test_unknown_task_rejected(): + """Typo'd task names must be rejected.""" + with pytest.raises(ValueError, match="Unknown task"): + validate(_wrap(["setup_grid", "nonexistent_task"])) + + +def test_reactive_before_ts_rejected(): + """reactive_power before a TS task violates the ordering rule.""" + with pytest.raises(ValueError, match="reactive_power"): + validate(_wrap(["setup_grid", "reactive_power", "worst_case_ts"])) + + +def test_reinforce_without_ts_rejected(): + """reinforce without any prior time-series step must fail.""" + with pytest.raises(ValueError, match="time series"): + validate(_wrap(["setup_grid", "reinforce"])) + + +def test_optimize_without_flex_rejected(): + """optimize requires at least one flex asset to be imported.""" + with pytest.raises(ValueError, match="flex asset"): + validate(_wrap(["setup_grid", "worst_case_ts", "optimize"])) + + +def test_flex_import_before_grid_rejected(): + """Flex imports require a loaded grid — pre-loading is not enough.""" + with pytest.raises(ValueError, match="loaded grid"): + validate(_wrap(["import_heat_pumps", "worst_case_ts", "reinforce"])) + + +def test_stage_load_from_missing_rejected(): + """``load_from: X`` where X has not run must fail.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce"]}, + {"name": "b", "load_from": "nonexistent", + "pipeline": ["reinforce"]}, + ]} + with pytest.raises(ValueError, match="load_from"): + validate(cfg) + + +def test_stage_load_from_requires_save_in_source(): + """A stage consumed by ``load_from`` must itself end with ``save``.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce"]}, # no save + {"name": "b", "load_from": "a", "pipeline": ["reinforce"]}, + ]} + with pytest.raises(ValueError, match="load_from"): + validate(cfg) + + +def test_stage_load_from_with_save_ok(): + """Stage chain with a save in the source must validate successfully.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce", "save"]}, + {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, + ]} + validate(cfg) From 0c8792eea11a41e6422c691aeb9e65f20e2cee53 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:18:51 +0200 Subject: [PATCH 11/66] Add example yaml for full example --- edisgo/run/presets/uc4_example_MS.yaml | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 edisgo/run/presets/uc4_example_MS.yaml diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example_MS.yaml new file mode 100644 index 000000000..a72573496 --- /dev/null +++ b/edisgo/run/presets/uc4_example_MS.yaml @@ -0,0 +1,56 @@ +_comment: | + UC3 — OPF with full flexibility: + Like UC1 but loads real egon_data time series (oedb) and runs a + powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) + before the final reinforce. Cost delta = extra reinforcement needed + under optimal flex dispatch. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_generators: from edon-data + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - reactive_power + - check_integrity + - optimize: pm_optimize with flex assets (SOC, opf v2) + - reinforce: final reinforcement + - save + +scenario: eGon2035 +grid: + ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + legacy_ding0_grids: false + +database: + ssh: + enabled: false + +timeindex: {start: "2035-01-01", periods: 24, freq: h} + +results: + directory: results/uc4_example + +pipeline: + - setup_grid + - base_reinforce + - import_generators + - import_home_batteries + - import_heat_pumps + - import_dsm + - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - oedb_ts: + dispatchable: {other: 0.7} + - reactive_power + - check_integrity + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - reinforce + - save From d3476eb9873fcf4626fa2c9b64bb6560d1a3f848 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:20:22 +0200 Subject: [PATCH 12/66] Add file for analysis-tasks, Add short cut for DSM --- edisgo/run/tasks/analysis.py | 321 +++++++++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 edisgo/run/tasks/analysis.py diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py new file mode 100644 index 000000000..f028bb31c --- /dev/null +++ b/edisgo/run/tasks/analysis.py @@ -0,0 +1,321 @@ +""" +Power-flow, reinforcement, and optimization tasks. + +The three analysis layers: + +* :func:`task_analyze` (``analyze``) — non-linear AC load flow over + the active time series; does not modify the topology. +* :func:`task_reinforce` (``reinforce``) — iterative reinforcement + that adds/upgrades equipment until all technical constraints are + met. Populates ``results.equipment_changes``. +* :func:`task_optimize` (``optimize``) — powermodels OPF over + flexibilities (heat pumps, EV, DSM, storage) to minimize + reinforcement need. + +In addition: + +* :func:`task_check_integrity` (``check_integrity``) — a cheap + sanity check before the expensive steps. +* :func:`task_base_reinforce` (``base_reinforce``) — two-phase helper: + worst-case TS → reinforce → reset ``equipment_changes``. Used to + produce a "base" grid whose subsequent reinforce costs reflect + only a scenario overlay. +""" +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + + +@register_task("check_integrity") +def task_check_integrity(edisgo, ctx): + """ + Run EDisGo's integrity checks on the topology and time series. + + Catches bus mismatches, missing time series for components, and + similar structural problems. Raises if something is off — do not + swallow it silently. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to check. + ctx : RunContext + Run context (unused). + + Returns + ------- + edisgo.EDisGo + The unchanged EDisGo instance. + + """ + edisgo.check_integrity() + return edisgo + + +@register_task("analyze") +def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, + raise_not_converged=False, troubleshooting_mode=None): + """ + Run AC power flow over the active time series. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to analyze. + ctx : RunContext + Run context. Stores the number of non-converged time steps + under ``ctx.flags['not_converged_steps']`` and warns if any. + mode : str, optional + ``None`` (default) runs the full grid; ``"mv"`` runs only the + medium-voltage level; ``"lv"`` runs only LV. + timesteps : pandas.DatetimeIndex, optional + Restrict the analysis to these time steps. + raise_not_converged : bool, optional + If ``True``, raise on non-convergence. Default ``False`` so + the pipeline can continue and ``reinforce`` can attempt to + resolve the issue. + troubleshooting_mode : str, optional + Extra diagnostic mode passed through to + :meth:`EDisGo.analyze`. + + Returns + ------- + edisgo.EDisGo + The analyzed EDisGo instance. + + """ + result = edisgo.analyze( + mode=mode, + timesteps=timesteps, + raise_not_converged=raise_not_converged, + troubleshooting_mode=troubleshooting_mode, + ) + if isinstance(result, tuple) and len(result) == 2: + converged, not_converged = result + ctx.flags["not_converged_steps"] = len(not_converged) + if len(not_converged) > 0: + ctx.logger.warning( + f"Power flow did not converge for {len(not_converged)} " + f"time steps." + ) + return edisgo + + +@register_task("reinforce") +def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, + copy_grid=False, max_while_iterations=20, + split_voltage_band=True, mode=None, + without_generator_import=False, n_minus_one=False, + catch_convergence_problems=False): + """ + Run iterative grid reinforcement. + + Adds/upgrades lines and transformers until voltage and loading + constraints are met for all time steps. Results accumulate in + :attr:`EDisGo.results.equipment_changes` and + :attr:`~EDisGo.results.grid_expansion_costs`. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to reinforce. + ctx : RunContext + Run context (unused beyond logging). + timesteps_pfa : pandas.DatetimeIndex, optional + Restrict the reinforcement's analysis to these time steps. + reduced_analysis : bool, optional + If ``True``, use a cheaper convergence check during + reinforcement. + copy_grid : bool, optional + If ``True``, operate on a copy and return it as a new + instance (default ``False``). + max_while_iterations : int, optional + Cap on the outer iteration loop. + split_voltage_band : bool, optional + Split the allowed voltage deviation between MV and LV + (typical MV/LV coupling rule). + mode : str, optional + ``None``, ``"mv"``, ``"lv"``, or ``"mvlv"``. Restricts + reinforcement to a voltage level. + without_generator_import : bool, optional + Skip the implicit generator import step. + n_minus_one : bool, optional + Enable (N-1) contingency reinforcement. Expensive. + catch_convergence_problems : bool, optional + Wrap in the catch-convergence helper for troublesome grids. + + Returns + ------- + edisgo.EDisGo + The reinforced EDisGo instance. + + """ + edisgo.reinforce( + timesteps_pfa=timesteps_pfa, + reduced_analysis=reduced_analysis, + copy_grid=copy_grid, + max_while_iterations=max_while_iterations, + split_voltage_band=split_voltage_band, + mode=mode, + without_generator_import=without_generator_import, + n_minus_one=n_minus_one, + catch_convergence_problems=catch_convergence_problems, + ) + return edisgo + + +@register_task("base_reinforce") +def task_base_reinforce(edisgo, ctx, *, cases=None, + reset_equipment_changes=True, save_artifact=True): + """ + Produce a base-reinforced grid and reset the cost accumulator. + + This is the composite step ported from eGo's two-phase reinforce + workflow: + + 1. Set synthetic worst-case time series (``feed-in_case`` + + ``load_case``). + 2. Run :meth:`EDisGo.reinforce` to bring the grid to a neutral + baseline. + 3. Optionally save the resulting grid so downstream stages can + ``load_from: ...``. + 4. Clear :attr:`Results.equipment_changes` so the next reinforce + captures only scenario-specific deltas. + 5. Restore the prior time index so the next TS-setting task + starts from a clean state. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to base-reinforce. + ctx : RunContext + Run context. ``ctx.results_dir`` is the artifact destination. + Sets ``ctx.flags['base_reinforced'] = True`` and + ``ctx.stage_artifacts['__base_reinforce__']`` on save. + cases : list of str, optional + Which worst cases to set (subset of + ``{"load_case", "feed-in_case"}``). Default is both. + reset_equipment_changes : bool, optional + Clear the equipment-changes DataFrame after reinforcement. + save_artifact : bool, optional + Write a ``grid_data_base_reinforcement.zip`` next to the + other results. + + Returns + ------- + edisgo.EDisGo + The base-reinforced EDisGo instance. + + """ + import os + + prev_timeindex = edisgo.timeseries.timeindex + + edisgo.set_time_series_worst_case_analysis(cases=cases) + edisgo.reinforce() + + if save_artifact and ctx.results_dir is not None: + artifact_dir = os.path.join( + str(ctx.results_dir), "grid_data_base_reinforcement" + ) + edisgo.save( + directory=artifact_dir, + save_topology=True, + save_timeseries=False, + save_results=True, + archive=True, + archive_type="zip", + parameters={"grid_expansion_results": ["equipment_changes"]}, + ) + ctx.stage_artifacts["__base_reinforce__"] = artifact_dir + ".zip" + + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + + if len(prev_timeindex) > 0: + edisgo.set_timeindex(prev_timeindex) + + ctx.flags["base_reinforced"] = True + return edisgo + + +@register_task("optimize") +def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, + flexible_hps=None, flexible_loads=None, + flexible_storage_units=None, opf_version=2, method="soc", + warm_start=False, s_base=1): + """ + Run a powermodels optimal-power-flow (OPF) over flexibilities. + + If ``flexible`` is given (high-level shortcut), it expands to the + lower-level ``flexible_*`` lists automatically: + + * ``"heat_pumps"`` → all loads of type ``heat_pump`` + * ``"charging_points"`` → all loads of type ``charging_point`` + * ``"storage"`` → all storage-unit indices + * ``"loads"`` → all DSM-ready load indices + + Explicit ``flexible_*`` kwargs override the shortcut. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to optimize. + ctx : RunContext + Run context (unused). + flexible : list of str, optional + High-level selector, subset of ``{"heat_pumps", + "charging_points", "storage"}``. If ``None``, nothing is + auto-populated. + flexible_cps : list of str, optional + Explicit list of flexible charging-point names. + flexible_hps : list of str, optional + Explicit list of flexible heat-pump load names. + flexible_loads : list of str, optional + Explicit list of flexible DSM load names. + flexible_storage_units : list of str, optional + Explicit list of flexible storage-unit names. + opf_version : int, optional + Powermodels OPF formulation version (1 or 2, default 2). + method : str, optional + OPF relaxation method, e.g. ``"soc"`` (second-order cone). + warm_start : bool, optional + Reuse a previous solution as the starting point. + s_base : float, optional + Per-unit base power for normalization. + + Returns + ------- + edisgo.EDisGo + The optimized EDisGo instance. + + """ + flexible = flexible or [] + + if flexible_hps is None and "heat_pumps" in flexible: + flexible_hps = edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "heat_pump" + ].index.tolist() + if flexible_cps is None and "charging_points" in flexible: + flexible_cps = edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "charging_point" + ].index.tolist() + if flexible_storage_units is None and "storage" in flexible: + flexible_storage_units = edisgo.topology.storage_units_df.index.tolist() + if flexible_loads is not None and "dsm" in flexbile: + flexible_loads = edisgo.dsm.p_min.columns.values + + + edisgo.pm_optimize( + flexible_cps=flexible_cps or [], + flexible_hps=flexible_hps or [], + flexible_loads=flexible_loads or [], + flexible_storage_units=flexible_storage_units or [], + opf_version=opf_version, + method=method, + warm_start=warm_start, + s_base=s_base, + ) + return edisgo From 25ae93e58954b4f377c9bd3ee81b58e73efb9da5 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:21:10 +0200 Subject: [PATCH 13/66] Add file for flex-tasks, Add flexibility band generation, --- edisgo/run/tasks/flex.py | 282 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 edisgo/run/tasks/flex.py diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py new file mode 100644 index 000000000..fd914a2a0 --- /dev/null +++ b/edisgo/run/tasks/flex.py @@ -0,0 +1,282 @@ +""" +Flex-asset import and operation-strategy tasks. + +These tasks either pull flex assets (heat pumps, home batteries, DSM, +electromobility, generators) from egon_data / OEP into the topology, +or apply an operating strategy on assets already present. They must +run AFTER the grid is loaded (``setup_grid`` or ``load_from_base``) +and typically BEFORE the time-series step, so the time series can +cover the new assets. +""" +from __future__ import annotations + +from edisgo.run.registry import register_task + + +@register_task("import_heat_pumps") +def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): + """ + Import heat pumps from egon_data into the topology. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets + ``ctx.flags['has_heat_pumps']`` to the observed count. + import_types : list of str, optional + Subset of ``["individual_heat_pumps", "central_heat_pumps"]``; + default imports both. + timeindex : pandas.DatetimeIndex, optional + Restrict COP / heat-demand time series to this index. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_heat_pumps( + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + timeindex=timeindex, + import_types=import_types, + ) + ctx.flags["has_heat_pumps"] = len( + edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "heat_pump" + ] + ) > 0 + return edisgo + + +@register_task("import_home_batteries") +def task_import_home_batteries(edisgo, ctx): + """ + Import home batteries from egon_data into the topology. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets + ``ctx.flags['has_home_batteries']``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_home_batteries( + scenario=ctx.scenario, engine=ctx.ensure_engine() + ) + ctx.flags["has_home_batteries"] = ( + not edisgo.topology.storage_units_df.empty + ) + return edisgo + + +@register_task("import_dsm") +def task_import_dsm(edisgo, ctx, *, timeindex=None): + """ + Import demand-side-management potential from egon_data. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets ``ctx.flags['has_dsm']``. + timeindex : pandas.DatetimeIndex, optional + Restrict DSM availability time series to this index. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_dsm( + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + timeindex=timeindex, + ) + ctx.flags["has_dsm"] = ( + edisgo.dsm.p_max is not None and not edisgo.dsm.p_max.empty + ) + return edisgo + + +@register_task("import_electromobility") +def task_import_electromobility(edisgo, ctx, *, data_source="oedb", + charging_strategy="dumb", + flexibility_bands_ucs = None, + import_electromobility_data_kwds=None, + allocate_charging_demand_kwds=None): + """ + Import electromobility data (charging processes + parks). + + Optionally applies a charging strategy directly after import to + turn the raw charging processes into active-power time series on + the charging points. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()`` (for ``data_source='oedb'``). Sets + ``ctx.flags['has_electromobility'] = True``. + data_source : str, optional + ``"oedb"`` (egon_data) or ``"directory"`` (requires + ``import_electromobility_data_kwds={"charging_processes_dir": + ..., "potential_charging_points_dir": ...}``). + charging_strategy : str or None, optional + Charging strategy applied right after import. ``"dumb"`` + (uncontrolled, default), ``"reduced"``, ``"residual"``, or + ``None`` to skip. + flexibility_bands_ucs : str or list of str, optional + Charging-point use case(s) to compute flexibility bands for + via :meth:`Electromobility.get_flexibility_bands` after import + and charging-strategy application. Valid entries: + ``"home"``, ``"work"``, ``"public"``, ``"hpc"``. Pass a single + string for one use case or a list for multiple. ``None`` + (default) skips flexibility-band computation. + import_electromobility_data_kwds : dict, optional + Extra kwargs passed through to the underlying importer. + allocate_charging_demand_kwds : dict, optional + Extra kwargs for charging-demand allocation. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_electromobility( + data_source=data_source, + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + import_electromobility_data_kwds=import_electromobility_data_kwds, + allocate_charging_demand_kwds=allocate_charging_demand_kwds, + ) + if charging_strategy: + edisgo.apply_charging_strategy(strategy=charging_strategy) + if flexibility_bands_ucs is not None: + edisgo.electromobility.get_flexibility_bands( + edisgo, + use_case=flexibility_bands_ucs, + ) + ctx.flags["has_electromobility"] = True + return edisgo + + +@register_task("apply_charging_strategy") +def task_apply_charging_strategy(edisgo, ctx, *, strategy="dumb", + charging_park_ids=None): + """ + Apply a charging strategy to the already-imported EV fleet. + + Standalone variant of the step that ``import_electromobility`` + does inline. Useful when you want to import once and then try + multiple strategies in different runs. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + strategy : str, optional + Strategy name (``"dumb"`` / ``"reduced"`` / ``"residual"``). + charging_park_ids : list of int, optional + Restrict the strategy to these charging-park IDs. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.apply_charging_strategy( + strategy=strategy, charging_park_ids=charging_park_ids + ) + return edisgo + + +@register_task("apply_heat_pump_strategy") +def task_apply_heat_pump_strategy(edisgo, ctx, *, strategy="uncontrolled", + heat_pump_names=None): + """ + Apply a heat-pump operating strategy. + + Skipped with an info-log if no heat pumps are present + (``ctx.flags['has_heat_pumps']`` is falsy), so pipelines can + safely include this step without a conditional guard. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + strategy : str, optional + Operating strategy (``"uncontrolled"``, ``"flexible"``, …). + heat_pump_names : list of str, optional + Restrict to specific heat-pump load names; default is all. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if not ctx.flags.get("has_heat_pumps"): + ctx.logger.info( + "Skipping 'apply_heat_pump_strategy': no heat pumps " + "present." + ) + return edisgo + edisgo.apply_heat_pump_operating_strategy( + strategy=strategy, heat_pump_names=heat_pump_names + ) + return edisgo + + +@register_task("import_generators") +def task_import_generators(edisgo, ctx, *, generator_scenario=None): + """ + Import future generators for the active scenario. + + Thin wrapper around :meth:`EDisGo.import_generators`. Mostly + useful when you want to split grid loading and generator import + into two separate pipeline steps. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. ``ctx.scenario`` is used if + ``generator_scenario`` is not given. + generator_scenario : str, optional + Scenario name, e.g. ``"nep2035"`` or ``"ego100"``. Defaults + to ``ctx.scenario``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_generators( + generator_scenario=generator_scenario or ctx.scenario + ) + return edisgo From 56dbf7264e721c575efbdd43df4e2226aaa38f2b Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:22:18 +0200 Subject: [PATCH 14/66] Add file for grid-tasks, Add timeindex in setup task --- edisgo/run/tasks/grid.py | 172 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 edisgo/run/tasks/grid.py diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py new file mode 100644 index 000000000..b472f3244 --- /dev/null +++ b/edisgo/run/tasks/grid.py @@ -0,0 +1,172 @@ +""" +Grid loading tasks — bring an EDisGo instance into existence. + +Two ways to start a pipeline: + +* :func:`task_setup_grid` (``setup_grid``) — read a ding0 topology + from disk. This is the typical first step of every pipeline. +* :func:`task_load_from_base` (``load_from_base``) — reload a + previously saved EDisGo instance. Used to split a computation into + a slow "base" phase and one or more fast "scenario" phases that + reuse the base-reinforced grid. +""" +from __future__ import annotations + +from edisgo.run.registry import register_task + + +@register_task("setup_grid") +def task_setup_grid(edisgo, ctx, *, timeindex = None, ding0_path=None, legacy_ding0_grids=None, + import_generators=False, generator_scenario=None): + """ + Load a ding0 grid into an EDisGo instance. + + If the runner was started without an EDisGo object (via + :func:`edisgo.run.run_edisgo`) this task creates one from the + ding0 CSV directory. If an EDisGo object is already present (via + :meth:`edisgo.EDisGo.run_pipeline`), it imports the topology into + that existing instance. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Current EDisGo instance, or ``None`` to create a fresh one. + ctx : RunContext + Run context. ``ctx.raw_config['grid']`` is consulted when + parameters are not passed explicitly. + ding0_path : str, optional + Path to the ding0 grid directory. Falls back to + ``ctx.raw_config['grid']['ding0_path']``. + legacy_ding0_grids : bool, optional + Whether to treat the ding0 directory as the legacy format. + Falls back to ``ctx.raw_config['grid']['legacy_ding0_grids']`` + and ultimately to ``False``. + import_generators : bool, optional + If ``True``, call :meth:`EDisGo.import_generators` after + loading the grid. + generator_scenario : str, optional + Generator scenario name passed to + :meth:`EDisGo.import_generators` (only if + ``import_generators=True``). + + Returns + ------- + edisgo.EDisGo + The EDisGo instance with the ding0 topology loaded. + + Raises + ------ + ValueError + If no ``ding0_path`` is given either as a task parameter or + under ``config.grid.ding0_path``. + + """ + from edisgo import EDisGo + + grid_cfg = ctx.raw_config.get("grid", {}) + ding0_path = ding0_path or grid_cfg.get("ding0_path") + if ding0_path is None: + raise ValueError( + "Task 'setup_grid' requires 'ding0_path' either as task " + "parameter or under config.grid.ding0_path." + ) + if legacy_ding0_grids is None: + legacy_ding0_grids = grid_cfg.get("legacy_ding0_grids", False) + + if edisgo is None: + edisgo = EDisGo( + ding0_grid=str(ding0_path), + legacy_ding0_grids=legacy_ding0_grids, + ) + else: + edisgo.import_ding0_grid( + path=str(ding0_path), legacy_ding0_grids=legacy_ding0_grids + ) + + if import_generators: + edisgo.import_generators(generator_scenario=generator_scenario) + + if timeindex is not None: + ti_df = pd.date_range( + start=timeindex["start"], + periods=timeindex["periods"], + freq=timeindex.get("freq", "h"), + ) + edisgo.set_timeindex(ti_df) + + ctx.flags["grid_loaded"] = True + return edisgo + + +@register_task("load_from_base") +def task_load_from_base(edisgo, ctx, *, path, reset_equipment_changes=True, + import_timeseries=False, import_results=False, + import_electromobility=False, import_heat_pump=False, + import_dsm=False, import_overlying_grid=False): + """ + Reload an EDisGo instance from a previously saved directory/zip. + + This is the two-phase R4MU workflow's entry point: stage 1 + produces a base-reinforced grid and saves it, stage 2 (or N) + starts from ``load_from_base`` to pick up that grid and apply + scenario-specific modifications. The cost of the scenario then + shows up cleanly in ``equipment_changes`` because we reset it on + load. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Unused — the task always replaces whatever was there. + ctx : RunContext + Run context (logger only). + path : str + Directory or ``.zip`` produced by :func:`task_save`. + reset_equipment_changes : bool, optional + If ``True`` (default), clear + :attr:`Results.equipment_changes` so only the scenario's + reinforce is tracked. + import_timeseries : bool, optional + Whether to import the saved time series. Default: ``False`` + so the next stage sets its own. + import_results : bool, optional + Whether to import saved results. Default: ``False``. + import_electromobility : bool, optional + Whether to import saved electromobility data. + import_heat_pump : bool, optional + Whether to import saved heat-pump data. + import_dsm : bool, optional + Whether to import saved DSM data. + import_overlying_grid : bool, optional + Whether to import saved overlying-grid data (eTraGo + specifications). + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import os + + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + path = str(path) + from_zip = path.endswith(".zip") or not os.path.isdir(path) + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=import_timeseries, + import_results=import_results, + import_electromobility=import_electromobility, + import_heat_pump=import_heat_pump, + import_dsm=import_dsm, + import_overlying_grid=import_overlying_grid, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + ctx.flags["grid_loaded"] = True + return edisgo From 579aba8c0a31e4e645bba2def97fe4550cfaab0e Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:23:27 +0200 Subject: [PATCH 15/66] Add file for timeseries-tasks --- edisgo/run/tasks/timeseries.py | 298 +++++++++++++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 edisgo/run/tasks/timeseries.py diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py new file mode 100644 index 000000000..f738aa056 --- /dev/null +++ b/edisgo/run/tasks/timeseries.py @@ -0,0 +1,298 @@ +""" +Time-series tasks — set active/reactive power profiles on EDisGo. + +Time series drive every downstream step: ``analyze``, ``reinforce`` +and ``optimize`` all operate on the time index and power time series +attached to the EDisGo object. The order inside a stage matters: + +1. Set the time index and active-power profiles with one of + :func:`task_worst_case_ts`, :func:`task_oedb_ts`, + :func:`task_manual_ts`, possibly :func:`task_set_timeindex`. +2. Finally call :func:`task_reactive_power` to fix reactive power + control — this MUST come last because it overwrites whatever + reactive power was set by the earlier steps. +""" +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + + +@register_task("worst_case_ts") +def task_worst_case_ts(edisgo, ctx, *, cases=None, + generators_names=None, loads_names=None, + storage_units_names=None): + """ + Set synthetic worst-case active-power time series. + + Produces two snapshots (load case and feed-in case) that + represent the network's extremes. Useful for a coarse first + reinforce that does not require real load/generation data. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['timeseries_set'] = True``. + cases : list of str, optional + Subset of ``{"load_case", "feed-in_case"}``. Default is both. + generators_names : list of str, optional + Restrict to these generator names; default is all. + loads_names : list of str, optional + Restrict to these load names; default is all. + storage_units_names : list of str, optional + Restrict to these storage units; default is all. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.set_time_series_worst_case_analysis( + cases=cases, + generators_names=generators_names, + loads_names=loads_names, + storage_units_names=storage_units_names, + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("set_timeindex") +def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, + freq="h"): + """ + Set the time index on the EDisGo object. + + Useful as a stand-alone step when you want a specific hourly + range without immediately attaching time-series data (the + ``oedb_ts`` task already accepts a ``timeindex`` argument and + does this internally). + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + start : str or pandas.Timestamp + First timestamp of the range. + periods : int, optional + Number of periods; mutually exclusive with ``end``. + end : str or pandas.Timestamp, optional + Last timestamp; mutually exclusive with ``periods``. + freq : str, optional + pandas frequency string, default hourly (``"h"``). + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + Raises + ------ + ValueError + If neither ``periods`` nor ``end`` is provided. + + """ + if end is not None: + timeindex = pd.date_range(start=start, end=end, freq=freq) + else: + if periods is None: + raise ValueError( + "set_timeindex needs either 'periods' or 'end'." + ) + timeindex = pd.date_range(start=start, periods=periods, freq=freq) + edisgo.set_timeindex(timeindex) + return edisgo + + +@register_task("oedb_ts") +def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, + fluctuating="oedb", conventional_loads="oedb", + charging_points_ts=None): + """ + Set active-power time series from egon_data (OEP) plus overrides. + + This is the "real data" path: wind and solar profiles come from + ``egon_era5_renewable_feedin``, conventional loads come from the + egon demand tables. Dispatchable generators (conventional, + etc.) are set via a per-technology-type profile since egon_data + does not dispatch them. Storage units default to zero if not + already set. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()`` when any source is ``"oedb"``. Sets + ``ctx.flags['timeseries_set'] = True``. + timeindex : dict, optional + ``{"start": ..., "periods": N, "freq": "h"}``. If present, a + matching :class:`~pandas.DatetimeIndex` is set before + importing data. + dispatchable : dict, optional + Per-technology scaling factors, e.g. ``{"other": 0.7}`` → + constant profile of 0.7 p.u. for all non-fluctuating + generators of type "other". + fluctuating : str or pandas.DataFrame, optional + How to populate wind/solar. ``"oedb"`` pulls egon_data, + ``"default"`` uses bundled standard profiles, or a DataFrame + with columns "solar" / "wind" is passed through. + conventional_loads : str, optional + Source for conventional loads (not heat pumps / charging + points). ``"oedb"`` or ``"demandlib"``. + charging_points_ts : pandas.DataFrame, optional + Explicit active-power profile for charging points; default + ``None`` leaves them untouched so + :func:`task_apply_charging_strategy` can set them. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if timeindex is not None: + ti_df = pd.date_range( + start=timeindex["start"], + periods=timeindex["periods"], + freq=timeindex.get("freq", "h"), + ) + edisgo.set_timeindex(ti_df) + + dispatchable_df = None + if dispatchable is not None: + ti = edisgo.timeseries.timeindex + dispatchable_df = pd.DataFrame(dispatchable, index=ti) + + conv_loads_names = None + if conventional_loads == "oedb": + conv_loads_names = edisgo.topology.loads_df.loc[ + ~edisgo.topology.loads_df.type.isin( + ["heat_pump", "charging_point"] + ) + ].index.tolist() + + edisgo.set_time_series_active_power_predefined( + fluctuating_generators_ts=fluctuating, + conventional_loads_ts=conventional_loads, + conventional_loads_names=conv_loads_names, + dispatchable_generators_ts=dispatchable_df, + charging_points_ts=charging_points_ts, + scenario=ctx.scenario, + engine=ctx.ensure_engine() if fluctuating == "oedb" + or conventional_loads == "oedb" else None, + ) + + su_names = edisgo.topology.storage_units_df.index + if len(su_names) > 0 and edisgo.timeseries.storage_units_active_power.empty: + edisgo.timeseries.storage_units_active_power = pd.DataFrame( + 0.0, index=edisgo.timeseries.timeindex, columns=su_names, + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("manual_ts") +def task_manual_ts(edisgo, ctx, *, + generators_active_power=None, + generators_reactive_power=None, + loads_active_power=None, + loads_reactive_power=None, + storage_units_active_power=None, + storage_units_reactive_power=None): + """ + Set active/reactive power time series from explicit DataFrames. + + Used when the caller already has the raw profiles (e.g. from a + coupled run) and wants to inject them directly. Any argument left + at ``None`` is not touched. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['timeseries_set'] = True``. + generators_active_power : dict or pandas.DataFrame, optional + Generator active-power profile(s). Converted via + :class:`pandas.DataFrame`. + generators_reactive_power : dict or pandas.DataFrame, optional + Generator reactive-power profile(s). + loads_active_power : dict or pandas.DataFrame, optional + Load active-power profile(s). + loads_reactive_power : dict or pandas.DataFrame, optional + Load reactive-power profile(s). + storage_units_active_power : dict or pandas.DataFrame, optional + Storage-unit active-power profile(s). + storage_units_reactive_power : dict or pandas.DataFrame, optional + Storage-unit reactive-power profile(s). + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + def _as_df(obj): + return pd.DataFrame(obj) if obj is not None else None + + edisgo.set_time_series_manual( + generators_active_power=_as_df(generators_active_power), + generators_reactive_power=_as_df(generators_reactive_power), + loads_active_power=_as_df(loads_active_power), + loads_reactive_power=_as_df(loads_reactive_power), + storage_units_active_power=_as_df(storage_units_active_power), + storage_units_reactive_power=_as_df(storage_units_reactive_power), + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("reactive_power") +def task_reactive_power(edisgo, ctx, *, control="fixed_cosphi", + generators_parametrisation="default", + loads_parametrisation="default", + storage_units_parametrisation="default"): + """ + Apply reactive-power control on top of the active-power time series. + + This MUST be the last time-series-altering step before + ``analyze`` / ``reinforce`` / ``optimize``. The validator + enforces this ordering rule statically. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['reactive_power_set'] = True``. + control : str, optional + Reactive-power control strategy; typically ``"fixed_cosphi"``. + generators_parametrisation : str or dict, optional + Per-generator parametrisation, ``"default"`` uses the config. + loads_parametrisation : str or dict, optional + Per-load parametrisation. + storage_units_parametrisation : str or dict, optional + Per-storage-unit parametrisation. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.set_time_series_reactive_power_control( + control=control, + generators_parametrisation=generators_parametrisation, + loads_parametrisation=loads_parametrisation, + storage_units_parametrisation=storage_units_parametrisation, + ) + ctx.flags["reactive_power_set"] = True + return edisgo From 65fa8fc0ba73c41eae0adb408dc39b2b26873f3a Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Tue, 19 May 2026 16:40:05 +0200 Subject: [PATCH 16/66] Edit uc4 example --- edisgo/run/presets/uc4_example_MS.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example_MS.yaml index a72573496..13e08b258 100644 --- a/edisgo/run/presets/uc4_example_MS.yaml +++ b/edisgo/run/presets/uc4_example_MS.yaml @@ -24,16 +24,21 @@ _workflow: scenario: eGon2035 grid: ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + # grid path should be set in run file legacy_ding0_grids: false + # legacy parameter too database: ssh: enabled: false timeindex: {start: "2035-01-01", periods: 24, freq: h} +# only set in preset, not set in run-functions (eDisGo and eGo) results: directory: results/uc4_example +# actually all parameters + pipeline: - setup_grid @@ -42,6 +47,7 @@ pipeline: - import_home_batteries - import_heat_pumps - import_dsm + # where is decided, which electromobility use cases are used? - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - apply_heat_pump_strategy: {strategy: uncontrolled} - oedb_ts: @@ -49,6 +55,7 @@ pipeline: - reactive_power - check_integrity - optimize: + # where is decided which flexibilities are used in the OPF? flexible: [heat_pumps, storage, charging_points, dsm] method: soc opf_version: 2 From 7ad649083bc70d1fc776b83f1fb6c6deb0e262f2 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Wed, 20 May 2026 18:17:15 +0200 Subject: [PATCH 17/66] Wire overlying-grid data through pipeline runner Adds optional overlying_grid_data kwarg to run_edisgo() that is stashed on RunContext for downstream tasks instead of being passed as a keyword to every task (which broke task signatures). task_import_overlying_grid_data now: - accepts the standard (edisgo, ctx, *, ...) signature - reads overlying_grid_data from ctx, falls back to overlying_grid.path in the runner config - loads dispatchable + renewables_potential CSVs and applies them via set_time_series_active_power_predefined - shifts the year and reindexes overlying-grid attributes to the active edisgo timeindex so CSV-based input lines up with OEDB time series task_set_timeindex now reduces existing time-series data to the new index (via reduce_timeseries_data_to_given_timeindex) instead of silently leaving stale data behind. Renames the uc4_example_MS preset to uc4_example. --- .../{uc4_example_MS.yaml => uc4_example.yaml} | 9 +- edisgo/run/runner.py | 8 +- edisgo/run/tasks/io.py | 158 ++++++++++++++++-- edisgo/run/tasks/timeseries.py | 85 ++++++---- 4 files changed, 214 insertions(+), 46 deletions(-) rename edisgo/run/presets/{uc4_example_MS.yaml => uc4_example.yaml} (88%) diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example.yaml similarity index 88% rename from edisgo/run/presets/uc4_example_MS.yaml rename to edisgo/run/presets/uc4_example.yaml index 13e08b258..2d2650279 100644 --- a/edisgo/run/presets/uc4_example_MS.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -23,7 +23,7 @@ _workflow: scenario: eGon2035 grid: - ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + ding0_path: "/path/to/ding0_grid" # grid path should be set in run file legacy_ding0_grids: false # legacy parameter too @@ -48,10 +48,13 @@ pipeline: - import_heat_pumps - import_dsm # where is decided, which electromobility use cases are used? - - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - - apply_heat_pump_strategy: {strategy: uncontrolled} + - import_electromobility: {charging_strategy: null, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - oedb_ts: dispatchable: {other: 0.7} + timeindex: {start: "2035-01-01", periods: 24, freq: h} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - import_overlying_grid_data - reactive_power - check_integrity - optimize: diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py index 63f30aa07..2d08fd3bc 100644 --- a/edisgo/run/runner.py +++ b/edisgo/run/runner.py @@ -27,6 +27,7 @@ * :func:`_run_pipeline_on` — starts from an existing EDisGo instance; used by :meth:`edisgo.EDisGo.run_pipeline`. """ + from __future__ import annotations import logging @@ -43,7 +44,7 @@ logger = logging.getLogger("edisgo.run.runner") -def run_edisgo(config) -> Any: +def run_edisgo(config, overlying_grid_data=None) -> Any: """ Run an eDisGo pipeline from a YAML/JSON config or dict. @@ -66,10 +67,10 @@ def run_edisgo(config) -> Any: stage. """ - return _run_pipeline_on(None, config) + return _run_pipeline_on(None, config, overlying_grid_data=overlying_grid_data) -def _run_pipeline_on(edisgo, config): +def _run_pipeline_on(edisgo, config, overlying_grid_data=None): """ Internal runner shared by :func:`run_edisgo` and the EDisGo method. @@ -97,6 +98,7 @@ def _run_pipeline_on(edisgo, config): cfg = load_config(config) validate(cfg) ctx = _build_context(cfg) + ctx.overlying_grid_data = overlying_grid_data for stage in cfg["stages"]: ctx.current_stage = stage["name"] diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 3f604914e..4e2e84d15 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -10,6 +10,7 @@ integrating scenario charging stations from a directory of CSV / GeoPackage files; implementation is deferred until needed. """ + from __future__ import annotations import os @@ -18,12 +19,24 @@ @register_task("save") -def task_save(edisgo, ctx, *, directory=None, save_topology=True, - save_timeseries=True, save_results=True, - save_electromobility=None, save_opf_results=False, - save_heatpump=None, save_overlying_grid=False, - save_dsm=None, archive=False, archive_type="zip", - reduce_memory=False, parameters=None): +def task_save( + edisgo, + ctx, + *, + directory=None, + save_topology=True, + save_timeseries=True, + save_results=True, + save_electromobility=None, + save_opf_results=False, + save_heatpump=None, + save_overlying_grid=False, + save_dsm=None, + archive=False, + archive_type="zip", + reduce_memory=False, + parameters=None, +): """ Save the current EDisGo state to disk. @@ -93,8 +106,7 @@ def task_save(edisgo, ctx, *, directory=None, save_topology=True, if directory is None: if ctx.results_dir is None: raise ValueError( - "Task 'save' needs a 'directory' parameter or " - "config.results.directory." + "Task 'save' needs a 'directory' parameter or config.results.directory." ) stage = ctx.current_stage or "main" directory = os.path.join(str(ctx.results_dir), stage) @@ -135,9 +147,9 @@ def task_save(edisgo, ctx, *, directory=None, save_topology=True, @register_task("load_charging_from_files") -def task_load_charging_from_files(edisgo, ctx, *, charging_dir, - use_case_to_sector=None, - mv_threshold_kw=100.0): +def task_load_charging_from_files( + edisgo, ctx, *, charging_dir, use_case_to_sector=None, mv_threshold_kw=100.0 +): """ Integrate scenario charging stations from files (R4MU workflow). @@ -177,3 +189,127 @@ def task_load_charging_from_files(edisgo, ctx, *, charging_dir, "_run_edisgo_task_load_charging_from_files when R4MU is " "needed." ) + + +@register_task("import_overlying_grid_data") +def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): + """ + Import overlying grid data into the EDisGo instance. + + When ``overlying_grid_data`` is a dict of DataFrames (as returned by + ``get_etrago_results_per_bus``), the overlying-grid attributes and + dispatchable/fluctuating generator time series are set from it. + + When ``overlying_grid_path`` is a directory path, the overlying-grid + attributes are loaded from CSV files in that directory, and + ``dispatchable_generators_active_power.csv`` / + ``renewables_potential.csv`` are applied as generator time series + if present. + + Falls back to ``ctx.raw_config['eDisGo']['overlying_grid_source']`` + as the directory path when neither argument is given. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + overlying_grid_path : str, optional + Directory containing overlying-grid CSV files. + overlying_grid_data : dict, optional + Dict of DataFrames as returned by ``get_etrago_results_per_bus``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + import pandas as pd + + overlying_grid_data = getattr(ctx, "overlying_grid_data", None) + + if overlying_grid_data is not None: + # eTraGo results dict — set standard overlying-grid attributes + for attr in edisgo.overlying_grid._attributes: + if attr in overlying_grid_data: + setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + # set generator time series + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=overlying_grid_data.get( + "dispatchable_generators_active_power" + ), + fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), + ) + return edisgo + + # resolve path: explicit arg → runner config overlying_grid.path → skip + if overlying_grid_path is None: + overlying_grid_path = (ctx.raw_config.get("overlying_grid") or {}).get("path") + + if overlying_grid_path is None: + ctx.logger.warning( + "task 'import_overlying_grid_data': no overlying_grid_data or " + "overlying_grid_path provided — skipping." + ) + return edisgo + + # load overlying-grid attributes from CSV directory + edisgo.overlying_grid.from_csv(overlying_grid_path) + + # reindex overlying-grid attributes to match edisgo timeindex + # CSVs may use a different year — shift year then reindex + edisgo_ti = edisgo.timeseries.timeindex + if not edisgo_ti.empty: + for attr in edisgo.overlying_grid._attributes: + ts = getattr(edisgo.overlying_grid, attr) + if ts.empty: + continue + csv_year = ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) + if isinstance(ts, pd.Series): + setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + else: + setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + + # load dispatchable generator and renewables time series from the same dir + disp_path = os.path.join( + overlying_grid_path, "dispatchable_generators_active_power.csv" + ) + if os.path.isfile(disp_path): + disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = disp_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + disp_ts.index = disp_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + disp_ts = disp_ts.reindex(edisgo_ti) + else: + disp_ts = None + + pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") + if os.path.isfile(pot_path): + pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = pot_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + pot_ts.index = pot_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + pot_ts = pot_ts.reindex(edisgo_ti) + else: + pot_ts = None + + if disp_ts is not None or pot_ts is not None: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + fluctuating_generators_ts=pot_ts, + ) + + return edisgo diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py index f738aa056..cf918c235 100644 --- a/edisgo/run/tasks/timeseries.py +++ b/edisgo/run/tasks/timeseries.py @@ -12,6 +12,7 @@ control — this MUST come last because it overwrites whatever reactive power was set by the earlier steps. """ + from __future__ import annotations import pandas as pd @@ -20,9 +21,15 @@ @register_task("worst_case_ts") -def task_worst_case_ts(edisgo, ctx, *, cases=None, - generators_names=None, loads_names=None, - storage_units_names=None): +def task_worst_case_ts( + edisgo, + ctx, + *, + cases=None, + generators_names=None, + loads_names=None, + storage_units_names=None, +): """ Set synthetic worst-case active-power time series. @@ -62,8 +69,7 @@ def task_worst_case_ts(edisgo, ctx, *, cases=None, @register_task("set_timeindex") -def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, - freq="h"): +def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, freq="h"): """ Set the time index on the EDisGo object. @@ -98,22 +104,32 @@ def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, If neither ``periods`` nor ``end`` is provided. """ + from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex + if end is not None: timeindex = pd.date_range(start=start, end=end, freq=freq) else: if periods is None: - raise ValueError( - "set_timeindex needs either 'periods' or 'end'." - ) + raise ValueError("set_timeindex needs either 'periods' or 'end'.") timeindex = pd.date_range(start=start, periods=periods, freq=freq) - edisgo.set_timeindex(timeindex) + if edisgo.timeseries.timeindex.empty: + edisgo.set_timeindex(timeindex) + else: + reduce_timeseries_data_to_given_timeindex(edisgo, timeindex) return edisgo @register_task("oedb_ts") -def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, - fluctuating="oedb", conventional_loads="oedb", - charging_points_ts=None): +def task_oedb_ts( + edisgo, + ctx, + *, + timeindex=None, + dispatchable=None, + fluctuating="oedb", + conventional_loads="oedb", + charging_points_ts=None, +): """ Set active-power time series from egon_data (OEP) plus overrides. @@ -174,9 +190,7 @@ def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, conv_loads_names = None if conventional_loads == "oedb": conv_loads_names = edisgo.topology.loads_df.loc[ - ~edisgo.topology.loads_df.type.isin( - ["heat_pump", "charging_point"] - ) + ~edisgo.topology.loads_df.type.isin(["heat_pump", "charging_point"]) ].index.tolist() edisgo.set_time_series_active_power_predefined( @@ -186,27 +200,34 @@ def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, dispatchable_generators_ts=dispatchable_df, charging_points_ts=charging_points_ts, scenario=ctx.scenario, - engine=ctx.ensure_engine() if fluctuating == "oedb" - or conventional_loads == "oedb" else None, + engine=ctx.ensure_engine() + if fluctuating == "oedb" or conventional_loads == "oedb" + else None, ) su_names = edisgo.topology.storage_units_df.index if len(su_names) > 0 and edisgo.timeseries.storage_units_active_power.empty: edisgo.timeseries.storage_units_active_power = pd.DataFrame( - 0.0, index=edisgo.timeseries.timeindex, columns=su_names, + 0.0, + index=edisgo.timeseries.timeindex, + columns=su_names, ) ctx.flags["timeseries_set"] = True return edisgo @register_task("manual_ts") -def task_manual_ts(edisgo, ctx, *, - generators_active_power=None, - generators_reactive_power=None, - loads_active_power=None, - loads_reactive_power=None, - storage_units_active_power=None, - storage_units_reactive_power=None): +def task_manual_ts( + edisgo, + ctx, + *, + generators_active_power=None, + generators_reactive_power=None, + loads_active_power=None, + loads_reactive_power=None, + storage_units_active_power=None, + storage_units_reactive_power=None, +): """ Set active/reactive power time series from explicit DataFrames. @@ -240,6 +261,7 @@ def task_manual_ts(edisgo, ctx, *, The modified EDisGo instance. """ + def _as_df(obj): return pd.DataFrame(obj) if obj is not None else None @@ -256,10 +278,15 @@ def _as_df(obj): @register_task("reactive_power") -def task_reactive_power(edisgo, ctx, *, control="fixed_cosphi", - generators_parametrisation="default", - loads_parametrisation="default", - storage_units_parametrisation="default"): +def task_reactive_power( + edisgo, + ctx, + *, + control="fixed_cosphi", + generators_parametrisation="default", + loads_parametrisation="default", + storage_units_parametrisation="default", +): """ Apply reactive-power control on top of the active-power time series. From 5fce2573c2dc66fdeb81d847ec39bf8bfc65e2a4 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Thu, 21 May 2026 16:52:24 +0200 Subject: [PATCH 18/66] Redirect to installed julia version --- edisgo/opf/powermodels_opf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edisgo/opf/powermodels_opf.py b/edisgo/opf/powermodels_opf.py index c26475141..6fbdd1e41 100644 --- a/edisgo/opf/powermodels_opf.py +++ b/edisgo/opf/powermodels_opf.py @@ -130,7 +130,7 @@ def _convert(o): logger.info("starting julia process") julia_process = subprocess.Popen( [ - "julia", + "/opt/julia-1.8.3/bin/julia", os.path.join(opf_dir, "eDisGo_OPF.jl/Main.jl"), pm["name"], solution_dir, From d6eddc27895a5a41eaf1e6a0a727de4512a415f0 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Thu, 21 May 2026 16:26:21 +0200 Subject: [PATCH 19/66] Fix OPF result write-back, flex resolution and overlying-grid SOC reindex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit powermodels_io.from_powermodels now slices the destination time index explicitly when writing OPF flex results back to the EDisGo timeseries (gen_nd, heatpumps, electromobility, dsm, storage). Using `loc[:, names]` overwrites every row of the underlying DataFrame even when the OPF only covered a subset of timesteps; restricting to `timeseries.timeindex` keeps untouched rows intact. task_optimize: - fixes typo `flexbile` → `flexible` in the dsm shortcut - corrects the dsm condition (was checking `is not None`, should populate when `None`) - materializes empty flex lists once instead of repeating `or []` at every call site task_import_overlying_grid_data reindexes the three SOC attributes (storage_units_soc, thermal_storage_units_{central,decentral}_soc) to timeindex + 1 extra step, because PowerModels expects the end-of-period SOC value; non-SOC overlying-grid attributes still reindex to the plain timeindex. uc4_example preset reworked: switches to opf_version 3 (HV constraints from overlying grid), drops base_reinforce + check_integrity from the pipeline, adds an explicit `overlying_grid.path` config slot, splits import_electromobility kwargs onto separate lines, and enables archive + save_opf_results on the final save step. --- edisgo/io/powermodels_io.py | 33 +++++++----- edisgo/run/presets/uc4_example.yaml | 47 ++++++++--------- edisgo/run/tasks/analysis.py | 79 +++++++++++++++++++++-------- edisgo/run/tasks/io.py | 14 +++-- 4 files changed, 108 insertions(+), 65 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index e82cc4734..6bf21affb 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -250,10 +250,10 @@ def from_powermodels( Base value of apparent power for per unit system. Default: 1 MVA. """ - if type(pm_results) == str: + if isinstance(pm_results, str): with open(pm_results) as f: pm = json.loads(json.load(f)) - elif type(pm_results) == dict: + elif isinstance(pm_results, dict): pm = pm_results else: raise ValueError( @@ -306,17 +306,20 @@ def from_powermodels( ] results = pd.DataFrame(index=timesteps, columns=names, data=data) if (flex == "gen_nd") & (pm["nw"]["1"]["opf_version"] in [3, 4]): - edisgo_object.timeseries._generators_active_power.loc[:, names] = ( + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._generators_active_power.loc[ti, names] = ( edisgo_object.timeseries.generators_active_power.loc[:, names].values - results[names].values ) elif flex in ["heatpumps", "electromobility"]: - edisgo_object.timeseries._loads_active_power.loc[:, names] = results[ + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._loads_active_power.loc[ti, names] = results[ names ].values elif flex == "dsm": - edisgo_object.timeseries._loads_active_power.loc[:, names] = ( - edisgo_object.timeseries._loads_active_power.loc[:, names].values + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._loads_active_power.loc[ti, names] = ( + edisgo_object.timeseries._loads_active_power.loc[ti, names].values + results[names].values ) elif flex == "storage": @@ -328,8 +331,9 @@ def from_powermodels( data=results[names].values, ) else: + ti = edisgo_object.timeseries.timeindex edisgo_object.timeseries._storage_units_active_power.loc[ - :, names + ti, names ] = results[names].values except AttributeError: setattr( @@ -787,8 +791,8 @@ def _build_branch(edisgo_obj, psa_net, pm, flexible_storage_units, s_base): # only modify r, x and l values if min value is too small branches[par] = val.clip(lower=min_value) logger.warning( - f"Min value of {text} is too small. Lowest {100 * quant}% of {text} values will be set " - f"to {min_value} {unit}" + f"Min value of {text} is too small. Lowest {100 * quant}% of " + f"{text} values will be set to {min_value} {unit}" ) for branch_i in np.arange(len(branches.index)): @@ -933,8 +937,8 @@ def _build_load( pf, sign = _get_pf(edisgo_obj, pm, idx_bus, "charging_point") else: logger.warning( - f"No type specified for load {loads_df.index[load_i]}. Power factor and sign will" - "be set for conventional load." + f"No type specified for load {loads_df.index[load_i]}. " + "Power factor and sign will be set for conventional load." ) pf, sign = _get_pf(edisgo_obj, pm, idx_bus, "conventional_load") p_d = psa_net.loads_t.p_set[loads_df.index[load_i]] @@ -1219,9 +1223,10 @@ def _build_heatpump(psa_net, pm, edisgo_obj, s_base, flexible_hps): comparison = (heat_df2[hp_p_nom.index] > hp_cop * hp_p_nom.squeeze()).any() if comparison.any(): logger.warning( - "Heat demand is higher than rated heatpump power" - f" of heatpumps: {comparison.index[comparison.values].values}. Demand can not be covered if no sufficient" - " heat storage capacities are available." + "Heat demand is higher than rated heatpump power of heatpumps: " + f"{comparison.index[comparison.values].values}. " + "Demand can not be covered if no sufficient heat storage " + "capacities are available." ) for hp_i in np.arange(len(heat_df.index)): idx_bus = _mapping(psa_net, edisgo_obj, heat_df.bus.iloc[hp_i]) diff --git a/edisgo/run/presets/uc4_example.yaml b/edisgo/run/presets/uc4_example.yaml index 2d2650279..d3f9efd90 100644 --- a/edisgo/run/presets/uc4_example.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -1,54 +1,51 @@ _comment: | - UC3 — OPF with full flexibility: - Like UC1 but loads real egon_data time series (oedb) and runs a - powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) - before the final reinforce. Cost delta = extra reinforcement needed - under optimal flex dispatch. + UC4 — OPF with full flexibility: + Loads real egon_data time series (oedb) and runs a powermodels OPF + over flexibilities (heat pumps, EV, DSM, storage) with HV requirements + from overlying grid. opf_version 3 activates HV-constraints from + overlying_grid CSV directory. _workflow: - - setup_grid: load ding0 topology, import generators - - base_reinforce: worst-case TS + reinforce + reset equipment_changes - - import_generators: from edon-data - - import_heat_pumps: from egon_data + - setup_grid: load ding0 topology + - import_generators: from egon_data - import_home_batteries: from egon_data + - import_heat_pumps: from egon_data - import_dsm: from egon_data - import_electromobility: from egon_data (dumb charging, flex bands) - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_charging_strategy: dumb - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) - - reactive_power - - check_integrity - - optimize: pm_optimize with flex assets (SOC, opf v2) - - reinforce: final reinforcement - - save + - import_overlying_grid_data: HV constraints from CSV dir + - optimize: pm_optimize with flex assets (SOC, opf v3 = HV constraints) scenario: eGon2035 + grid: ding0_path: "/path/to/ding0_grid" - # grid path should be set in run file legacy_ding0_grids: false - # legacy parameter too database: ssh: enabled: false timeindex: {start: "2035-01-01", periods: 24, freq: h} -# only set in preset, not set in run-functions (eDisGo and eGo) + +overlying_grid: + path: "/path/to/overlying_grid_csv_dir" results: directory: results/uc4_example -# actually all parameters pipeline: - setup_grid - - base_reinforce - import_generators - import_home_batteries - import_heat_pumps - import_dsm - # where is decided, which electromobility use cases are used? - - import_electromobility: {charging_strategy: null, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} + - import_electromobility: + charging_strategy: null + flexibility_bands_ucs: ["home", "work", "public", "hpc"] - oedb_ts: dispatchable: {other: 0.7} timeindex: {start: "2035-01-01", periods: 24, freq: h} @@ -56,11 +53,11 @@ pipeline: - apply_heat_pump_strategy: {strategy: uncontrolled} - import_overlying_grid_data - reactive_power - - check_integrity - optimize: - # where is decided which flexibilities are used in the OPF? flexible: [heat_pumps, storage, charging_points, dsm] method: soc - opf_version: 2 + opf_version: 3 - reinforce - - save + - save: + archive: true + save_opf_results: true diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py index f028bb31c..becff93b9 100644 --- a/edisgo/run/tasks/analysis.py +++ b/edisgo/run/tasks/analysis.py @@ -21,6 +21,7 @@ produce a "base" grid whose subsequent reinforce costs reflect only a scenario overlay. """ + from __future__ import annotations import pandas as pd @@ -55,8 +56,15 @@ def task_check_integrity(edisgo, ctx): @register_task("analyze") -def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, - raise_not_converged=False, troubleshooting_mode=None): +def task_analyze( + edisgo, + ctx, + *, + mode=None, + timesteps=None, + raise_not_converged=False, + troubleshooting_mode=None, +): """ Run AC power flow over the active time series. @@ -97,18 +105,26 @@ def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, ctx.flags["not_converged_steps"] = len(not_converged) if len(not_converged) > 0: ctx.logger.warning( - f"Power flow did not converge for {len(not_converged)} " - f"time steps." + f"Power flow did not converge for {len(not_converged)} time steps." ) return edisgo @register_task("reinforce") -def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, - copy_grid=False, max_while_iterations=20, - split_voltage_band=True, mode=None, - without_generator_import=False, n_minus_one=False, - catch_convergence_problems=False): +def task_reinforce( + edisgo, + ctx, + *, + timesteps_pfa=None, + reduced_analysis=False, + copy_grid=False, + max_while_iterations=20, + split_voltage_band=True, + mode=None, + without_generator_import=False, + n_minus_one=False, + catch_convergence_problems=False, +): """ Run iterative grid reinforcement. @@ -167,8 +183,9 @@ def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, @register_task("base_reinforce") -def task_base_reinforce(edisgo, ctx, *, cases=None, - reset_equipment_changes=True, save_artifact=True): +def task_base_reinforce( + edisgo, ctx, *, cases=None, reset_equipment_changes=True, save_artifact=True +): """ Produce a base-reinforced grid and reset the cost accumulator. @@ -242,10 +259,20 @@ def task_base_reinforce(edisgo, ctx, *, cases=None, @register_task("optimize") -def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, - flexible_hps=None, flexible_loads=None, - flexible_storage_units=None, opf_version=2, method="soc", - warm_start=False, s_base=1): +def task_optimize( + edisgo, + ctx, + *, + flexible=None, + flexible_cps=None, + flexible_hps=None, + flexible_loads=None, + flexible_storage_units=None, + opf_version=2, + method="soc", + warm_start=False, + s_base=1, +): """ Run a powermodels optimal-power-flow (OPF) over flexibilities. @@ -304,15 +331,23 @@ def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, ].index.tolist() if flexible_storage_units is None and "storage" in flexible: flexible_storage_units = edisgo.topology.storage_units_df.index.tolist() - if flexible_loads is not None and "dsm" in flexbile: - flexible_loads = edisgo.dsm.p_min.columns.values - + if flexible_loads is None and "dsm" in flexible: + flexible_loads = edisgo.dsm.p_min.columns.values + + if flexible_cps is None: + flexible_cps = [] + if flexible_hps is None: + flexible_hps = [] + if flexible_loads is None: + flexible_loads = [] + if flexible_storage_units is None: + flexible_storage_units = [] edisgo.pm_optimize( - flexible_cps=flexible_cps or [], - flexible_hps=flexible_hps or [], - flexible_loads=flexible_loads or [], - flexible_storage_units=flexible_storage_units or [], + flexible_cps=flexible_cps, + flexible_hps=flexible_hps, + flexible_loads=flexible_loads, + flexible_storage_units=flexible_storage_units, opf_version=opf_version, method=method, warm_start=warm_start, diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 4e2e84d15..c2b748cf0 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -262,6 +262,14 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): # CSVs may use a different year — shift year then reindex edisgo_ti = edisgo.timeseries.timeindex if not edisgo_ti.empty: + # SOC needs one extra step at the end (end-of-period state) + ti_freq = edisgo_ti.freq or (edisgo_ti[1] - edisgo_ti[0]) + edisgo_ti_plus1 = edisgo_ti.union([edisgo_ti[-1] + ti_freq]) + soc_attrs = { + "storage_units_soc", + "thermal_storage_units_decentral_soc", + "thermal_storage_units_central_soc", + } for attr in edisgo.overlying_grid._attributes: ts = getattr(edisgo.overlying_grid, attr) if ts.empty: @@ -270,10 +278,8 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): edisgo_year = edisgo_ti[0].year if csv_year != edisgo_year: ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) - if isinstance(ts, pd.Series): - setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) - else: - setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti + setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) # load dispatchable generator and renewables time series from the same dir disp_path = os.path.join( From fc84a14bc5d3b8f524a028f8ed11aeb4fc3ca02b Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Thu, 21 May 2026 17:40:36 +0200 Subject: [PATCH 20/66] Add overly_grid boolean --- edisgo/run/tasks/io.py | 171 +++++++++++++++++++++-------------------- 1 file changed, 87 insertions(+), 84 deletions(-) diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index c2b748cf0..258148eb8 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -227,95 +227,98 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): """ import pandas as pd - + overlying_grid_data = getattr(ctx, "overlying_grid_data", None) - if overlying_grid_data is not None: - # eTraGo results dict — set standard overlying-grid attributes - for attr in edisgo.overlying_grid._attributes: - if attr in overlying_grid_data: - setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) - # set generator time series - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=overlying_grid_data.get( - "dispatchable_generators_active_power" - ), - fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), - ) - return edisgo + if overlying_grid: + + + if overlying_grid_data is not None: + # eTraGo results dict — set standard overlying-grid attributes + for attr in edisgo.overlying_grid._attributes: + if attr in overlying_grid_data: + setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + # set generator time series + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=overlying_grid_data.get( + "dispatchable_generators_active_power" + ), + fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), + ) + return edisgo - # resolve path: explicit arg → runner config overlying_grid.path → skip - if overlying_grid_path is None: - overlying_grid_path = (ctx.raw_config.get("overlying_grid") or {}).get("path") + # resolve path: explicit arg → runner config overlying_grid.path → skip + if overlying_grid_path is None: + overlying_grid_path = (ctx.raw_config.get("overlying_grid_path") or {}).get("path") - if overlying_grid_path is None: - ctx.logger.warning( - "task 'import_overlying_grid_data': no overlying_grid_data or " - "overlying_grid_path provided — skipping." - ) - return edisgo - - # load overlying-grid attributes from CSV directory - edisgo.overlying_grid.from_csv(overlying_grid_path) - - # reindex overlying-grid attributes to match edisgo timeindex - # CSVs may use a different year — shift year then reindex - edisgo_ti = edisgo.timeseries.timeindex - if not edisgo_ti.empty: - # SOC needs one extra step at the end (end-of-period state) - ti_freq = edisgo_ti.freq or (edisgo_ti[1] - edisgo_ti[0]) - edisgo_ti_plus1 = edisgo_ti.union([edisgo_ti[-1] + ti_freq]) - soc_attrs = { - "storage_units_soc", - "thermal_storage_units_decentral_soc", - "thermal_storage_units_central_soc", - } - for attr in edisgo.overlying_grid._attributes: - ts = getattr(edisgo.overlying_grid, attr) - if ts.empty: - continue - csv_year = ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) - target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti - setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) - - # load dispatchable generator and renewables time series from the same dir - disp_path = os.path.join( - overlying_grid_path, "dispatchable_generators_active_power.csv" - ) - if os.path.isfile(disp_path): - disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) - if not edisgo_ti.empty: - csv_year = disp_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - disp_ts.index = disp_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - disp_ts = disp_ts.reindex(edisgo_ti) - else: - disp_ts = None - - pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") - if os.path.isfile(pot_path): - pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) + if overlying_grid_path is None: + ctx.logger.warning( + "task 'import_overlying_grid_data': no overlying_grid_data or " + "overlying_grid_path provided — skipping." + ) + return edisgo + + # load overlying-grid attributes from CSV directory + edisgo.overlying_grid.from_csv(overlying_grid_path) + + # reindex overlying-grid attributes to match edisgo timeindex + # CSVs may use a different year — shift year then reindex + edisgo_ti = edisgo.timeseries.timeindex if not edisgo_ti.empty: - csv_year = pot_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - pot_ts.index = pot_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - pot_ts = pot_ts.reindex(edisgo_ti) - else: - pot_ts = None - - if disp_ts is not None or pot_ts is not None: - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=disp_ts, - fluctuating_generators_ts=pot_ts, + # SOC needs one extra step at the end (end-of-period state) + ti_freq = edisgo_ti.freq or (edisgo_ti[1] - edisgo_ti[0]) + edisgo_ti_plus1 = edisgo_ti.union([edisgo_ti[-1] + ti_freq]) + soc_attrs = { + "storage_units_soc", + "thermal_storage_units_decentral_soc", + "thermal_storage_units_central_soc", + } + for attr in edisgo.overlying_grid._attributes: + ts = getattr(edisgo.overlying_grid, attr) + if ts.empty: + continue + csv_year = ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) + target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti + setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) + + # load dispatchable generator and renewables time series from the same dir + disp_path = os.path.join( + overlying_grid_path, "dispatchable_generators_active_power.csv" ) + if os.path.isfile(disp_path): + disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = disp_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + disp_ts.index = disp_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + disp_ts = disp_ts.reindex(edisgo_ti) + else: + disp_ts = None + + pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") + if os.path.isfile(pot_path): + pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = pot_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + pot_ts.index = pot_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + pot_ts = pot_ts.reindex(edisgo_ti) + else: + pot_ts = None + + if disp_ts is not None or pot_ts is not None: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + fluctuating_generators_ts=pot_ts, + ) return edisgo From d4672e370d7af9c6e22468074d098234cab5d238 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Fri, 22 May 2026 10:48:29 +0200 Subject: [PATCH 21/66] Add full-flex distribution OPF configuration without overlying-grid constraints --- edisgo/run/presets/flex_opf_full.yaml | 61 +++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 edisgo/run/presets/flex_opf_full.yaml diff --git a/edisgo/run/presets/flex_opf_full.yaml b/edisgo/run/presets/flex_opf_full.yaml new file mode 100644 index 000000000..2ddb5cca7 --- /dev/null +++ b/edisgo/run/presets/flex_opf_full.yaml @@ -0,0 +1,61 @@ +_comment: | + Full-flex distribution OPF, no overlying-grid constraints. + Loads real egon_data time series (oedb) and runs a powermodels OPF + over the full flexibility set (heat pumps, EV, DSM, storage) on the + distribution grid alone. opf_version 2 — no HV requirements from an + overlying grid (use the variant with opf_version 3 for that). + Suitable for runs without eTraGo / without an overlying-grid CSV. + +_workflow: + - setup_grid: load ding0 topology + - import_generators: from egon_data + - import_home_batteries: from egon_data + - import_heat_pumps: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_charging_strategy: dumb + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - optimize: pm_optimize with full flex set (SOC, opf v2) + - reinforce: final reinforcement under optimized dispatch + - save + +scenario: eGon2035 + +grid: + ding0_path: "/path/to/ding0_grid" + legacy_ding0_grids: false + +database: + ssh: + enabled: false + +timeindex: {start: "2035-01-01", periods: 24, freq: h} + +results: + directory: results/flex_opf_full + + +pipeline: + - setup_grid + - import_generators + - import_home_batteries + - import_heat_pumps + - import_dsm + - import_electromobility: + charging_strategy: null + flexibility_bands_ucs: ["home", "work", "public", "hpc"] + - oedb_ts: + dispatchable: {other: 0.7} + timeindex: {start: "2035-01-01", periods: 24, freq: h} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - reinforce + - save: + archive: true + save_opf_results: true From 8d6a947c72dad238af556eac8927284244e0c232 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Wed, 27 May 2026 10:00:09 +0200 Subject: [PATCH 22/66] Add overlying grid enabler and source selection --- edisgo/run/config.py | 2 + edisgo/run/presets/uc4_example.yaml | 6 ++- edisgo/run/tasks/io.py | 82 +++++++++++++++++------------ 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/edisgo/run/config.py b/edisgo/run/config.py index 5c4ee8573..1dddb5690 100644 --- a/edisgo/run/config.py +++ b/edisgo/run/config.py @@ -399,6 +399,8 @@ def _adapt_ego_legacy(cfg: dict) -> dict: "grid": {"ding0_path": edisgo_cfg.get("grid_path")}, "results": {"directory": edisgo_cfg.get("results")}, "pipeline": mapped, + "overlying_grid": {"path": edisgo_cfg.get("overlying_grid_source")}, + "overlying_grid": {"selection": edisgo_cfg.get("overlying_grid")} } if "database" in cfg: adapted["database"] = cfg["database"] diff --git a/edisgo/run/presets/uc4_example.yaml b/edisgo/run/presets/uc4_example.yaml index d3f9efd90..b3cc3cfdc 100644 --- a/edisgo/run/presets/uc4_example.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -31,7 +31,9 @@ database: timeindex: {start: "2035-01-01", periods: 24, freq: h} overlying_grid: - path: "/path/to/overlying_grid_csv_dir" + enabled: false # master switch — set true to activate import_overlying_grid_data + source: csv # "csv" (load from path) or "etrago" (consume overlying_grid_data kwarg) + path: "/path/to/overlying_grid_csv_dir" # required when source == csv; full leaf dir for ONE grid (like ding0_path) results: directory: results/uc4_example @@ -56,7 +58,7 @@ pipeline: - optimize: flexible: [heat_pumps, storage, charging_points, dsm] method: soc - opf_version: 3 + opf_version: 2 - reinforce - save: archive: true diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 258148eb8..0bf51bf3a 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -196,29 +196,35 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): """ Import overlying grid data into the EDisGo instance. - When ``overlying_grid_data`` is a dict of DataFrames (as returned by - ``get_etrago_results_per_bus``), the overlying-grid attributes and - dispatchable/fluctuating generator time series are set from it. + Behavior controlled by ``ctx.raw_config['overlying_grid']``: - When ``overlying_grid_path`` is a directory path, the overlying-grid - attributes are loaded from CSV files in that directory, and - ``dispatchable_generators_active_power.csv`` / - ``renewables_potential.csv`` are applied as generator time series - if present. + * ``enabled`` (bool) — master switch. Falsy → task no-ops. + * ``source`` (str) — ``"etrago"`` or ``"csv"``. - Falls back to ``ctx.raw_config['eDisGo']['overlying_grid_source']`` - as the directory path when neither argument is given. + ``source: etrago`` consumes ``ctx.overlying_grid_data`` (a dict of + DataFrames as returned by ``get_etrago_results_per_bus``), injected + via the ``overlying_grid_data=`` kwarg of + :func:`edisgo.run.run_edisgo`. Sets overlying-grid attributes and + dispatchable/fluctuating generator time series from it. + + ``source: csv`` loads overlying-grid attributes from CSVs in + ``overlying_grid.path`` (full directory path for ONE grid — same + leaf-dir convention as ``grid.ding0_path``; callers handling many + grids must compose the per-grid subdirectory themselves). + ``dispatchable_generators_active_power.csv`` and + ``renewables_potential.csv``, if present in that dir, are applied + as generator time series. Parameters ---------- edisgo : edisgo.EDisGo EDisGo instance to modify in place. ctx : RunContext - Run context. + Run context. Reads ``raw_config['overlying_grid']`` and + ``overlying_grid_data`` attribute. overlying_grid_path : str, optional - Directory containing overlying-grid CSV files. - overlying_grid_data : dict, optional - Dict of DataFrames as returned by ``get_etrago_results_per_bus``. + CSV directory override (takes precedence over + ``overlying_grid.path`` from the config) when ``source='csv'``. Returns ------- @@ -228,33 +234,38 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): """ import pandas as pd - overlying_grid_data = getattr(ctx, "overlying_grid_data", None) + og_cfg = ctx.raw_config.get("overlying_grid") or {} + if not og_cfg.get("enabled"): + return edisgo - if overlying_grid: - + source = og_cfg.get("source") + overlying_grid_data = getattr(ctx, "overlying_grid_data", None) - if overlying_grid_data is not None: - # eTraGo results dict — set standard overlying-grid attributes - for attr in edisgo.overlying_grid._attributes: - if attr in overlying_grid_data: - setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) - # set generator time series - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=overlying_grid_data.get( - "dispatchable_generators_active_power" - ), - fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), + if source == "etrago": + if overlying_grid_data is None: + ctx.logger.warning( + "task 'import_overlying_grid_data': source='etrago' but no " + "overlying_grid_data passed to run_edisgo — skipping." ) return edisgo + for attr in edisgo.overlying_grid._attributes: + if attr in overlying_grid_data: + setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=overlying_grid_data.get( + "dispatchable_generators_active_power" + ), + fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), + ) + return edisgo - # resolve path: explicit arg → runner config overlying_grid.path → skip + if source == "csv": if overlying_grid_path is None: - overlying_grid_path = (ctx.raw_config.get("overlying_grid_path") or {}).get("path") - + overlying_grid_path = og_cfg.get("path") if overlying_grid_path is None: ctx.logger.warning( - "task 'import_overlying_grid_data': no overlying_grid_data or " - "overlying_grid_path provided — skipping." + "task 'import_overlying_grid_data': source='csv' but no " + "overlying_grid.path configured — skipping." ) return edisgo @@ -320,5 +331,10 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): dispatchable_generators_ts=disp_ts, fluctuating_generators_ts=pot_ts, ) + return edisgo + ctx.logger.warning( + f"task 'import_overlying_grid_data': unknown source={source!r} " + "(expected 'etrago' or 'csv') — skipping." + ) return edisgo From 04aad53141c9244b081341e731607abab87a76d8 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Wed, 27 May 2026 11:29:08 +0200 Subject: [PATCH 23/66] Change opf_version --- edisgo/run/presets/uc4_example.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edisgo/run/presets/uc4_example.yaml b/edisgo/run/presets/uc4_example.yaml index b3cc3cfdc..5ec024f6c 100644 --- a/edisgo/run/presets/uc4_example.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -58,7 +58,7 @@ pipeline: - optimize: flexible: [heat_pumps, storage, charging_points, dsm] method: soc - opf_version: 2 + opf_version: 3 - reinforce - save: archive: true From f1be570aa2e54fdd31b30533b7e1638031a97bf3 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 27 May 2026 11:38:12 +0200 Subject: [PATCH 24/66] Add renewables_potential and dispatchable_generator timeseries to OverlyingGrid object --- edisgo/network/overlying_grid.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/edisgo/network/overlying_grid.py b/edisgo/network/overlying_grid.py index 241768a20..05edc2168 100644 --- a/edisgo/network/overlying_grid.py +++ b/edisgo/network/overlying_grid.py @@ -84,6 +84,17 @@ def __init__(self, **kwargs): "feedin_district_heating", pd.DataFrame(dtype="float64") ) + self.dispatchable_generators_active_power = kwargs.get( + "dispatchable_generators_active_power", pd.DataFrame(dtype="float64") + ) + + self.dispatchable_generators_reactive_power = kwargs.get( + "dispatchable_generators_reactive_power", pd.DataFrame(dtype="float64") + ) + + self.renewables_potential = kwargs.get( + "renewables_potential", pd.Series(dtype="float64") + ) @property def _attributes(self): return [ @@ -97,6 +108,9 @@ def _attributes(self): "heat_pump_central_active_power", "thermal_storage_units_central_soc", "feedin_district_heating", + "dispatchable_generators_active_power", + "dispatchable_generators_reactive_power", + "renewables_potential", ] def reduce_memory(self, attr_to_reduce=None, to_type="float32"): From e79dcebeab748417c2154ae93a7340c7e4ea28cf Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 27 May 2026 11:40:52 +0200 Subject: [PATCH 25/66] Import overlying_grid_data only from csv files when it is not directly handed over --- edisgo/run/tasks/io.py | 107 ++++++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 44 deletions(-) diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index c2b748cf0..b03027795 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -235,14 +235,30 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): for attr in edisgo.overlying_grid._attributes: if attr in overlying_grid_data: setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + + if not overlying_grid_data.get( + "dispatchable_generators_active_power" + ).empty: # set generator time series - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=overlying_grid_data.get( - "dispatchable_generators_active_power" - ), - fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), - ) - return edisgo + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=overlying_grid_data.get( + "dispatchable_generators_active_power" + ), + ) + if not overlying_grid_data.get("renewables_potential").empty: + pot_ts = overlying_grid_data.get("renewables_potential") + edisgo_ti = edisgo.timeseries.timeindex + csv_year = pot_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + pot_ts.index = pot_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + pot_ts = pot_ts.reindex(edisgo_ti) + + edisgo.set_time_series_active_power_predefined( + fluctuating_generators_ts=pot_ts, + ) # resolve path: explicit arg → runner config overlying_grid.path → skip if overlying_grid_path is None: @@ -255,8 +271,9 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): ) return edisgo - # load overlying-grid attributes from CSV directory - edisgo.overlying_grid.from_csv(overlying_grid_path) + if overlying_grid_data is None: + # load overlying-grid attributes from CSV directory + edisgo.overlying_grid.from_csv(overlying_grid_path) # reindex overlying-grid attributes to match edisgo timeindex # CSVs may use a different year — shift year then reindex @@ -281,41 +298,43 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) - # load dispatchable generator and renewables time series from the same dir - disp_path = os.path.join( - overlying_grid_path, "dispatchable_generators_active_power.csv" - ) - if os.path.isfile(disp_path): - disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) - if not edisgo_ti.empty: - csv_year = disp_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - disp_ts.index = disp_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - disp_ts = disp_ts.reindex(edisgo_ti) - else: - disp_ts = None - - pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") - if os.path.isfile(pot_path): - pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) - if not edisgo_ti.empty: - csv_year = pot_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - pot_ts.index = pot_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - pot_ts = pot_ts.reindex(edisgo_ti) - else: - pot_ts = None - - if disp_ts is not None or pot_ts is not None: - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=disp_ts, - fluctuating_generators_ts=pot_ts, + if overlying_grid_data is None: + # load dispatchable generator and renewables time series from the same dir + disp_path = os.path.join( + overlying_grid_path, "dispatchable_generators_active_power.csv" ) + if os.path.isfile(disp_path): + disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = disp_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + disp_ts.index = disp_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + disp_ts = disp_ts.reindex(edisgo_ti) + else: + disp_ts = None + + pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") + if os.path.isfile(pot_path): + pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = pot_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + pot_ts.index = pot_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + pot_ts = pot_ts.reindex(edisgo_ti) + else: + pot_ts = None + + + if disp_ts is not None or pot_ts is not None: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + fluctuating_generators_ts=pot_ts, + ) return edisgo From f31cf31180fd15746affb4f50aba6e9645a22be9 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 27 May 2026 11:42:05 +0200 Subject: [PATCH 26/66] Select timesteps in overlying_grid_data that are relevant for eDisGo's optimization --- edisgo/io/powermodels_io.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 6bf21affb..30d753439 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1015,8 +1015,17 @@ def _build_battery_storage( """ branches = pd.concat([psa_net.lines, psa_net.transformers]) if not edisgo_obj.overlying_grid.storage_units_soc.empty: + # Select relevant timesteps + timesteps = edisgo_obj.timeseries.timeindex.union( + [ + edisgo_obj.timeseries.timeindex[-1] + + edisgo_obj.timeseries.timeindex.freq + ] + ) + if edisgo_obj.overlying_grid.storage_units_soc.index[0].year==2011: + timesteps = timesteps.map(lambda t: t.replace(year=2011)) data = pd.concat( - [edisgo_obj.overlying_grid.storage_units_soc] + [edisgo_obj.overlying_grid.storage_units_soc.loc[timesteps]] * len(edisgo_obj.topology.storage_units_df), axis=1, ).values From 58d8f73e154736d5592e7595a8907259d4cfff37 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 27 May 2026 11:43:09 +0200 Subject: [PATCH 27/66] Workaround for generators_dispatch stored as pandas.DataFrame --- edisgo/io/powermodels_io.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 30d753439..4c8979d6e 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1602,11 +1602,18 @@ def _build_hv_requirements( ) for i in np.arange(len(opf_flex)): - pm["HV_requirements"][str(i + 1)] = { - "P": hv_flex_dict[opf_flex[i]].iloc[0], - "name": opf_flex[i], - "count": count, - } + if type(hv_flex_dict[opf_flex[i]]) == pd.DataFrame: + pm["HV_requirements"][str(i + 1)] = { + "P": hv_flex_dict[opf_flex[i]].sum(axis=1).iloc[0], + "name": opf_flex[i], + "count": count, + } + else: + pm["HV_requirements"][str(i + 1)] = { + "P": hv_flex_dict[opf_flex[i]].iloc[0], + "name": opf_flex[i], + "count": count, + } def _build_timeseries( @@ -1932,9 +1939,14 @@ def _build_component_timeseries( if (kind == "HV_requirements") & (pm["opf_version"] in [3, 4]): for i in np.arange(len(opf_flex)): - pm_comp[(str(i + 1))] = { - "P": hv_flex_dict[opf_flex[i]].round(20).tolist(), - } + if type(hv_flex_dict[opf_flex[i]])==pd.DataFrame: + pm_comp[(str(i + 1))] = { + "P": hv_flex_dict[opf_flex[i]].sum(axis=1).round(20).tolist(), + } + else: + pm_comp[(str(i + 1))] = { + "P": hv_flex_dict[opf_flex[i]].round(20).tolist(), + } pm["time_series"][kind] = pm_comp From b367c8120e978adf89561d96b1f40e8395bbf52f Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 27 May 2026 13:59:11 +0200 Subject: [PATCH 28/66] Workarround for pd.DataFrames in flex_opt data --- edisgo/io/powermodels_io.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 4c8979d6e..66bdd8464 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -365,13 +365,22 @@ def from_powermodels( # calculate relative error df2 = deepcopy(df) for flex in df2.columns: - abs_error = abs(df2[flex].values - hv_flex_dict[flex].values) - rel_error = [ - abs_error[i] / hv_flex_dict[flex].iloc[i] - if ((abs_error > 0.01)[i] & (hv_flex_dict[flex].iloc[i] != 0)) - else 0 - for i in range(len(abs_error)) - ] + if type(hv_flex_dict[flex]) == pd.Series: + abs_error = abs(df2[flex].values - hv_flex_dict[flex].values) + rel_error = [ + abs_error[i] / hv_flex_dict[flex].iloc[i] + if ((abs_error > 0.01)[i] & (hv_flex_dict[flex].iloc[i] != 0)) + else 0 + for i in range(len(abs_error)) + ] + else: + abs_error = abs(df2[flex].values - hv_flex_dict[flex].sum(axis=1).values) + rel_error = [ + abs_error[i] / hv_flex_dict[flex].sum(axis=1).iloc[i] + if ((abs_error > 0.01)[i] & (hv_flex_dict[flex].sum(axis=1).iloc[i] != 0)) + else 0 + for i in range(len(abs_error)) + ] df2[flex] = rel_error # write results to edisgo object edisgo_object.opf_results.overlying_grid = pd.DataFrame( From abf7f703c4cc54dc456163ac1a92c6c113f6b429 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 3 Jun 2026 15:18:05 +0200 Subject: [PATCH 29/66] Replace hard-coded year and select it from the timeindex instead --- edisgo/io/powermodels_io.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 66bdd8464..a0551ee64 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1031,8 +1031,13 @@ def _build_battery_storage( + edisgo_obj.timeseries.timeindex.freq ] ) - if edisgo_obj.overlying_grid.storage_units_soc.index[0].year==2011: - timesteps = timesteps.map(lambda t: t.replace(year=2011)) + + # If the overlying grid data uses another year in the timeindex then + # edisgo.timindex, unify them + og_year = edisgo_obj.overlying_grid.storage_units_soc.index[0].year + if og_year != edisgo_obj.timeseries.timeindex[0].year: + timesteps = timesteps.map(lambda t: t.replace(year=og_year)) + data = pd.concat( [edisgo_obj.overlying_grid.storage_units_soc.loc[timesteps]] * len(edisgo_obj.topology.storage_units_df), From 76491a3a222e9ee8564c17ba9d36bfeddfe195b7 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 3 Jun 2026 15:39:02 +0200 Subject: [PATCH 30/66] make ding0 grid path variable optional --- edisgo/run/tasks/grid.py | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py index b472f3244..b074b7e68 100644 --- a/edisgo/run/tasks/grid.py +++ b/edisgo/run/tasks/grid.py @@ -10,14 +10,25 @@ a slow "base" phase and one or more fast "scenario" phases that reuse the base-reinforced grid. """ + from __future__ import annotations +import pandas as pd + from edisgo.run.registry import register_task @register_task("setup_grid") -def task_setup_grid(edisgo, ctx, *, timeindex = None, ding0_path=None, legacy_ding0_grids=None, - import_generators=False, generator_scenario=None): +def task_setup_grid( + edisgo, + ctx, + *, + timeindex=None, + ding0_path=None, + legacy_ding0_grids=None, + import_generators=False, + generator_scenario=None, +): """ Load a ding0 grid into an EDisGo instance. @@ -99,10 +110,19 @@ def task_setup_grid(edisgo, ctx, *, timeindex = None, ding0_path=None, legacy_di @register_task("load_from_base") -def task_load_from_base(edisgo, ctx, *, path, reset_equipment_changes=True, - import_timeseries=False, import_results=False, - import_electromobility=False, import_heat_pump=False, - import_dsm=False, import_overlying_grid=False): +def task_load_from_base( + edisgo, + ctx, + *, + path=None, + reset_equipment_changes=True, + import_timeseries=False, + import_results=False, + import_electromobility=False, + import_heat_pump=False, + import_dsm=False, + import_overlying_grid=False, +): """ Reload an EDisGo instance from a previously saved directory/zip. @@ -152,6 +172,14 @@ def task_load_from_base(edisgo, ctx, *, path, reset_equipment_changes=True, from edisgo.edisgo import import_edisgo_from_files + if path is None: + grid_cfg = ctx.raw_config.get("grid", {}) or {} + path = grid_cfg.get("ding0_path") + if path is None: + raise ValueError( + "Task 'load_from_base' requires 'path' either as task " + "parameter or under config.grid.ding0_path." + ) path = str(path) from_zip = path.endswith(".zip") or not os.path.isdir(path) edisgo = import_edisgo_from_files( From 55376727bf275a69c9bcc5cb97f7da452b6c4f41 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Wed, 10 Jun 2026 09:56:18 +0200 Subject: [PATCH 31/66] Revert "Redirect to installed julia version" This reverts commit 5fce2573c2dc66fdeb81d847ec39bf8bfc65e2a4. --- edisgo/opf/powermodels_opf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edisgo/opf/powermodels_opf.py b/edisgo/opf/powermodels_opf.py index 6fbdd1e41..c26475141 100644 --- a/edisgo/opf/powermodels_opf.py +++ b/edisgo/opf/powermodels_opf.py @@ -130,7 +130,7 @@ def _convert(o): logger.info("starting julia process") julia_process = subprocess.Popen( [ - "/opt/julia-1.8.3/bin/julia", + "julia", os.path.join(opf_dir, "eDisGo_OPF.jl/Main.jl"), pm["name"], solution_dir, From 74d6bd1c025887f064e2cc46086461c7b7f6db04 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Tue, 23 Jun 2026 15:47:15 +0200 Subject: [PATCH 32/66] fix: resolve 10 review findings in the run pipeline framework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - timeseries.task_manual_ts: pass set_time_series_manual's real kwargs (generators_p/loads_p/... instead of *_active_power) — task was unusable. - config._adapt_ego_legacy: merge the duplicated 'overlying_grid' dict key (the 'path' entry was silently dropped). - tasks.io.import_overlying_grid_data: rewrite tangled control flow — guard .get(...).empty against None, drop the double from_csv, only warn on a genuinely unknown source, and infer the SOC extra step only when derivable (no IndexError on a single-timestamp timeindex). - validator: load_from no longer satisfies the time-series/flex prerequisite for analyze/reinforce/optimize, since _load_artifact reloads with import_timeseries=False and drops flex data (+ regression test). - powermodels_io: unify SOC year via DateOffset instead of replace(year=) to avoid a Feb-29 ValueError on leap-to-non-leap remapping. - context.ensure_engine: connect directly to a configured local postgres (psycopg2, pool_pre_ping) when SSH is disabled and host params are given. - config._resolve_extends: resolve a relative 'extends' against the including file before falling back to a bundled preset. --- edisgo/io/powermodels_io.py | 7 +- edisgo/run/config.py | 25 +++-- edisgo/run/context.py | 36 ++++++- edisgo/run/tasks/io.py | 166 +++++++++++++++------------------ edisgo/run/tasks/timeseries.py | 12 +-- edisgo/run/validator.py | 12 ++- tests/run/test_validator.py | 20 +++- 7 files changed, 161 insertions(+), 117 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 4438824ae..2f55104b3 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1040,8 +1040,11 @@ def _build_battery_storage( # If the overlying grid data uses another year in the timeindex then # edisgo.timindex, unify them og_year = edisgo_obj.overlying_grid.storage_units_soc.index[0].year - if og_year != edisgo_obj.timeseries.timeindex[0].year: - timesteps = timesteps.map(lambda t: t.replace(year=og_year)) + year_diff = og_year - edisgo_obj.timeseries.timeindex[0].year + if year_diff != 0: + # Shift by whole years instead of Timestamp.replace(year=...), + # which raises on Feb 29 when the target year is not a leap year. + timesteps = timesteps + pd.DateOffset(years=year_diff) data = pd.concat( [edisgo_obj.overlying_grid.storage_units_soc.loc[timesteps]] diff --git a/edisgo/run/config.py b/edisgo/run/config.py index 1dddb5690..6f095133f 100644 --- a/edisgo/run/config.py +++ b/edisgo/run/config.py @@ -128,9 +128,10 @@ def _resolve_extends(cfg: dict, base_dir: Path) -> dict: Resolve an ``extends:`` reference and deep-merge parent into child. The parent is loaded recursively, so a chain of ``extends:`` works. - References are looked up as (1) a bundled preset name under - :mod:`edisgo.run.presets`, (2) a path relative to ``base_dir``. - The child's keys override the parent's on conflicts. + A relative reference is looked up as (1) a path relative to + ``base_dir``, (2) a bundled preset name under + :mod:`edisgo.run.presets`. The child's keys override the parent's on + conflicts. Parameters ---------- @@ -156,11 +157,15 @@ def _resolve_extends(cfg: dict, base_dir: Path) -> dict: return cfg ext_path = Path(ext).expanduser() if not ext_path.is_absolute(): - preset_path = _preset_path(str(ext_path)) - if preset_path is not None: - ext_path = preset_path + # Resolve relative to the including file first (least surprise: a + # local file next to the config wins), then fall back to a bundled + # preset of that name. + local_path = (base_dir / ext_path).resolve() + if local_path.is_file(): + ext_path = local_path else: - ext_path = (base_dir / ext_path).resolve() + preset_path = _preset_path(str(ext_path)) + ext_path = preset_path if preset_path is not None else local_path if not ext_path.is_file(): raise FileNotFoundError(f"extends: file not found: {ext_path}") parent = _read_file(ext_path) @@ -399,8 +404,10 @@ def _adapt_ego_legacy(cfg: dict) -> dict: "grid": {"ding0_path": edisgo_cfg.get("grid_path")}, "results": {"directory": edisgo_cfg.get("results")}, "pipeline": mapped, - "overlying_grid": {"path": edisgo_cfg.get("overlying_grid_source")}, - "overlying_grid": {"selection": edisgo_cfg.get("overlying_grid")} + "overlying_grid": { + "path": edisgo_cfg.get("overlying_grid_source"), + "selection": edisgo_cfg.get("overlying_grid"), + }, } if "database" in cfg: adapted["database"] = cfg["database"] diff --git a/edisgo/run/context.py b/edisgo/run/context.py index c2fbce234..f07effadf 100644 --- a/edisgo/run/context.py +++ b/edisgo/run/context.py @@ -105,11 +105,43 @@ def ensure_engine(self): "Task needs a database engine but no 'database' section " "is configured." ) + ssh_cfg = db_cfg.get("ssh") or {} + ssh_enabled = bool(ssh_cfg.get("enabled", False)) + + # Direct local database: when SSH is disabled and explicit + # connection parameters are given (host/port/user/password as + # passed by eGo), connect straight to that postgres via + # psycopg2. This avoids edisgo.io.db.engine(ssh=False), which + # is hard-wired to the remote OpenEnergyPlatform (oedialect) + # and can stall for hours on large queries. + host = db_cfg.get("host") + if not ssh_enabled and host: + from sqlalchemy import create_engine + + user = db_cfg.get("user") + password = db_cfg.get("password") + port = db_cfg.get("port") + name = db_cfg.get("database_name") or db_cfg.get("database") + self.logger.info( + f"ensure_engine: using local database " + f"{user}@{host}:{port}/{name} (no OEP, no SSH tunnel)." + ) + self.engine = create_engine( + f"postgresql+psycopg2://{user}:{password}@{host}:{port}/{name}", + connect_args={"connect_timeout": 10}, + # The engine is cached and reused across long-running tasks + # (e.g. electromobility can idle the connection for many + # minutes). pool_pre_ping detects connections the server/SSH + # tunnel dropped while idle and transparently reconnects, + # avoiding "server closed the connection unexpectedly". + pool_pre_ping=True, + ) + return self.engine + from edisgo.io.db import engine as egon_engine - ssh_cfg = db_cfg.get("ssh") or {} self.engine = egon_engine( path=db_cfg.get("credentials_path"), - ssh=bool(ssh_cfg.get("enabled", False)), + ssh=ssh_enabled, ) return self.engine diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 470526a6b..77ad9f4c2 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -233,14 +233,53 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): """ import pandas as pd - + og_cfg = ctx.raw_config.get("overlying_grid") or {} if not og_cfg.get("enabled"): return edisgo source = og_cfg.get("source") overlying_grid_data = getattr(ctx, "overlying_grid_data", None) + edisgo_ti = edisgo.timeseries.timeindex + soc_attrs = { + "storage_units_soc", + "thermal_storage_units_decentral_soc", + "thermal_storage_units_central_soc", + } + + def _to_edisgo_timeindex(ts, extra_step=False): + """ + Shift ``ts``'s index year to match the edisgo timeindex and reindex + onto it. ``extra_step`` appends one trailing step (for SOC series, + which carry an end-of-period state). Returns ``ts`` unchanged for + empty inputs or an empty edisgo timeindex. + """ + if ts is None or ts.empty or edisgo_ti.empty: + return ts + year_diff = edisgo_ti[0].year - ts.index[0].year + if year_diff != 0: + ts = ts.copy() + ts.index = ts.index + pd.DateOffset(years=year_diff) + target = edisgo_ti + if extra_step: + # Derive the step only when it can be inferred; a single-timestamp + # timeindex with no freq cannot, so fall back to no extra step. + freq = edisgo_ti.freq or ( + edisgo_ti[1] - edisgo_ti[0] if len(edisgo_ti) > 1 else None + ) + if freq is not None: + target = edisgo_ti.union([edisgo_ti[-1] + freq]) + return ts.reindex(target) + + if source not in ("etrago", "csv"): + ctx.logger.warning( + f"task 'import_overlying_grid_data': unknown source={source!r} " + "(expected 'etrago' or 'csv') — skipping." + ) + return edisgo + + # --- 1) load the overlying-grid attributes for the chosen source --- if source == "etrago": if overlying_grid_data is None: ctx.logger.warning( @@ -251,111 +290,54 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): for attr in edisgo.overlying_grid._attributes: if attr in overlying_grid_data: setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) - - if not overlying_grid_data.get( - "dispatchable_generators_active_power" - ).empty: - # set generator time series - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=overlying_grid_data.get( - "dispatchable_generators_active_power" - ), - ) - if not overlying_grid_data.get("renewables_potential").empty: - pot_ts = overlying_grid_data.get("renewables_potential") - edisgo_ti = edisgo.timeseries.timeindex - csv_year = pot_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - pot_ts.index = pot_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - pot_ts = pot_ts.reindex(edisgo_ti) - - edisgo.set_time_series_active_power_predefined( - fluctuating_generators_ts=pot_ts, - ) - - if source == "csv": - if overlying_grid_path is None: - overlying_grid_path = og_cfg.get("path") + else: # source == "csv" + overlying_grid_path = overlying_grid_path or og_cfg.get("path") if overlying_grid_path is None: ctx.logger.warning( "task 'import_overlying_grid_data': source='csv' but no " "overlying_grid.path configured — skipping." ) return edisgo - - # load overlying-grid attributes from CSV directory - edisgo.overlying_grid.from_csv(overlying_grid_path) - - if overlying_grid_data is None: - # load overlying-grid attributes from CSV directory edisgo.overlying_grid.from_csv(overlying_grid_path) - # reindex overlying-grid attributes to match edisgo timeindex - # CSVs may use a different year — shift year then reindex - edisgo_ti = edisgo.timeseries.timeindex - if not edisgo_ti.empty: - # SOC needs one extra step at the end (end-of-period state) - ti_freq = edisgo_ti.freq or (edisgo_ti[1] - edisgo_ti[0]) - edisgo_ti_plus1 = edisgo_ti.union([edisgo_ti[-1] + ti_freq]) - soc_attrs = { - "storage_units_soc", - "thermal_storage_units_decentral_soc", - "thermal_storage_units_central_soc", - } - for attr in edisgo.overlying_grid._attributes: - ts = getattr(edisgo.overlying_grid, attr) - if ts.empty: - continue - csv_year = ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) - target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti - setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) - - if overlying_grid_data is None: - # load dispatchable generator and renewables time series from the same dir - disp_path = os.path.join( - overlying_grid_path, "dispatchable_generators_active_power.csv" + # --- 2) reindex the overlying-grid attributes onto the edisgo timeindex + # (data may use a different year; SOC series carry one extra end step) --- + for attr in edisgo.overlying_grid._attributes: + ts = getattr(edisgo.overlying_grid, attr) + if ts is None or ts.empty: + continue + setattr( + edisgo.overlying_grid, + attr, + _to_edisgo_timeindex(ts, extra_step=attr in soc_attrs), ) - if os.path.isfile(disp_path): - disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) - if not edisgo_ti.empty: - csv_year = disp_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - disp_ts.index = disp_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - disp_ts = disp_ts.reindex(edisgo_ti) - else: - disp_ts = None - - pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") - if os.path.isfile(pot_path): - pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) - if not edisgo_ti.empty: - csv_year = pot_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - pot_ts.index = pot_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - pot_ts = pot_ts.reindex(edisgo_ti) - else: - pot_ts = None + # --- 3) set dispatchable/fluctuating generator time series --- + if source == "etrago": + disp_ts = overlying_grid_data.get("dispatchable_generators_active_power") + pot_ts = overlying_grid_data.get("renewables_potential") + if disp_ts is not None and not disp_ts.empty: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + ) + if pot_ts is not None and not pot_ts.empty: + edisgo.set_time_series_active_power_predefined( + fluctuating_generators_ts=_to_edisgo_timeindex(pot_ts), + ) + else: # source == "csv": load the two generator-TS CSVs from the dir + def _load_generator_ts(filename): + path = os.path.join(overlying_grid_path, filename) + if not os.path.isfile(path): + return None + ts = pd.read_csv(path, index_col=0, parse_dates=True) + return _to_edisgo_timeindex(ts) + + disp_ts = _load_generator_ts("dispatchable_generators_active_power.csv") + pot_ts = _load_generator_ts("renewables_potential.csv") if disp_ts is not None or pot_ts is not None: edisgo.set_time_series_active_power_predefined( dispatchable_generators_ts=disp_ts, fluctuating_generators_ts=pot_ts, ) - ctx.logger.warning( - f"task 'import_overlying_grid_data': unknown source={source!r} " - "(expected 'etrago' or 'csv') — skipping." - ) return edisgo diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py index cf918c235..1264a9b95 100644 --- a/edisgo/run/tasks/timeseries.py +++ b/edisgo/run/tasks/timeseries.py @@ -266,12 +266,12 @@ def _as_df(obj): return pd.DataFrame(obj) if obj is not None else None edisgo.set_time_series_manual( - generators_active_power=_as_df(generators_active_power), - generators_reactive_power=_as_df(generators_reactive_power), - loads_active_power=_as_df(loads_active_power), - loads_reactive_power=_as_df(loads_reactive_power), - storage_units_active_power=_as_df(storage_units_active_power), - storage_units_reactive_power=_as_df(storage_units_reactive_power), + generators_p=_as_df(generators_active_power), + generators_q=_as_df(generators_reactive_power), + loads_p=_as_df(loads_active_power), + loads_q=_as_df(loads_reactive_power), + storage_units_p=_as_df(storage_units_active_power), + storage_units_q=_as_df(storage_units_reactive_power), ) ctx.flags["timeseries_set"] = True return edisgo diff --git a/edisgo/run/validator.py b/edisgo/run/validator.py index 3989b8398..f8f998828 100644 --- a/edisgo/run/validator.py +++ b/edisgo/run/validator.py @@ -117,21 +117,23 @@ def validate(cfg: dict) -> None: f"a loaded grid (setup_grid or " f"load_from_base) before it." ) - if task_name in {"analyze", "reinforce"} and not ( - ts_set or load_from - ): + # load_from does NOT satisfy these: _load_artifact reloads the + # grid with import_timeseries=False and drops flex data, so a + # time-series (and, for optimize, a flex-import) task must run + # in the stage itself even after a load_from. + if task_name in {"analyze", "reinforce"} and not ts_set: raise ValueError( f"Stage '{name}': task '{task_name}' requires time " f"series to be set (e.g. worst_case_ts or " f"oedb_ts) before it." ) if task_name == "optimize": - if not ts_set and not load_from: + if not ts_set: raise ValueError( f"Stage '{name}': 'optimize' requires time " f"series." ) - if not flex_imported and not load_from: + if not flex_imported: raise ValueError( f"Stage '{name}': 'optimize' requires at least " f"one flex asset to be imported." diff --git a/tests/run/test_validator.py b/tests/run/test_validator.py index 4b86f40cf..1d375f10d 100644 --- a/tests/run/test_validator.py +++ b/tests/run/test_validator.py @@ -92,6 +92,24 @@ def test_stage_load_from_with_save_ok(): cfg = {"stages": [ {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", "reinforce", "save"]}, - {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, + # load_from reloads the grid with import_timeseries=False, so the + # consuming stage must set time series itself before reinforce. + {"name": "b", "load_from": "a", + "pipeline": ["worst_case_ts", "reinforce", "save"]}, ]} validate(cfg) + + +def test_stage_load_from_without_ts_rejected(): + """ + load_from does NOT satisfy the time-series prerequisite: the artifact is + reloaded with import_timeseries=False, so reinforce after a bare load_from + (no time-series task in the stage) must be rejected. + """ + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce", "save"]}, + {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, + ]} + with pytest.raises(ValueError, match="requires time series"): + validate(cfg) From 031e7431f88d4286ed61f74118c23c3fa217cfd2 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 1 Jul 2026 13:08:18 +0200 Subject: [PATCH 33/66] refactor: declarative task metadata for the validator + shared artifact loader - registry: register_task now records requires/provides/ts_altering metadata (TaskMeta) exposed via get_task_meta. Tasks declare their pre-/post- conditions (setup_grid provides 'grid', TS tasks provide 'timeseries', flex imports require 'grid' and provide 'flex', analyze/reinforce require 'timeseries', optimize requires 'timeseries'+'flex', base_reinforce requires 'grid'). - validator: check those metadata generically instead of maintaining parallel hard-coded task-name sets, so it stays in sync with the tasks. load_from provides only 'grid' (not timeseries/flex), matching _load_artifact. Compute the known-task set once instead of per step. - runner._load_artifact and tasks.grid.task_load_from_base now share one load_saved_edisgo() helper instead of duplicating the import_edisgo_from_files policy. --- edisgo/run/registry.py | 71 ++++++++++++++++++++++- edisgo/run/runner.py | 24 ++------ edisgo/run/tasks/analysis.py | 8 +-- edisgo/run/tasks/flex.py | 8 +-- edisgo/run/tasks/grid.py | 85 +++++++++++++++++++++------ edisgo/run/tasks/timeseries.py | 8 +-- edisgo/run/validator.py | 103 +++++++++++++++------------------ 7 files changed, 199 insertions(+), 108 deletions(-) diff --git a/edisgo/run/registry.py b/edisgo/run/registry.py index 8aed4f3f5..c7ffca338 100644 --- a/edisgo/run/registry.py +++ b/edisgo/run/registry.py @@ -18,12 +18,44 @@ """ from __future__ import annotations -from typing import Callable +from typing import Callable, NamedTuple _TASKS: dict[str, Callable] = {} -def register_task(name: str) -> Callable[[Callable], Callable]: +class TaskMeta(NamedTuple): + """ + Declarative metadata describing a task's pipeline pre-/post-conditions. + + Attributes + ---------- + requires : frozenset of str + Capabilities that must already be satisfied in the stage before + this task runs (e.g. ``{"grid"}``, ``{"timeseries"}``, ``{"flex"}``). + provides : frozenset of str + Capabilities this task establishes for later tasks in the stage. + ts_altering : bool + Whether the task sets/alters the active-power time series. Such + tasks must not appear after ``reactive_power``. The validator uses + this metadata so it stays in sync with the actual tasks instead of + maintaining a parallel hard-coded list. + """ + + requires: frozenset = frozenset() + provides: frozenset = frozenset() + ts_altering: bool = False + + +_META: dict[str, TaskMeta] = {} + + +def register_task( + name: str, + *, + requires=frozenset(), + provides=frozenset(), + ts_altering: bool = False, +) -> Callable[[Callable], Callable]: """ Decorator to register a task function under the given name. @@ -37,6 +69,14 @@ def register_task(name: str) -> Callable[[Callable], Callable]: ---------- name : str Unique task name used in pipeline definitions. + requires : iterable of str, optional + Capabilities the task needs (see :class:`TaskMeta`). Used by the + validator for static ordering checks. + provides : iterable of str, optional + Capabilities the task establishes for later tasks. + ts_altering : bool, optional + Whether the task alters the active-power time series (must precede + ``reactive_power``). Returns ------- @@ -50,7 +90,8 @@ def register_task(name: str) -> Callable[[Callable], Callable]: Examples -------- - >>> @register_task("set_timeindex_weekly") + >>> @register_task("set_timeindex_weekly", provides={"timeseries"}, + ... ts_altering=True) ... def task_weekly(edisgo, ctx, *, start): ... import pandas as pd ... edisgo.set_timeindex(pd.date_range(start, periods=168, freq="h")) @@ -64,11 +105,35 @@ def deco(fn: Callable) -> Callable: f"new={fn.__qualname__})." ) _TASKS[name] = fn + _META[name] = TaskMeta( + requires=frozenset(requires), + provides=frozenset(provides), + ts_altering=ts_altering, + ) return fn return deco +def get_task_meta(name: str) -> TaskMeta: + """ + Return the :class:`TaskMeta` for a registered task. + + Parameters + ---------- + name : str + Task name. + + Returns + ------- + TaskMeta + The task's declared metadata. Unregistered names yield an empty + :class:`TaskMeta` (no requirements, no provided capabilities). + + """ + return _META.get(name, TaskMeta()) + + def get_task(name: str) -> Callable: """ Look up a registered task function by name. diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py index 2d08fd3bc..d3464010e 100644 --- a/edisgo/run/runner.py +++ b/edisgo/run/runner.py @@ -176,25 +176,11 @@ def _load_artifact(path: str): The restored EDisGo instance. """ - import pandas as pd - - from edisgo.edisgo import import_edisgo_from_files - - from_zip = path.endswith(".zip") - edisgo = import_edisgo_from_files( - edisgo_path=path, - import_topology=True, - import_timeseries=False, - import_results=True, - import_electromobility=False, - import_heat_pump=False, - import_dsm=False, - import_overlying_grid=False, - from_zip_archive=from_zip, - ) - edisgo.legacy_grids = False - edisgo.results.equipment_changes = pd.DataFrame() - return edisgo + from edisgo.run.tasks.grid import load_saved_edisgo + + # Topology + results only; time series and flex data are dropped so the + # consuming stage sets them fresh, and equipment_changes is reset. + return load_saved_edisgo(path, import_results=True) def _resolve_templating(step_params: dict, stage_params: dict) -> dict: diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py index becff93b9..a0d0f4ebd 100644 --- a/edisgo/run/tasks/analysis.py +++ b/edisgo/run/tasks/analysis.py @@ -55,7 +55,7 @@ def task_check_integrity(edisgo, ctx): return edisgo -@register_task("analyze") +@register_task("analyze", requires={"timeseries"}) def task_analyze( edisgo, ctx, @@ -110,7 +110,7 @@ def task_analyze( return edisgo -@register_task("reinforce") +@register_task("reinforce", requires={"timeseries"}) def task_reinforce( edisgo, ctx, @@ -182,7 +182,7 @@ def task_reinforce( return edisgo -@register_task("base_reinforce") +@register_task("base_reinforce", requires={"grid"}) def task_base_reinforce( edisgo, ctx, *, cases=None, reset_equipment_changes=True, save_artifact=True ): @@ -258,7 +258,7 @@ def task_base_reinforce( return edisgo -@register_task("optimize") +@register_task("optimize", requires={"timeseries", "flex"}) def task_optimize( edisgo, ctx, diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py index fd914a2a0..87ba4a35e 100644 --- a/edisgo/run/tasks/flex.py +++ b/edisgo/run/tasks/flex.py @@ -13,7 +13,7 @@ from edisgo.run.registry import register_task -@register_task("import_heat_pumps") +@register_task("import_heat_pumps", requires={"grid"}, provides={"flex"}) def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): """ Import heat pumps from egon_data into the topology. @@ -52,7 +52,7 @@ def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): return edisgo -@register_task("import_home_batteries") +@register_task("import_home_batteries", requires={"grid"}, provides={"flex"}) def task_import_home_batteries(edisgo, ctx): """ Import home batteries from egon_data into the topology. @@ -81,7 +81,7 @@ def task_import_home_batteries(edisgo, ctx): return edisgo -@register_task("import_dsm") +@register_task("import_dsm", requires={"grid"}, provides={"flex"}) def task_import_dsm(edisgo, ctx, *, timeindex=None): """ Import demand-side-management potential from egon_data. @@ -113,7 +113,7 @@ def task_import_dsm(edisgo, ctx, *, timeindex=None): return edisgo -@register_task("import_electromobility") +@register_task("import_electromobility", requires={"grid"}, provides={"flex"}) def task_import_electromobility(edisgo, ctx, *, data_source="oedb", charging_strategy="dumb", flexibility_bands_ucs = None, diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py index b074b7e68..348438bf5 100644 --- a/edisgo/run/tasks/grid.py +++ b/edisgo/run/tasks/grid.py @@ -18,7 +18,7 @@ from edisgo.run.registry import register_task -@register_task("setup_grid") +@register_task("setup_grid", provides={"grid"}) def task_setup_grid( edisgo, ctx, @@ -109,7 +109,70 @@ def task_setup_grid( return edisgo -@register_task("load_from_base") +def load_saved_edisgo( + path, + *, + reset_equipment_changes=True, + import_timeseries=False, + import_results=False, + import_electromobility=False, + import_heat_pump=False, + import_dsm=False, + import_overlying_grid=False, +): + """ + Reload a previously saved EDisGo object from a directory or ``.zip``. + + Shared by the ``load_from_base`` task and the runner's stage-level + ``load_from`` handling so both load artifacts with the same policy. + Topology is always imported; time series and flex data default to off + (the consuming stage sets them fresh). ``legacy_grids`` is cleared and, + by default, ``results.equipment_changes`` is reset so a subsequent + reinforce reflects only the current scenario. + + Parameters + ---------- + path : str or pathlib.Path + Directory or ``.zip`` produced by the ``save`` task. + reset_equipment_changes : bool, optional + If ``True`` (default), clear ``results.equipment_changes``. + import_timeseries, import_results, import_electromobility, \ + import_heat_pump, import_dsm, import_overlying_grid : bool, optional + Which saved sub-datasets to import (all off by default except as + overridden by the caller). + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import os + + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + path = str(path) + from_zip = path.endswith(".zip") or not os.path.isdir(path) + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=import_timeseries, + import_results=import_results, + import_electromobility=import_electromobility, + import_heat_pump=import_heat_pump, + import_dsm=import_dsm, + import_overlying_grid=import_overlying_grid, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + return edisgo + + +@register_task("load_from_base", provides={"grid"}) def task_load_from_base( edisgo, ctx, @@ -166,12 +229,6 @@ def task_load_from_base( The restored EDisGo instance. """ - import os - - import pandas as pd - - from edisgo.edisgo import import_edisgo_from_files - if path is None: grid_cfg = ctx.raw_config.get("grid", {}) or {} path = grid_cfg.get("ding0_path") @@ -180,21 +237,15 @@ def task_load_from_base( "Task 'load_from_base' requires 'path' either as task " "parameter or under config.grid.ding0_path." ) - path = str(path) - from_zip = path.endswith(".zip") or not os.path.isdir(path) - edisgo = import_edisgo_from_files( - edisgo_path=path, - import_topology=True, + edisgo = load_saved_edisgo( + path, + reset_equipment_changes=reset_equipment_changes, import_timeseries=import_timeseries, import_results=import_results, import_electromobility=import_electromobility, import_heat_pump=import_heat_pump, import_dsm=import_dsm, import_overlying_grid=import_overlying_grid, - from_zip_archive=from_zip, ) - edisgo.legacy_grids = False - if reset_equipment_changes: - edisgo.results.equipment_changes = pd.DataFrame() ctx.flags["grid_loaded"] = True return edisgo diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py index 1264a9b95..d4a6c8a30 100644 --- a/edisgo/run/tasks/timeseries.py +++ b/edisgo/run/tasks/timeseries.py @@ -20,7 +20,7 @@ from edisgo.run.registry import register_task -@register_task("worst_case_ts") +@register_task("worst_case_ts", provides={"timeseries"}, ts_altering=True) def task_worst_case_ts( edisgo, ctx, @@ -68,7 +68,7 @@ def task_worst_case_ts( return edisgo -@register_task("set_timeindex") +@register_task("set_timeindex", provides={"timeseries"}, ts_altering=True) def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, freq="h"): """ Set the time index on the EDisGo object. @@ -119,7 +119,7 @@ def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, freq="h"): return edisgo -@register_task("oedb_ts") +@register_task("oedb_ts", provides={"timeseries"}, ts_altering=True) def task_oedb_ts( edisgo, ctx, @@ -216,7 +216,7 @@ def task_oedb_ts( return edisgo -@register_task("manual_ts") +@register_task("manual_ts", provides={"timeseries"}, ts_altering=True) def task_manual_ts( edisgo, ctx, diff --git a/edisgo/run/validator.py b/edisgo/run/validator.py index f8f998828..5ca548499 100644 --- a/edisgo/run/validator.py +++ b/edisgo/run/validator.py @@ -27,16 +27,21 @@ from typing import Any -from edisgo.run.registry import known_tasks - -_TS_TASKS = {"worst_case_ts", "oedb_ts", "manual_ts", "set_timeindex"} -_GRID_CREATING_TASKS = {"setup_grid", "load_from_base"} -_FLEX_IMPORTS = { - "import_heat_pumps", - "import_home_batteries", - "import_dsm", - "import_electromobility", +from edisgo.run.registry import get_task_meta, known_tasks + +# Human-readable message per required capability. The wording keeps the +# substrings the validator tests assert on ("loaded grid", "time series", +# "flex asset"). +_REQUIREMENT_MESSAGES = { + "grid": "requires a loaded grid (setup_grid or load_from_base) before it", + "timeseries": ( + "requires time series to be set (e.g. worst_case_ts or oedb_ts) " + "before it" + ), + "flex": "requires at least one flex asset to be imported", } +# Order in which a missing capability is reported when several are missing. +_REQUIREMENT_PRIORITY = ("grid", "timeseries", "flex") def validate(cfg: dict) -> None: @@ -68,6 +73,7 @@ def validate(cfg: dict) -> None: if not stages: raise ValueError("Config has no stages to run.") + known = set(known_tasks()) available_artifacts: set[str] = set() for stage in stages: @@ -82,67 +88,50 @@ def validate(cfg: dict) -> None: f"{sorted(available_artifacts)}" ) - grid_available = load_from is not None - ts_set = False + # Capabilities established so far in this stage. A stage-level + # load_from reloads the grid topology only — _load_artifact drops + # time series and flex data (import_timeseries=False) — so it + # provides "grid" but NOT "timeseries"/"flex". A task's requirements + # must therefore be satisfied by tasks run in this stage itself. + satisfied: set[str] = {"grid"} if load_from is not None else set() reactive_set = False - flex_imported = False has_save = False for step in pipeline: task_name, _params = _split_step(step) - if task_name not in known_tasks(): + if task_name not in known: raise ValueError( f"Unknown task '{task_name}' in stage '{name}'. " - f"Known: {known_tasks()}" + f"Known: {sorted(known)}" ) - if task_name in _GRID_CREATING_TASKS: - grid_available = True - if task_name in _TS_TASKS: - if reactive_set: - raise ValueError( - f"Stage '{name}': time-series task " - f"'{task_name}' comes after 'reactive_power' " - f"— reactive_power must be the last " - f"time-series-altering step." - ) - ts_set = True - if task_name == "reactive_power": - reactive_set = True - if task_name in _FLEX_IMPORTS: - flex_imported = True - if not grid_available: - raise ValueError( - f"Stage '{name}': task '{task_name}' requires " - f"a loaded grid (setup_grid or " - f"load_from_base) before it." - ) - # load_from does NOT satisfy these: _load_artifact reloads the - # grid with import_timeseries=False and drops flex data, so a - # time-series (and, for optimize, a flex-import) task must run - # in the stage itself even after a load_from. - if task_name in {"analyze", "reinforce"} and not ts_set: + meta = get_task_meta(task_name) + + # reactive_power must be the last time-series-altering step. + if meta.ts_altering and reactive_set: raise ValueError( - f"Stage '{name}': task '{task_name}' requires time " - f"series to be set (e.g. worst_case_ts or " - f"oedb_ts) before it." + f"Stage '{name}': time-series task '{task_name}' comes " + f"after 'reactive_power' — reactive_power must be the " + f"last time-series-altering step." + ) + + # Check declared requirements against what the stage provides. + missing = meta.requires - satisfied + if missing: + cap = next( + (c for c in _REQUIREMENT_PRIORITY if c in missing), + sorted(missing)[0], + ) + detail = _REQUIREMENT_MESSAGES.get( + cap, f"requires '{cap}' to be established before it" ) - if task_name == "optimize": - if not ts_set: - raise ValueError( - f"Stage '{name}': 'optimize' requires time " - f"series." - ) - if not flex_imported: - raise ValueError( - f"Stage '{name}': 'optimize' requires at least " - f"one flex asset to be imported." - ) - if task_name == "base_reinforce" and not grid_available: raise ValueError( - f"Stage '{name}': 'base_reinforce' requires a " - f"loaded grid before it." + f"Stage '{name}': task '{task_name}' {detail}." ) + + satisfied |= meta.provides + if task_name == "reactive_power": + reactive_set = True if task_name == "save": has_save = True From 9b696083ea30650f957cea2261333ea967628a34 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 1 Jul 2026 13:08:33 +0200 Subject: [PATCH 34/66] refactor: shared time-series year-alignment helper + small cleanups - tools.align_series_to_timeindex: single helper that shifts a series' year (via DateOffset, leap-safe) and reindexes onto the edisgo time index, with an optional end-of-period step for SOC series. Used by both tasks.io.import_overlying_grid_data and powermodels_io (which now reindexes the SOC series instead of .loc, so a missing step yields NaN not KeyError). - context: declare overlying_grid_data as a RunContext field instead of a dynamically-set attribute; the task reads it directly. - EDisGo.run_pipeline: accept and forward overlying_grid_data (API symmetry with run_edisgo). - config._adapt_ego_legacy: deep-copy cfg['database'] before injecting ssh so the caller's config is not mutated. --- edisgo/edisgo.py | 10 ++++++-- edisgo/io/powermodels_io.py | 28 ++++++++------------- edisgo/run/config.py | 6 +++-- edisgo/run/context.py | 6 +++++ edisgo/run/tasks/io.py | 28 ++++----------------- edisgo/tools/tools.py | 49 +++++++++++++++++++++++++++++++++++++ 6 files changed, 82 insertions(+), 45 deletions(-) diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 1775d3e25..0f0d258a3 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -243,7 +243,7 @@ def config(self): def config(self, kwargs): self._config = Config(**kwargs) - def run_pipeline(self, config): + def run_pipeline(self, config, overlying_grid_data=None): """ Run a YAML/JSON task pipeline on this EDisGo instance. @@ -253,6 +253,10 @@ def run_pipeline(self, config): ---------- config : str, :class:`pathlib.Path`, or dict Pipeline config as path to a YAML/JSON file or as a dict. + overlying_grid_data : dict, optional + Overlying-grid data (e.g. eTraGo results) consumed by the + ``import_overlying_grid_data`` task when + ``overlying_grid.source == "etrago"``. Returns ------- @@ -262,7 +266,9 @@ def run_pipeline(self, config): """ from edisgo.run import _run_pipeline_on - return _run_pipeline_on(self, config) + return _run_pipeline_on( + self, config, overlying_grid_data=overlying_grid_data + ) def import_ding0_grid(self, path, legacy_ding0_grids=True): """ diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 2f55104b3..86a264f71 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1029,26 +1029,18 @@ def _build_battery_storage( """ branches = pd.concat([psa_net.lines, psa_net.transformers]) if not edisgo_obj.overlying_grid.storage_units_soc.empty: - # Select relevant timesteps - timesteps = edisgo_obj.timeseries.timeindex.union( - [ - edisgo_obj.timeseries.timeindex[-1] - + edisgo_obj.timeseries.timeindex.freq - ] + # Align the SOC series (which may use another year) onto the edisgo + # time index plus one end-of-period step. Uses reindex, so a missing + # step yields NaN instead of a KeyError. + from edisgo.tools.tools import align_series_to_timeindex + + soc_aligned = align_series_to_timeindex( + edisgo_obj.overlying_grid.storage_units_soc, + edisgo_obj.timeseries.timeindex, + extra_step=True, ) - - # If the overlying grid data uses another year in the timeindex then - # edisgo.timindex, unify them - og_year = edisgo_obj.overlying_grid.storage_units_soc.index[0].year - year_diff = og_year - edisgo_obj.timeseries.timeindex[0].year - if year_diff != 0: - # Shift by whole years instead of Timestamp.replace(year=...), - # which raises on Feb 29 when the target year is not a leap year. - timesteps = timesteps + pd.DateOffset(years=year_diff) - data = pd.concat( - [edisgo_obj.overlying_grid.storage_units_soc.loc[timesteps]] - * len(edisgo_obj.topology.storage_units_df), + [soc_aligned] * len(edisgo_obj.topology.storage_units_df), axis=1, ).values else: diff --git a/edisgo/run/config.py b/edisgo/run/config.py index 6f095133f..ed50438ff 100644 --- a/edisgo/run/config.py +++ b/edisgo/run/config.py @@ -410,9 +410,11 @@ def _adapt_ego_legacy(cfg: dict) -> dict: }, } if "database" in cfg: - adapted["database"] = cfg["database"] + # Deep-copy so injecting ssh below does not mutate the caller's + # cfg["database"] (which is merged again in _deep_merge afterwards). + adapted["database"] = copy.deepcopy(cfg["database"]) if "ssh" in cfg: - adapted["database"]["ssh"] = cfg["ssh"] + adapted["database"]["ssh"] = copy.deepcopy(cfg["ssh"]) for side_key in ("eGo", "eTraGo", "ssh", "_comment", "_workflow"): cfg.pop(side_key, None) cfg.pop("eDisGo", None) diff --git a/edisgo/run/context.py b/edisgo/run/context.py index f07effadf..04b308d48 100644 --- a/edisgo/run/context.py +++ b/edisgo/run/context.py @@ -62,6 +62,11 @@ class RunContext: The fully resolved pipeline config (after ``extends``, ``external_config``, and eGo-legacy adaptation). Tasks can read supplementary keys like ``database.*`` from here. + overlying_grid_data : dict or None + Overlying-grid data (e.g. eTraGo results) injected via the + ``overlying_grid_data=`` argument of :func:`edisgo.run.run_edisgo`. + Consumed by the ``import_overlying_grid_data`` task when + ``overlying_grid.source == "etrago"``. """ @@ -75,6 +80,7 @@ class RunContext: stage_artifacts: dict[str, Path] = field(default_factory=dict) current_stage: str | None = None raw_config: dict[str, Any] = field(default_factory=dict) + overlying_grid_data: Any = None def ensure_engine(self): """ diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 77ad9f4c2..6ffba2ee8 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -239,7 +239,7 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): return edisgo source = og_cfg.get("source") - overlying_grid_data = getattr(ctx, "overlying_grid_data", None) + overlying_grid_data = ctx.overlying_grid_data edisgo_ti = edisgo.timeseries.timeindex soc_attrs = { @@ -248,29 +248,11 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): "thermal_storage_units_central_soc", } + from edisgo.tools.tools import align_series_to_timeindex + def _to_edisgo_timeindex(ts, extra_step=False): - """ - Shift ``ts``'s index year to match the edisgo timeindex and reindex - onto it. ``extra_step`` appends one trailing step (for SOC series, - which carry an end-of-period state). Returns ``ts`` unchanged for - empty inputs or an empty edisgo timeindex. - """ - if ts is None or ts.empty or edisgo_ti.empty: - return ts - year_diff = edisgo_ti[0].year - ts.index[0].year - if year_diff != 0: - ts = ts.copy() - ts.index = ts.index + pd.DateOffset(years=year_diff) - target = edisgo_ti - if extra_step: - # Derive the step only when it can be inferred; a single-timestamp - # timeindex with no freq cannot, so fall back to no extra step. - freq = edisgo_ti.freq or ( - edisgo_ti[1] - edisgo_ti[0] if len(edisgo_ti) > 1 else None - ) - if freq is not None: - target = edisgo_ti.union([edisgo_ti[-1] + freq]) - return ts.reindex(target) + # bind the stage's edisgo time index to the shared aligner + return align_series_to_timeindex(ts, edisgo_ti, extra_step=extra_step) if source not in ("etrago", "csv"): ctx.logger.warning( diff --git a/edisgo/tools/tools.py b/edisgo/tools/tools.py index 29633f752..fd1860cba 100644 --- a/edisgo/tools/tools.py +++ b/edisgo/tools/tools.py @@ -38,6 +38,55 @@ logger = logging.getLogger(__name__) +def align_series_to_timeindex(ts, timeindex, extra_step=False): + """ + Align a time series to a target time index, tolerating a year mismatch. + + Data imported for the overlying grid (from CSV or eTraGo) may be indexed + in a different year than the EDisGo time index. This helper shifts the + series' index by whole years to match ``timeindex`` (using + :class:`pandas.DateOffset`, which — unlike ``Timestamp.replace(year=...)`` + — does not raise on a Feb-29 timestamp when the target year is not a leap + year) and reindexes onto it. Missing steps become ``NaN`` rather than + raising a ``KeyError``. + + Parameters + ---------- + ts : :pandas:`pandas.Series` or \ + :pandas:`pandas.DataFrame` or None + The time series to align. Returned unchanged if ``None``, empty, or + when ``timeindex`` is empty. + timeindex : :pandas:`pandas.DatetimeIndex` + Target time index to align to. + extra_step : bool, optional + If ``True``, append one trailing step to the target index (used for + state-of-charge series that carry an end-of-period value). The step + width is taken from ``timeindex.freq``, falling back to the spacing + of the first two entries; if neither is available (single-entry + index without freq) no extra step is added. + + Returns + ------- + Same type as ``ts`` + ``ts`` reindexed onto the (optionally extended) target index. + + """ + if ts is None or ts.empty or timeindex.empty: + return ts + year_diff = timeindex[0].year - ts.index[0].year + if year_diff != 0: + ts = ts.copy() + ts.index = ts.index + pd.DateOffset(years=year_diff) + target = timeindex + if extra_step: + freq = timeindex.freq or ( + timeindex[1] - timeindex[0] if len(timeindex) > 1 else None + ) + if freq is not None: + target = timeindex.union([timeindex[-1] + freq]) + return ts.reindex(target) + + def select_worstcase_snapshots(edisgo_obj): """ Select two worst-case snapshots from time series From 595754633f050493b6904b48e94bd4110290f583 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 1 Jul 2026 13:08:33 +0200 Subject: [PATCH 35/66] test: add task-level tests for the run pipeline Cover the previously-untested task control flow (source of the review bugs): task_manual_ts applies its kwargs, import_overlying_grid_data handles the disabled/unknown/etrago-without-data/empty-etrago/csv-without-path branches without crashing, and every bundled preset passes the metadata-driven validator. --- tests/run/test_tasks.py | 104 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tests/run/test_tasks.py diff --git a/tests/run/test_tasks.py b/tests/run/test_tasks.py new file mode 100644 index 000000000..c6c12631e --- /dev/null +++ b/tests/run/test_tasks.py @@ -0,0 +1,104 @@ +""" +Unit tests for individual pipeline tasks in :mod:`edisgo.run.tasks`. + +These cover the task control flow that unit tests previously missed — the +task modules were the source of every bug found in the review. They run +without a database or SSH tunnel: a small self-constructed ding0 grid is +enough, and the DB-free branches of ``import_overlying_grid_data`` are +exercised directly. +""" +import glob +import os + +import pandas as pd +import pytest + +import edisgo.run as edisgo_run + +from edisgo.edisgo import EDisGo +from edisgo.run.config import load_config +from edisgo.run.context import RunContext +from edisgo.run.tasks.io import task_import_overlying_grid_data +from edisgo.run.tasks.timeseries import task_manual_ts +from edisgo.run.validator import validate + + +@pytest.fixture +def edisgo_obj(): + """Small ding0 grid with a 3-step time index, no DB access.""" + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) + edisgo.set_timeindex(pd.date_range("2011-01-01", periods=3, freq="h")) + return edisgo + + +class TestManualTs: + def test_manual_ts_applies_active_power(self, edisgo_obj): + """ + task_manual_ts must forward the eGo-style ``*_active_power`` args to + EDisGo.set_time_series_manual's real parameter names (regression: the + task used to pass unsupported kwargs and always raised TypeError). + """ + ti = edisgo_obj.timeseries.timeindex + gen = edisgo_obj.topology.generators_df.index[0] + df = pd.DataFrame({gen: [0.1, 0.2, 0.3]}, index=ti) + + ctx = RunContext() + result = task_manual_ts(edisgo_obj, ctx, generators_active_power=df) + + assert gen in result.timeseries.generators_active_power.columns + assert ctx.flags["timeseries_set"] is True + + +class TestImportOverlyingGridData: + def _ctx(self, og_cfg, overlying_grid_data=None): + return RunContext( + raw_config={"overlying_grid": og_cfg}, + overlying_grid_data=overlying_grid_data, + ) + + def test_disabled_returns_unchanged(self): + """enabled: false short-circuits before the grid is even touched.""" + sentinel = object() + ctx = self._ctx({"enabled": False}) + assert task_import_overlying_grid_data(sentinel, ctx) is sentinel + + def test_unknown_source_warns(self, edisgo_obj, caplog): + ctx = self._ctx({"enabled": True, "source": "bogus"}) + result = task_import_overlying_grid_data(edisgo_obj, ctx) + assert result is edisgo_obj + assert "unknown source" in caplog.text + + def test_etrago_without_data_warns(self, edisgo_obj, caplog): + ctx = self._ctx({"enabled": True, "source": "etrago"}, + overlying_grid_data=None) + result = task_import_overlying_grid_data(edisgo_obj, ctx) + assert result is edisgo_obj + assert "no" in caplog.text.lower() + + def test_etrago_empty_data_does_not_crash(self, edisgo_obj): + """ + A partial/empty etrago dict must not raise (regression: the task used + to call .empty on dict.get() results that were None). + """ + ctx = self._ctx({"enabled": True, "source": "etrago"}, + overlying_grid_data={}) + # must simply return without AttributeError + assert task_import_overlying_grid_data(edisgo_obj, ctx) is edisgo_obj + + def test_csv_without_path_warns(self, edisgo_obj, caplog): + ctx = self._ctx({"enabled": True, "source": "csv"}) + result = task_import_overlying_grid_data(edisgo_obj, ctx) + assert result is edisgo_obj + assert "path" in caplog.text.lower() + + +def test_all_bundled_presets_validate(): + """ + Every bundled preset must pass the (metadata-driven) validator — this + keeps the task requires/provides declarations in sync with real configs. + """ + presets_dir = os.path.join(os.path.dirname(edisgo_run.__file__), "presets") + presets = sorted(glob.glob(os.path.join(presets_dir, "*.yaml"))) + assert presets, "no bundled presets found" + for path in presets: + validate(load_config(path)) From 1565b26a85dc97e57ddc4901bd7cabd19c04df9b Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Thu, 2 Jul 2026 16:56:43 +0200 Subject: [PATCH 36/66] Add configurable database source (local egon-data / OEP) and fix SSH tunnel key Add engine_from_settings() mapping a scenario "database" section onto engine(): - source: "local" -> egon-data via SSH tunnel, using the optional config_path or the default ~/.ssh/egon-data.configuration.yaml (default_config_path(), overridable via EGON_DATA_CONFIG) - source: "oep" (or omitted) -> Open Energy Platform, as before engine(ssh=True) now falls back to the default config location when no path is given. RunContext.ensure_engine() is source-driven and delegates to engine_from_settings(); the legacy ssh.enabled flag and explicit direct-local database (host given) remain supported. Fix ssh_tunnel(): pass ssh_pkey as a string. Passing the pathlib.Path from credentials() made sshtunnel silently ignore the key and fall back to the default keys in ~/.ssh, failing gateway authentication. --- edisgo/io/db.py | 101 +++++++++++++++++++++++++++++++++++++++--- edisgo/run/context.py | 55 +++++++++++------------ 2 files changed, 121 insertions(+), 35 deletions(-) diff --git a/edisgo/io/db.py b/edisgo/io/db.py index fd12673af..daf1d0dce 100644 --- a/edisgo/io/db.py +++ b/edisgo/io/db.py @@ -36,6 +36,33 @@ logger = logging.getLogger(__name__) +#: Default location of the egon-data SSH tunnel configuration file. Used when +#: no explicit config path is passed and the connection mode is not forced. +#: Can be overridden through the ``EGON_DATA_CONFIG`` environment variable. +DEFAULT_EGON_DATA_CONFIG = "~/.ssh/egon-data.configuration.yaml" + + +def default_config_path() -> Path | None: + """ + Return the path to the egon-data SSH configuration file, or ``None``. + + The location is read from the ``EGON_DATA_CONFIG`` environment variable and + falls back to :data:`DEFAULT_EGON_DATA_CONFIG` + (``~/.ssh/egon-data.configuration.yaml``). ``None`` is returned when the + resolved path does not point to an existing file, which callers use as the + signal to fall back to the Open Energy Platform (OEP). + + Returns + ------- + pathlib.Path or None + Path to an existing egon-data configuration file, or ``None`` if none + was found. + + """ + raw = os.environ.get("EGON_DATA_CONFIG", DEFAULT_EGON_DATA_CONFIG) + path = Path(raw).expanduser() + return path if path.is_file() else None + def config_settings(path: Path | str) -> dict[str, dict[str, str | int | Path]]: """ @@ -155,7 +182,11 @@ def ssh_tunnel(cred: dict) -> str: server = SSHTunnelForwarder( ssh_address_or_host=(cred["SSH_HOST"], 22), ssh_username=cred["SSH_USER"], - ssh_pkey=cred["SSH_PKEY"], + # SSHTunnelForwarder only accepts a string path (or a loaded paramiko + # PKey) here. Passing the pathlib.Path produced by credentials() makes + # sshtunnel silently ignore the key and fall back to the default keys + # in ~/.ssh, which fails authentication against the gateway. + ssh_pkey=str(cred["SSH_PKEY"]), remote_bind_address=(cred["PGRES_HOST"], cred["PORT"]), ) server.start() @@ -172,12 +203,17 @@ def engine( Parameters ---------- path : str or pathlib.Path, optional (default=None) - Path to configuration YAML file of egon-data database. + Path to configuration YAML file of egon-data database. Only used when + ``ssh=True``. If None, the default location is used + (``EGON_DATA_CONFIG`` environment variable or + ``~/.ssh/egon-data.configuration.yaml``, see + :func:`default_config_path`). ssh : bool (default=False) - If False, connects to the remote Open Energy Platform database (using the - token, see parameter `token`). If True, establishes an ssh tunnel to a local - egon-data database using the connection information in the configuration YAML - given through `path`. + If False, connects to the remote Open Energy Platform database (using + the token, see parameter `token`). If True, establishes an ssh tunnel + to a local egon-data database using the connection information in the + configuration YAML given through `path` (or the default location if + `path` is None). token : str or pathlib.Path, optional (default=None) Token for database connection or path to text file containing token. If empty the default token file in the config folder OEP_TOKEN.txt @@ -239,6 +275,15 @@ def engine( echo=False, ) + if path is None: + path = default_config_path() + if path is None: + raise ValueError( + "SSH connection requested but no egon-data configuration file " + "was found (checked the EGON_DATA_CONFIG environment variable " + f"and the default location {DEFAULT_EGON_DATA_CONFIG})." + ) + cred = credentials(path=path) local_port = ssh_tunnel(cred) @@ -250,6 +295,50 @@ def engine( ) +def engine_from_settings(database: dict | None = None) -> Engine: + """ + Build a database engine from a scenario ``database`` settings section. + + This maps the data source configured in the scenario JSON onto + :func:`engine`. Recognised keys of `database`: + + * ``source`` — ``"local"`` connects to a local egon-data database through + an SSH tunnel; ``"oep"`` (or a missing/empty value) connects to the + remote Open Energy Platform (OEP), i.e. the previous default behaviour. + * ``config_path`` — optional path to the egon-data configuration YAML. + Only relevant for ``source="local"``. If omitted, the default location + is used (``EGON_DATA_CONFIG`` environment variable or + ``~/.ssh/egon-data.configuration.yaml``, see :func:`default_config_path`). + + Parameters + ---------- + database : dict or None + The ``database`` section of the scenario configuration. If None or + empty, an OEP engine is returned. + + Returns + ------- + :sqlalchemy:`sqlalchemy.Engine` + Database engine. + + """ + database = database or {} + source = str(database.get("source") or "oep").lower() + + if source in ("local", "ssh", "egon-data", "egon_data"): + # config_path may be given explicitly; otherwise engine() falls back to + # the default location (~/.ssh/egon-data.configuration.yaml). + config_path = database.get("config_path") or database.get("credentials_path") + logger.info( + f"engine_from_settings: source='local', using egon-data database " + f"via SSH tunnel (config {config_path or 'default (~/.ssh/...)'})." + ) + return engine(path=config_path, ssh=True) + + logger.info("engine_from_settings: source='oep', connecting to the OEP.") + return engine(ssh=False) + + @contextmanager def session_scope_egon_data(engine: Engine): """Provide a transactional scope around a series of operations.""" diff --git a/edisgo/run/context.py b/edisgo/run/context.py index 04b308d48..e41a07fc9 100644 --- a/edisgo/run/context.py +++ b/edisgo/run/context.py @@ -86,42 +86,37 @@ def ensure_engine(self): """ Return a database engine, creating it on first call. - Reads the ``database`` section of :attr:`raw_config` and calls - :func:`edisgo.io.db.engine`. Caches the engine on the context - so subsequent calls reuse the same connection. + The data source is chosen from the ``database`` section of + :attr:`raw_config`: + + * ``source: "local"`` — egon-data database via SSH tunnel, using + ``config_path`` if given, otherwise the default location + (``~/.ssh/egon-data.configuration.yaml``). + * ``source: "oep"`` or no ``database`` section — remote Open Energy + Platform (previous default behaviour). + + A legacy explicit direct-local database (``host`` given with SSH + disabled) is still honoured for backward compatibility. The engine is + cached on the context so subsequent calls reuse the same connection. Returns ------- sqlalchemy.engine.Engine The active database engine. - Raises - ------ - RuntimeError - If the config has no ``database`` section — indicates the - pipeline wants to reach the database without configuring - it. - """ if self.engine is not None: return self.engine - db_cfg = self.raw_config.get("database") - if not db_cfg: - raise RuntimeError( - "Task needs a database engine but no 'database' section " - "is configured." - ) + db_cfg = self.raw_config.get("database") or {} + source = str(db_cfg.get("source") or "").lower() + + # Legacy explicit direct local database: SSH disabled and explicit + # connection parameters given (host/port/user/password as passed by + # eGo). Connect straight to that postgres via psycopg2. ssh_cfg = db_cfg.get("ssh") or {} ssh_enabled = bool(ssh_cfg.get("enabled", False)) - - # Direct local database: when SSH is disabled and explicit - # connection parameters are given (host/port/user/password as - # passed by eGo), connect straight to that postgres via - # psycopg2. This avoids edisgo.io.db.engine(ssh=False), which - # is hard-wired to the remote OpenEnergyPlatform (oedialect) - # and can stall for hours on large queries. host = db_cfg.get("host") - if not ssh_enabled and host: + if source not in ("local", "oep") and host and not ssh_enabled: from sqlalchemy import create_engine user = db_cfg.get("user") @@ -144,10 +139,12 @@ def ensure_engine(self): ) return self.engine - from edisgo.io.db import engine as egon_engine + # Source-driven engine: source="local" -> egon-data via SSH tunnel + # (config_path or ~/.ssh default), source="oep"/absent -> OEP. + from edisgo.io.db import engine_from_settings - self.engine = egon_engine( - path=db_cfg.get("credentials_path"), - ssh=ssh_enabled, - ) + # A legacy ssh.enabled flag maps to source "local". + if not source and ssh_enabled: + db_cfg = {**db_cfg, "source": "local"} + self.engine = engine_from_settings(db_cfg) return self.engine From 289dbeefbd6abb290b7a3a96b54e91ccebf665f3 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Thu, 2 Jul 2026 16:56:43 +0200 Subject: [PATCH 37/66] Pin paramiko < 4.0 for SSH tunnel support The SSH tunnel to the local egon-data database (via sshtunnel/paramiko) requires paramiko < 4.0; newer versions break key handling / the tunnel. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 26d01c3f8..a2a4422b3 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ def read(fname): # sqlalchemy leads to new errors.. should be fixed at some point "numpy ==1.26.4", "pandas >= 1.4.0, < 2.2.0", + "paramiko < 4.0", "plotly < 6.0", "pydot < 4.1.0", "pypower < 5.2.0", From 8ea04cf8df7c0d177bfd7d5561b32cc7ba3b1136 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Fri, 3 Jul 2026 09:19:37 +0200 Subject: [PATCH 38/66] feat: enhance database engine support and schema handling in pipeline tasks --- edisgo/io/heat_pump_import.py | 7 ++++++- edisgo/run/runner.py | 27 ++++++++++++++++++++++++--- edisgo/run/tasks/flex.py | 3 ++- edisgo/run/tasks/grid.py | 5 ++++- edisgo/tools/config.py | 17 ++++++++++++----- 5 files changed, 48 insertions(+), 11 deletions(-) diff --git a/edisgo/io/heat_pump_import.py b/edisgo/io/heat_pump_import.py index c796f9f89..b95867a0b 100644 --- a/edisgo/io/heat_pump_import.py +++ b/edisgo/io/heat_pump_import.py @@ -337,8 +337,13 @@ def _get_individual_heat_pump_capacity(): ["egon_map_zensus_mvgd_buildings", "egon_map_zensus_weather_cell"], "boundaries", ) + # egon_etrago_bus/egon_etrago_link live in schema "grid" in a local + # egon-data database, whereas the OEP path resolves them via the table/schema + # alias mapping keyed on "supply". Only switch the schema for the local + # (SSH/psycopg2) backend; keep "supply" for the remote OEP. + etrago_schema = "supply" if "openenergyplatform" in str(engine.url) else "grid" egon_etrago_bus, egon_etrago_link = config.import_tables_from_oep( - engine, ["egon_etrago_bus", "egon_etrago_link"], "supply" + engine, ["egon_etrago_bus", "egon_etrago_link"], etrago_schema ) building_ids = edisgo_object.topology.loads_df.building_id.unique() diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py index d3464010e..cf1f531cc 100644 --- a/edisgo/run/runner.py +++ b/edisgo/run/runner.py @@ -44,7 +44,7 @@ logger = logging.getLogger("edisgo.run.runner") -def run_edisgo(config, overlying_grid_data=None) -> Any: +def run_edisgo(config, overlying_grid_data=None, engine=None) -> Any: """ Run an eDisGo pipeline from a YAML/JSON config or dict. @@ -58,6 +58,16 @@ def run_edisgo(config, overlying_grid_data=None) -> Any: config : str, pathlib.Path, or dict Path to a YAML/JSON pipeline config, or an in-memory dict of the same shape. + overlying_grid_data : dict, optional + Overlying-grid data (e.g. eTraGo results) consumed by the + ``import_overlying_grid_data`` task. + engine : sqlalchemy.engine.Engine, optional + Pre-built database engine to use for all DB-backed tasks. When + given, it is cached on the :class:`~edisgo.run.context.RunContext` + so every task reuses it (via :meth:`RunContext.ensure_engine`) + instead of building its own from the config. This lets a caller + (e.g. eGo) supply a single connection that overrides the + ``database`` section of the config/preset. Returns ------- @@ -67,10 +77,12 @@ def run_edisgo(config, overlying_grid_data=None) -> Any: stage. """ - return _run_pipeline_on(None, config, overlying_grid_data=overlying_grid_data) + return _run_pipeline_on( + None, config, overlying_grid_data=overlying_grid_data, engine=engine + ) -def _run_pipeline_on(edisgo, config, overlying_grid_data=None): +def _run_pipeline_on(edisgo, config, overlying_grid_data=None, engine=None): """ Internal runner shared by :func:`run_edisgo` and the EDisGo method. @@ -99,6 +111,15 @@ def _run_pipeline_on(edisgo, config, overlying_grid_data=None): validate(cfg) ctx = _build_context(cfg) ctx.overlying_grid_data = overlying_grid_data + # A caller-supplied engine (e.g. from eGo) overrides the config/preset + # database section: caching it on the context makes ensure_engine() return + # it for every DB-backed task. + if engine is not None: + ctx.engine = engine + ctx.logger.info( + f"run_edisgo: using caller-supplied database engine " + f"'{getattr(engine.url, 'database', engine)}' for all tasks." + ) for stage in cfg["stages"]: ctx.current_stage = stage["name"] diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py index 87ba4a35e..815b28daa 100644 --- a/edisgo/run/tasks/flex.py +++ b/edisgo/run/tasks/flex.py @@ -277,6 +277,7 @@ def task_import_generators(edisgo, ctx, *, generator_scenario=None): """ edisgo.import_generators( - generator_scenario=generator_scenario or ctx.scenario + generator_scenario=generator_scenario or ctx.scenario, + engine=ctx.ensure_engine(), ) return edisgo diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py index 348438bf5..45e050076 100644 --- a/edisgo/run/tasks/grid.py +++ b/edisgo/run/tasks/grid.py @@ -95,7 +95,10 @@ def task_setup_grid( ) if import_generators: - edisgo.import_generators(generator_scenario=generator_scenario) + edisgo.import_generators( + generator_scenario=generator_scenario, + engine=ctx.ensure_engine(), + ) if timeindex is not None: ti_df = pd.date_range( diff --git a/edisgo/tools/config.py b/edisgo/tools/config.py index 6111c1483..2a986e81e 100644 --- a/edisgo/tools/config.py +++ b/edisgo/tools/config.py @@ -312,12 +312,19 @@ def import_tables_from_oep( table_name, metadata, autoload_with=engine, schema=schema_name ) + # The declarative mapper requires a primary key. Some egon-data + # tables/views have none reflected; declare all columns as a + # composite primary key so the ORM class can be built. This + # mirrors what saio does on the OEP path ("assuming primary + # key") and only affects mapping, not the data read back. + class_dict = {"__tablename__": table_name, "__table__": table} + if not list(table.primary_key.columns): + class_dict["__mapper_args__"] = { + "primary_key": list(table.columns) + } + # dynamisch eine ORM-Klasse erzeugen - orm_class = type( - table_name, - (Base,), - {"__tablename__": table_name, "__table__": table}, - ) + orm_class = type(table_name, (Base,), class_dict) orm_classes.append(orm_class) return orm_classes From c36fc8cc1bc51b5e0258c6f44540860cf2a2e87c Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Fri, 3 Jul 2026 14:04:41 +0200 Subject: [PATCH 39/66] Keep pooled DB connections alive across long multi-grid runs A cached SSH-tunneled egon-data engine is reused across many grids in an eGo run. During a grid's multi-minute OPF the pooled connection sits idle and the server/SSH tunnel closes it, so a later grid got a dead connection ("server closed the connection unexpectedly"). - pool_pre_ping=True: validate (and transparently replace) a connection before use. - pool_recycle=3600: proactively drop connections older than an hour. - set_keepalive=30.0 on the SSH tunnel: keep the transport alive during long idle periods so the tunnel is not torn down. --- edisgo/io/db.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/edisgo/io/db.py b/edisgo/io/db.py index daf1d0dce..cc3b375d3 100644 --- a/edisgo/io/db.py +++ b/edisgo/io/db.py @@ -188,6 +188,10 @@ def ssh_tunnel(cred: dict) -> str: # in ~/.ssh, which fails authentication against the gateway. ssh_pkey=str(cred["SSH_PKEY"]), remote_bind_address=(cred["PGRES_HOST"], cred["PORT"]), + # Keep the SSH transport alive during long idle periods (e.g. a + # multi-minute OPF between database queries in multi-grid eGo runs) so + # the tunnel is not torn down and connections stay usable. + set_keepalive=30.0, ) server.start() @@ -292,6 +296,16 @@ def engine( f"{cred['POSTGRES_PASSWORD']}@{cred['PGRES_HOST']}:" f"{local_port}/{cred['POSTGRES_DB']}", echo=False, + # This engine is typically cached and reused across many long-running + # tasks/grids (e.g. one eGo run computes grid after grid, each with a + # multi-minute OPF during which the pooled connection sits idle). The + # server or SSH tunnel closes such idle connections, so a later grid + # would otherwise get a dead connection ("server closed the connection + # unexpectedly"). pool_pre_ping validates (and transparently replaces) + # a connection before use; pool_recycle proactively drops connections + # older than an hour. + pool_pre_ping=True, + pool_recycle=3600, ) From 6c529950b57b4bb3d071daadd0e31856465f2f3d Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Tue, 7 Jul 2026 15:11:28 +0200 Subject: [PATCH 40/66] Accept feasible non-optimal OPF solutions instead of computing an IIS Large SOC OPF instances often end with barrier status SUBOPTIMAL: a feasible primal point exists but the convergence tolerances were not met. The previous check treated every status != OPTIMAL as infeasible and called compute_conflict!, which raises Gurobi error 10015 ("Cannot compute IIS on a feasible model"), killing the Julia process and failing the grid run. Now the solution is used whenever the solver proves optimality or reports a feasible primal point; the IIS conflict is only computed when there is genuinely no primal solution. The SOC tightness check remains restricted to proven optima. --- edisgo/opf/eDisGo_OPF.jl/Main.jl | 35 ++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/edisgo/opf/eDisGo_OPF.jl/Main.jl b/edisgo/opf/eDisGo_OPF.jl/Main.jl index 0084b7fd7..4b0413328 100644 --- a/edisgo/opf/eDisGo_OPF.jl/Main.jl +++ b/edisgo/opf/eDisGo_OPF.jl/Main.jl @@ -66,25 +66,34 @@ function optimize_edisgo() println("Starting convex SOC AC-OPF with Gurobi.") result_soc, pm = eDisGo_OPF.solve_mn_opf_bf_flex(data_edisgo_mn, SOCBFPowerModelEdisgo, gurobi) #println("Termination status: "*result_soc["termination_status"]) - if result_soc["termination_status"] != MOI.OPTIMAL - # if result_soc["termination_status"] == MOI.SUBOPTIMAL_TERMINATION - # PowerModels.update_data!(data_edisgo_mn, result_soc["solution"]) - # else + # A feasible solution exists if the solver proved optimality OR reports a + # feasible primal point (e.g. SUBOPTIMAL / ALMOST_OPTIMAL under the barrier + # tolerances set above). Only when there is genuinely no primal solution do + # we diagnose the infeasibility via an IIS conflict — calling + # compute_conflict! on a feasible model raises Gurobi error 10015. + has_solution = result_soc["termination_status"] == MOI.OPTIMAL || + MOI.get(pm.model, MOI.PrimalStatus()) == MOI.FEASIBLE_POINT + if !has_solution JuMP.compute_conflict!(pm.model) if MOI.get(pm.model, MOI.ConflictStatus()) == MOI.CONFLICT_FOUND iis_model, _ = copy_conflict(pm.model) print(iis_model) end - #end - elseif result_soc["termination_status"] == MOI.OPTIMAL - # Check if SOC constraint is tight - soc_tight, soc_dict = eDisGo_OPF.check_SOC_equality(result_soc, data_edisgo) - # Save SOC violations if SOC is not tight - if !soc_tight - open(joinpath(results_path, ding0_grid*"_"*join(data_edisgo["flexibilities"])*".json"), "w") do f - write(f, JSON.json(soc_dict)) + else + # Check if SOC constraint is tight (only meaningful for a proven optimum). + soc_tight = true + if result_soc["termination_status"] == MOI.OPTIMAL + soc_tight, soc_dict = eDisGo_OPF.check_SOC_equality(result_soc, data_edisgo) + # Save SOC violations if SOC is not tight + if !soc_tight + open(joinpath(results_path, ding0_grid*"_"*join(data_edisgo["flexibilities"])*".json"), "w") do f + write(f, JSON.json(soc_dict)) + end + println("SOC solution is not tight!") end - println("SOC solution is not tight!") + else + println("SOC model terminated feasible but not optimal ("* + string(result_soc["termination_status"])*"); using the solution.") end PowerModels.update_data!(data_edisgo_mn, result_soc["solution"]) data_edisgo_mn["solve_time"] = result_soc["solve_time"] From 80a24e2e4abf4f6f64d5ed965888a94f8717ee74 Mon Sep 17 00:00:00 2001 From: joda9 <66819219+joda9@users.noreply.github.com> Date: Wed, 8 Jul 2026 13:33:42 +0200 Subject: [PATCH 41/66] feat: configurable timestep selection for the run pipeline (#663) (#692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `select_timesteps` pipeline task that reduces the analysed time index to a selected subset, in two modes: - manual: an explicit set/range of timesteps (positioned before the imports so egon_data downloads are restricted to the selected steps); - auto: the most critical time intervals, via either a power flow (`get_most_critical_time_intervals(by="power_flow")`) or the residual load (`by="residual_load"`, no power flow — intervals centered on the highest/lowest residual-load steps, snapped to `time_step_day_start`). Auto selection may yield two disconnected intervals. `pm_optimize` now detects a non-contiguous time index and runs a separate, independent OPF per contiguous interval (storage/heat state does not carry across the gap), merging the per-interval results and reporting per-interval solve status. `reinforce` handles the reduced index unchanged. Architecture: pipeline tasks stay thin (mode selection + data transfer); the computation lives in edisgo core: - selection logic in tools.temporal_complexity_reduction (`get_most_critical_time_steps`/`_intervals` gain a `by` mode; public `select_two_intervals`/`intervals_overlap`); - the multi-interval OPF split/merge in opf.powermodels_opf.pm_optimize. Also: - EV flexibility bands built in a dedicated `build_flexibility_bands` task and aligned (year + frequency) to the analysis index in `reduce_timeseries_data_to_given_timeindex`; - `OPFResults.interval_results` for the per-interval solve report; - config surface: a top-level `timeseries_selection` block + `uc5_select_timesteps` preset + `run_example_05.py`; - tests in tests/run, tests/tools, tests/opf. Co-authored-by: Moritz Schloesser --- edisgo/io/powermodels_io.py | 33 +- edisgo/opf/powermodels_opf.py | 274 ++++++++++++++- edisgo/opf/results/opf_result_class.py | 8 + edisgo/run/presets/uc4_example.yaml | 6 +- edisgo/run/presets/uc5_select_timesteps.yaml | 117 +++++++ edisgo/run/tasks/__init__.py | 17 +- edisgo/run/tasks/analysis.py | 20 +- edisgo/run/tasks/flex.py | 119 +++++-- edisgo/run/tasks/timeseries.py | 306 +++++++++++++++- edisgo/tools/temporal_complexity_reduction.py | 331 +++++++++++++++++- edisgo/tools/tools.py | 21 +- run_example_05.py | 39 +++ tests/opf/test_powermodels_opf.py | 191 ++++++++++ tests/run/test_tasks.py | 247 ++++++++++++- tests/run/test_validator.py | 96 +++-- .../test_temporal_complexity_reduction.py | 89 +++++ 16 files changed, 1831 insertions(+), 83 deletions(-) create mode 100644 edisgo/run/presets/uc5_select_timesteps.yaml create mode 100644 run_example_05.py diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index a978a0805..223d96cf3 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -370,7 +370,7 @@ def from_powermodels( # calculate relative error df2 = deepcopy(df) for flex in df2.columns: - if type(hv_flex_dict[flex]) == pd.Series: + if isinstance(hv_flex_dict[flex], pd.Series): abs_error = abs(df2[flex].values - hv_flex_dict[flex].values) rel_error = [ abs_error[i] / hv_flex_dict[flex].iloc[i] @@ -379,10 +379,15 @@ def from_powermodels( for i in range(len(abs_error)) ] else: - abs_error = abs(df2[flex].values - hv_flex_dict[flex].sum(axis=1).values) + abs_error = abs( + df2[flex].values - hv_flex_dict[flex].sum(axis=1).values + ) rel_error = [ abs_error[i] / hv_flex_dict[flex].sum(axis=1).iloc[i] - if ((abs_error > 0.01)[i] & (hv_flex_dict[flex].sum(axis=1).iloc[i] != 0)) + if ( + (abs_error > 0.01)[i] + & (hv_flex_dict[flex].sum(axis=1).iloc[i] != 0) + ) else 0 for i in range(len(abs_error)) ] @@ -1061,6 +1066,18 @@ def _build_battery_storage( * edisgo_obj.topology.storage_units_df.max_hours ) + # The end-of-period SoC step (timeindex[-1] + freq) is only used as the OPF + # boundary (soc_end) and is not an optimized time step. When the time index + # is a reduced, non-contiguous selection, that step can fall in a gap and be + # missing from the source SoC series (which only carried a trailing step for + # the very last interval), leaving it NaN. A NaN boundary makes the Julia OPF + # fail with "Inf - Inf". Forward-fill (then back-fill) so the boundary takes + # the interval's last valid SoC — a harmless approximation for a throwaway + # scaffolding step. + edisgo_obj.overlying_grid.storage_units_soc = ( + edisgo_obj.overlying_grid.storage_units_soc.ffill().bfill() + ) + for stor_i in np.arange(len(flexible_storage_units)): idx_bus = _mapping( psa_net, @@ -1357,6 +1374,12 @@ def _build_heat_storage(psa_net, pm, edisgo_obj, s_base, flexible_hps, opf_versi edisgo_obj.overlying_grid.heat_storage_units_soc = pd.concat( [df_decentral, df_central], axis=1 ) + # Fill the end-of-period boundary SoC step (see storage note above) so a + # reduced, non-contiguous time index does not leave a NaN boundary that + # breaks the Julia OPF. + edisgo_obj.overlying_grid.heat_storage_units_soc = ( + edisgo_obj.overlying_grid.heat_storage_units_soc.ffill().bfill() + ) heat_storage_df = heat_storage_df.loc[flexible_hps] for stor_i in np.arange(len(flexible_hps)): @@ -1616,7 +1639,7 @@ def _build_hv_requirements( ) for i in np.arange(len(opf_flex)): - if type(hv_flex_dict[opf_flex[i]]) == pd.DataFrame: + if isinstance(hv_flex_dict[opf_flex[i]], pd.DataFrame): pm["HV_requirements"][str(i + 1)] = { "P": hv_flex_dict[opf_flex[i]].sum(axis=1).iloc[0], "name": opf_flex[i], @@ -1957,7 +1980,7 @@ def _build_component_timeseries( if (kind == "HV_requirements") & (pm["opf_version"] in [3, 4]): for i in np.arange(len(opf_flex)): - if type(hv_flex_dict[opf_flex[i]])==pd.DataFrame: + if isinstance(hv_flex_dict[opf_flex[i]], pd.DataFrame): pm_comp[(str(i + 1))] = { "P": hv_flex_dict[opf_flex[i]].sum(axis=1).round(20).tolist(), } diff --git a/edisgo/opf/powermodels_opf.py b/edisgo/opf/powermodels_opf.py index 22841f78b..a22b9c3e6 100644 --- a/edisgo/opf/powermodels_opf.py +++ b/edisgo/opf/powermodels_opf.py @@ -9,6 +9,7 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later +import copy import json import logging import os @@ -16,6 +17,7 @@ import sys import numpy as np +import pandas as pd from edisgo.flex_opt import exceptions from edisgo.io.powermodels_io import from_powermodels @@ -23,6 +25,120 @@ logger = logging.getLogger(__name__) +# Time-indexed opf_results attributes that from_powermodels overwrites on each +# call. When the OPF is run separately per interval, these must be concatenated +# across intervals so opf_results covers the full (reduced) time index. Nested +# containers (LineVariables etc.) are listed via their sub-frame attribute names. +_OPF_FLAT_TIME_FRAMES = ( + "slack_generator_t", + "hv_requirement_slacks_t", +) +_OPF_NESTED_TIME_FRAMES = ( + "lines_t", + "heat_storage_t", + "grid_slacks_t", + "battery_storage_t", +) + + +def _with_freq(index): + """Return the DatetimeIndex with its frequency inferred/attached if regular. + + Reducing/uniting time indices drops the ``freq`` attribute; several + downstream consumers (notably the powermodels OPF) do + ``timeindex[-1] + timeindex.freq`` and break on ``freq is None``. This + re-attaches the freq when the index is regularly spaced (a no-op otherwise). + """ + if index.freq is not None or len(index) < 2: + return index + inferred = pd.infer_freq(index) + if inferred is not None: + try: + return pd.DatetimeIndex(index, freq=inferred) + except (ValueError, TypeError): + return index + return index + + +def _contiguous_intervals(timeindex): + """ + Split a time index into contiguous intervals. + + Automatic timestep selection can reduce the time index to disconnected + intervals (e.g. one load-case and one feed-in-case week). This helper detects + the gap(s) so the OPF can be run separately per interval — storage/heat state + does not carry across a gap, so a single OPF over the concatenated steps would + be wrong. + + A boundary is placed wherever the spacing between two consecutive time steps + exceeds the regular step (the smallest spacing in the index). A contiguous + index therefore yields a single interval. Each returned interval has its + ``freq`` restored (set operations that produced the reduced index drop it). + + Parameters + ---------- + timeindex : pandas.DatetimeIndex + + Returns + ------- + list of pandas.DatetimeIndex + One entry per contiguous interval, in chronological order. Empty index + in -> empty list out; a single time step -> one interval. + """ + timeindex = timeindex.sort_values() + if len(timeindex) <= 1: + return [timeindex] if len(timeindex) else [] + diffs = timeindex[1:] - timeindex[:-1] + step = diffs.min() + breaks = [i + 1 for i, d in enumerate(diffs) if d > step] + starts = [0] + breaks + ends = breaks + [len(timeindex)] + return [_with_freq(timeindex[s:e]) for s, e in zip(starts, ends)] + + +def _snapshot_opf_time_frames(opf_results): + """Copy the time-indexed opf_results frames produced by one interval's OPF.""" + snap = {} + for attr in _OPF_FLAT_TIME_FRAMES: + snap[attr] = getattr(opf_results, attr).copy() + for attr in _OPF_NESTED_TIME_FRAMES: + container = getattr(opf_results, attr) + snap[attr] = { + sub: getattr(container, sub).copy() for sub in container._attributes() + } + return snap + + +def _merge_opf_time_frames(opf_results, snapshots): + """ + Concatenate per-interval opf_results snapshots by time index and write them + back onto ``opf_results``, so its detailed frames cover the full reduced + index rather than only the last interval's. + """ + + def _concat(frames): + frames = [f for f in frames if f is not None and not f.empty] + if not frames: + return pd.DataFrame() + return pd.concat(frames).sort_index() + + for attr in _OPF_FLAT_TIME_FRAMES: + setattr(opf_results, attr, _concat([s[attr] for s in snapshots])) + for attr in _OPF_NESTED_TIME_FRAMES: + container = getattr(opf_results, attr) + for sub in container._attributes(): + setattr(container, sub, _concat([s[attr][sub] for s in snapshots])) + + # Recompute the overlying_grid summary (opf_version 3/4) from the merged HV + # requirement slacks, since it is a reduction over the whole time index. + hv = opf_results.hv_requirement_slacks_t + if not hv.empty: + opf_results.overlying_grid = pd.DataFrame( + columns=["Highest error", "Mean error", "Sum error"], + index=hv.columns, + data=pd.concat([hv.max(), hv.mean(), hv.sum()], axis=1).values, + ) + def pm_optimize( edisgo_obj, @@ -37,8 +153,162 @@ def pm_optimize( silence_moi: bool = False, ) -> None: """ - Run OPF for edisgo object in julia subprocess and write results of OPF to edisgo - object. Results of OPF are time series of operation schedules of flexibilities. + Run OPF for the edisgo object and write results back to it. + + If the time index is a single contiguous interval, this runs one OPF + (:func:`_pm_optimize_single`). If the time index is NON-contiguous + (disconnected intervals, e.g. from automatic timestep selection), each + contiguous interval is optimized separately and independently — storage/heat + state does not carry across the gap — and the results are combined: + + * per-interval operation schedules accumulate in ``edisgo.timeseries``; + * the detailed ``edisgo.opf_results`` frames are merged by time index; + * a per-interval solve report is stored in + ``edisgo.opf_results.interval_results``; + * if any interval was infeasible, an + :class:`~.flex_opt.exceptions.InfeasibleModelError` is raised after the + feasible intervals' results have been stored. + + The overlying-grid SOC attributes and reactive-power time series (which + ``to_powermodels`` / ``from_powermodels`` mutate or replace on the current + interval) are snapshotted and restored pristine before each interval so a + later interval sees intact input. Parameters are as for + :func:`_pm_optimize_single`. + """ + opf_kwargs = dict( + s_base=s_base, + flexible_cps=flexible_cps, + flexible_hps=flexible_hps, + flexible_loads=flexible_loads, + flexible_storage_units=flexible_storage_units, + opf_version=opf_version, + method=method, + warm_start=warm_start, + silence_moi=silence_moi, + ) + + intervals = _contiguous_intervals(edisgo_obj.timeseries.timeindex) + if len(intervals) <= 1: + # single contiguous optimization. Re-set the (freq-restored) interval so + # the OPF sees a time index with a frequency — set operations upstream + # (e.g. timestep selection) drop it, and the OPF needs timeindex.freq. + if intervals: + edisgo_obj.set_timeindex(intervals[0]) + _pm_optimize_single(edisgo_obj, **opf_kwargs) + return + + logger.info( + f"pm_optimize: time index has {len(intervals)} disconnected intervals; " + f"running a separate OPF per interval." + ) + full_timeindex = edisgo_obj.timeseries.timeindex + + # Snapshot the shared input state that per-interval OPF runs mutate: + # * overlying-grid SOC attributes are rewritten in place by to_powermodels; + # * the reactive-power time series are fully REPLACED (not .loc-updated) by + # the set_time_series_reactive_power_control() call inside from_powermodels. + # Reactive power was set on the full reduced index before this call; restore + # this input pristine before each interval. Active-power frames are NOT + # restored — they accumulate each interval's OPF results via .loc. + og = edisgo_obj.overlying_grid + og_snapshot = {attr: copy.deepcopy(getattr(og, attr)) for attr in og._attributes} + reactive_attrs = [ + "_generators_reactive_power", + "_loads_reactive_power", + "_storage_units_reactive_power", + ] + reactive_snapshot = { + attr: copy.deepcopy(getattr(edisgo_obj.timeseries, attr, None)) + for attr in reactive_attrs + } + + def _restore_pristine_inputs(): + for attr, value in og_snapshot.items(): + setattr(og, attr, copy.deepcopy(value)) + for attr, value in reactive_snapshot.items(): + if value is not None: + setattr(edisgo_obj.timeseries, attr, copy.deepcopy(value)) + + # Pre-allocate the storage active-power schedule over the FULL reduced index + # so from_powermodels .loc-accumulates each interval's storage result instead + # of replacing the frame with an interval-only one (which would drop earlier + # intervals' storage schedules). + su_names = edisgo_obj.topology.storage_units_df.index + if len(su_names) > 0 and edisgo_obj.timeseries.storage_units_active_power.empty: + edisgo_obj.timeseries.storage_units_active_power = pd.DataFrame( + 0.0, index=full_timeindex, columns=su_names + ) + + snapshots = [] + report = [] + try: + for interval in intervals: + _restore_pristine_inputs() + edisgo_obj.set_timeindex(interval) + entry = { + "start": interval[0], + "end": interval[-1], + "status": None, + "solver": None, + "solution_time": None, + } + try: + _pm_optimize_single(edisgo_obj, **opf_kwargs) + entry["status"] = edisgo_obj.opf_results.status + entry["solver"] = edisgo_obj.opf_results.solver + entry["solution_time"] = edisgo_obj.opf_results.solution_time + snapshots.append(_snapshot_opf_time_frames(edisgo_obj.opf_results)) + except exceptions.InfeasibleModelError: + entry["status"] = "infeasible" + logger.warning( + f"pm_optimize: OPF infeasible for interval " + f"{interval[0]}..{interval[-1]}." + ) + report.append(entry) + finally: + # restore the full (reduced) index so all intervals' schedules are exposed + # and undo the per-interval mutations of the overlying-grid/reactive input. + _restore_pristine_inputs() + edisgo_obj.set_timeindex(full_timeindex) + + _merge_opf_time_frames(edisgo_obj.opf_results, snapshots) + edisgo_obj.opf_results.interval_results = report + solution_times = [ + e["solution_time"] for e in report if e["solution_time"] is not None + ] + edisgo_obj.opf_results.solution_time = ( + sum(solution_times) if solution_times else None + ) + statuses = [e["status"] for e in report] + infeasible = [e for e in report if e["status"] == "infeasible"] + edisgo_obj.opf_results.status = ( + "infeasible" if infeasible else (statuses[0] if statuses else None) + ) + if infeasible: + raise exceptions.InfeasibleModelError( + f"OPF infeasible for {len(infeasible)} of {len(intervals)} time " + f"intervals; see edisgo.opf_results.interval_results. Results for " + f"feasible intervals have been stored." + ) + + +def _pm_optimize_single( + edisgo_obj, + s_base: int = 1, + flexible_cps: np.ndarray | None = None, + flexible_hps: np.ndarray | None = None, + flexible_loads: np.ndarray | None = None, + flexible_storage_units: np.ndarray | None = None, + opf_version: int = 1, + method: str = "soc", + warm_start: bool = False, + silence_moi: bool = False, +) -> None: + """ + Run a single-interval OPF for the edisgo object in a julia subprocess and + write results back to the edisgo object. Assumes the time index is a single + contiguous interval; :func:`pm_optimize` is the public entry point that + handles non-contiguous indices by calling this per interval. Parameters ---------- diff --git a/edisgo/opf/results/opf_result_class.py b/edisgo/opf/results/opf_result_class.py index f109c681e..83ac1e1a7 100644 --- a/edisgo/opf/results/opf_result_class.py +++ b/edisgo/opf/results/opf_result_class.py @@ -176,6 +176,13 @@ class OPFResults: Aggregated exchange with the overlying grid. battery_storage_t : :class:`~.opf.results.opf_result_class.BatteryStorage` Battery-storage results. + interval_results : list of dict + Per-interval solve report, populated when the OPF is run separately over + several disconnected time intervals (see the ``optimize`` pipeline task, + which splits a non-contiguous time index — e.g. from automatic timestep + selection — into independent optimizations). Each entry has keys + ``start``, ``end``, ``status``, ``solver`` and ``solution_time``. Empty + for a single contiguous optimization. """ @@ -190,6 +197,7 @@ def __init__(self): self.grid_slacks_t = GridSlacks() self.overlying_grid = pd.DataFrame() self.battery_storage_t = BatteryStorage() + self.interval_results = [] def to_csv(self, directory, attributes=None): """ diff --git a/edisgo/run/presets/uc4_example.yaml b/edisgo/run/presets/uc4_example.yaml index 5ec024f6c..02b26ee53 100644 --- a/edisgo/run/presets/uc4_example.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -31,9 +31,9 @@ database: timeindex: {start: "2035-01-01", periods: 24, freq: h} overlying_grid: - enabled: false # master switch — set true to activate import_overlying_grid_data + enabled: true # master switch — set true to activate import_overlying_grid_data source: csv # "csv" (load from path) or "etrago" (consume overlying_grid_data kwarg) - path: "/path/to/overlying_grid_csv_dir" # required when source == csv; full leaf dir for ONE grid (like ding0_path) + path: "/storage/JoDa/edisgo_playground/overlying_grid_data" # required when source == csv; full leaf dir for ONE grid (like ding0_path) results: directory: results/uc4_example @@ -58,7 +58,7 @@ pipeline: - optimize: flexible: [heat_pumps, storage, charging_points, dsm] method: soc - opf_version: 3 + opf_version: 2 - reinforce - save: archive: true diff --git a/edisgo/run/presets/uc5_select_timesteps.yaml b/edisgo/run/presets/uc5_select_timesteps.yaml new file mode 100644 index 000000000..426ce1231 --- /dev/null +++ b/edisgo/run/presets/uc5_select_timesteps.yaml @@ -0,0 +1,117 @@ +_comment: | + UC5 — OPF with configurable timestep selection (manual OR auto): + Like UC4 (full flexibility OPF with HV requirements), but the time index + is reduced to a selected subset instead of a fixed window. The mode is + chosen in the timeseries_selection block below (typically overridden in + the run script). + + The pipeline carries TWO select_timesteps steps, each with a `position`: + - position: pre_import (before import_heat_pumps) — acts only in MANUAL + mode. It sets the explicit time index, which the heat-pump/DSM imports + and oedb_ts then use to fetch only the selected steps (cheap). + - position: post_grid (after import_overlying_grid_data, before + reactive_power) — acts only in AUTO mode. It needs all active-power + time series (incl. overlying-grid generation) set to run the scoring + power flow via get_most_critical_time_intervals. + Whichever mode is configured, the other positioned step is a no-op. + + Auto mode normally yields two disconnected intervals (one overloading, + one voltage). They are kept separate (a gap in the time index); if they + overlap, a non-overlapping pair is chosen if possible, otherwise they are + concatenated into one interval. A later optimize step can detect the gap + and run separate optimizations per interval. + +_workflow: + - setup_grid: load ding0 topology + - import_generators / import_home_batteries + - select_timesteps (pre_import): manual only — set explicit time index + - import_heat_pumps / import_dsm: fetch only selected steps (manual) + - import_electromobility: dumb charging, flex bands + - oedb_ts: real wind/solar + load time series + - apply_charging_strategy / apply_heat_pump_strategy + - build_flexibility_bands: EV bands on the fixed hourly index + - import_overlying_grid_data: HV constraints from CSV dir + - select_timesteps (post_grid): auto only — reduce to critical intervals + - reactive_power: fixed cosphi on the reduced index + - optimize: pm_optimize with flex assets + - reinforce / save + +# Self-contained (no `extends`): everything uc4_example provided is inlined +# below, so this preset can be run on its own via +# run_edisgo({"extends": "uc5_select_timesteps", "grid": {"ding0_path": ...}}) +scenario: eGon2035 + +grid: + ding0_path: "/path/to/ding0_grid" + legacy_ding0_grids: false + +database: + ssh: + enabled: false + +# No explicit base time index is set. oedb_ts falls back to a full year derived +# from the scenario when none is given, which is what auto interval selection +# needs (week-long critical intervals to pick from). For manual selection the +# pre-import select_timesteps step sets the index instead. + +overlying_grid: + enabled: true # set true to activate import_overlying_grid_data + source: csv # "csv" (load from path) or "etrago" (kwarg) + path: "/storage/JoDa/edisgo_playground/overlying_grid_data" + +results: + directory: results/uc5_select_timesteps + +# Top-level block read by the select_timesteps task via ctx.raw_config. +# eGo can inject this block the same way it injects overlying_grid. +# Set `mode` (and its parameters) here or override it in the run script. +timeseries_selection: + mode: auto + # auto method: "power_flow" (default, scores intervals via a power flow) or + # "residual_load" (no power flow — the weeks ending at the max/min residual-load + # time steps; requires overlying-grid data). + method: residual_load + # --- shared auto parameters (both methods) --- + time_steps_per_time_interval: 168 # one week (must be a multiple of 24) + time_step_day_start: 4 # hour of day the intervals start/end on + # --- power_flow method parameters --- + percentage: 1.0 + save_steps: true # write selected intervals CSV to results_dir + use_troubleshooting_mode: true # handle power-flow non-convergence + overloading_factor: 0.95 + voltage_deviation_factor: 0.95 + # --- manual parameters (used when mode: manual) --- + # timestamps: ["2035-01-15 08:00", "2035-01-15 9:00", "2035-01-15 10:00"] + # or a range instead of `timestamps`: + start: "2035-01-15 00:00" + periods: 24 + freq: h + +pipeline: + - setup_grid + - import_generators + - import_home_batteries + - select_timesteps: {position: pre_import} # acts in manual mode only + - import_heat_pumps + - import_dsm + - import_electromobility: + charging_strategy: null + # flexibility bands are built later (build_flexibility_bands), once the + # analysis time index is fixed, so they are resampled to it + - oedb_ts: + dispatchable: {other: 0.7} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - build_flexibility_bands # hourly bands on the 2035 index + - import_overlying_grid_data + - select_timesteps: {position: post_grid} # acts in auto mode only + - reactive_power + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - reinforce: + catch_convergence_problems: true + - save: + archive: true + save_opf_results: true diff --git a/edisgo/run/tasks/__init__.py b/edisgo/run/tasks/__init__.py index 0d59ea02a..9a1be82a8 100644 --- a/edisgo/run/tasks/__init__.py +++ b/edisgo/run/tasks/__init__.py @@ -1,3 +1,13 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later """ Task implementations for the eDisGo pipeline runner. @@ -10,9 +20,9 @@ ``set_timeindex``, ``reactive_power`` * :mod:`.flex` — flex imports (``import_heat_pumps``, ``import_home_batteries``, ``import_dsm``, - ``import_electromobility``, ``import_generators``) and operating - strategies (``apply_charging_strategy``, - ``apply_heat_pump_strategy``) + ``import_electromobility``, ``import_generators``), + ``build_flexibility_bands``, and operating strategies + (``apply_charging_strategy``, ``apply_heat_pump_strategy``) * :mod:`.analysis` — ``check_integrity``, ``analyze``, ``reinforce``, ``base_reinforce``, ``optimize`` * :mod:`.io` — ``save``, ``load_charging_from_files`` @@ -22,4 +32,5 @@ returned value, if non-None, replaces the current one in the runner's loop). """ + from edisgo.run.tasks import analysis, flex, grid, io, timeseries # noqa: F401 diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py index a0d0f4ebd..7bbb4ba3f 100644 --- a/edisgo/run/tasks/analysis.py +++ b/edisgo/run/tasks/analysis.py @@ -1,3 +1,13 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later """ Power-flow, reinforcement, and optimization tasks. @@ -286,12 +296,18 @@ def task_optimize( Explicit ``flexible_*`` kwargs override the shortcut. + This task only performs the ``flexible`` shortcut expansion (mode selection) + and calls :meth:`EDisGo.pm_optimize`. Handling of a non-contiguous (reduced) + time index — running a separate OPF per contiguous interval and merging the + results — lives in :func:`~.opf.powermodels_opf.pm_optimize`. + Parameters ---------- edisgo : edisgo.EDisGo EDisGo instance to optimize. ctx : RunContext - Run context (unused). + Run context. Used for logging and, for multi-interval runs, nothing + else is required from it. flexible : list of str, optional High-level selector, subset of ``{"heat_pumps", "charging_points", "storage"}``. If ``None``, nothing is @@ -343,6 +359,8 @@ def task_optimize( if flexible_storage_units is None: flexible_storage_units = [] + # pm_optimize handles a non-contiguous (reduced) time index internally: + # it runs one OPF per contiguous interval and merges the results. edisgo.pm_optimize( flexible_cps=flexible_cps, flexible_hps=flexible_hps, diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py index 815b28daa..35cce7dce 100644 --- a/edisgo/run/tasks/flex.py +++ b/edisgo/run/tasks/flex.py @@ -1,3 +1,13 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later """ Flex-asset import and operation-strategy tasks. @@ -8,6 +18,7 @@ and typically BEFORE the time-series step, so the time series can cover the new assets. """ + from __future__ import annotations from edisgo.run.registry import register_task @@ -30,7 +41,10 @@ def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): Subset of ``["individual_heat_pumps", "central_heat_pumps"]``; default imports both. timeindex : pandas.DatetimeIndex, optional - Restrict COP / heat-demand time series to this index. + Restrict COP / heat-demand time series to this index. If None, + falls back to ``ctx.flags['selected_timeindex']`` (set by a + preceding ``select_timesteps`` manual step) so the download is + restricted to the selected steps. Returns ------- @@ -38,17 +52,18 @@ def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): The modified EDisGo instance. """ + if timeindex is None: + timeindex = ctx.flags.get("selected_timeindex") edisgo.import_heat_pumps( scenario=ctx.scenario, engine=ctx.ensure_engine(), timeindex=timeindex, import_types=import_types, ) - ctx.flags["has_heat_pumps"] = len( - edisgo.topology.loads_df.loc[ - edisgo.topology.loads_df.type == "heat_pump" - ] - ) > 0 + ctx.flags["has_heat_pumps"] = ( + len(edisgo.topology.loads_df.loc[edisgo.topology.loads_df.type == "heat_pump"]) + > 0 + ) return edisgo @@ -72,12 +87,8 @@ def task_import_home_batteries(edisgo, ctx): The modified EDisGo instance. """ - edisgo.import_home_batteries( - scenario=ctx.scenario, engine=ctx.ensure_engine() - ) - ctx.flags["has_home_batteries"] = ( - not edisgo.topology.storage_units_df.empty - ) + edisgo.import_home_batteries(scenario=ctx.scenario, engine=ctx.ensure_engine()) + ctx.flags["has_home_batteries"] = not edisgo.topology.storage_units_df.empty return edisgo @@ -94,7 +105,10 @@ def task_import_dsm(edisgo, ctx, *, timeindex=None): Run context. Uses ``ctx.scenario`` and ``ctx.ensure_engine()``. Sets ``ctx.flags['has_dsm']``. timeindex : pandas.DatetimeIndex, optional - Restrict DSM availability time series to this index. + Restrict DSM availability time series to this index. If None, + falls back to ``ctx.flags['selected_timeindex']`` (set by a + preceding ``select_timesteps`` manual step) so the download is + restricted to the selected steps. Returns ------- @@ -102,23 +116,28 @@ def task_import_dsm(edisgo, ctx, *, timeindex=None): The modified EDisGo instance. """ + if timeindex is None: + timeindex = ctx.flags.get("selected_timeindex") edisgo.import_dsm( scenario=ctx.scenario, engine=ctx.ensure_engine(), timeindex=timeindex, ) - ctx.flags["has_dsm"] = ( - edisgo.dsm.p_max is not None and not edisgo.dsm.p_max.empty - ) + ctx.flags["has_dsm"] = edisgo.dsm.p_max is not None and not edisgo.dsm.p_max.empty return edisgo @register_task("import_electromobility", requires={"grid"}, provides={"flex"}) -def task_import_electromobility(edisgo, ctx, *, data_source="oedb", - charging_strategy="dumb", - flexibility_bands_ucs = None, - import_electromobility_data_kwds=None, - allocate_charging_demand_kwds=None): +def task_import_electromobility( + edisgo, + ctx, + *, + data_source="oedb", + charging_strategy="dumb", + flexibility_bands_ucs=None, + import_electromobility_data_kwds=None, + allocate_charging_demand_kwds=None, +): """ Import electromobility data (charging processes + parks). @@ -148,7 +167,11 @@ def task_import_electromobility(edisgo, ctx, *, data_source="oedb", and charging-strategy application. Valid entries: ``"home"``, ``"work"``, ``"public"``, ``"hpc"``. Pass a single string for one use case or a list for multiple. ``None`` - (default) skips flexibility-band computation. + (default) skips flexibility-band computation — build them later + with the standalone :func:`task_build_flexibility_bands` once the + analysis time index is fixed, so the bands are resampled to it + (mirrors heat-pump handling, where the HP time series are not set + inside ``import_heat_pumps``). import_electromobility_data_kwds : dict, optional Extra kwargs passed through to the underlying importer. allocate_charging_demand_kwds : dict, optional @@ -178,9 +201,47 @@ def task_import_electromobility(edisgo, ctx, *, data_source="oedb", return edisgo +@register_task("build_flexibility_bands", requires={"flex"}) +def task_build_flexibility_bands(edisgo, ctx, *, use_case=None): + """ + Build EV charging flexibility bands from imported electromobility data. + + Standalone variant of the band computation that + :func:`task_import_electromobility` can do inline. Running it as a + separate step lets it execute *after* the analysis time index is fixed + (e.g. after ``oedb_ts`` / timestep selection), so + :meth:`Electromobility.get_flexibility_bands` resamples the bands to the + edisgo time-series frequency instead of leaving them at the raw SimBEV + resolution. This mirrors how the heat-pump time series are set outside + ``import_heat_pumps``, and is more efficient than building bands over a + non-final index. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + use_case : str or list of str, optional + Charging-point use case(s) to compute bands for. Valid entries: + ``"home"``, ``"work"``, ``"public"``, ``"hpc"``. Defaults to all + four. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + """ + if use_case is None: + use_case = ["home", "work", "public", "hpc"] + edisgo.electromobility.get_flexibility_bands(edisgo, use_case=use_case) + return edisgo + + @register_task("apply_charging_strategy") -def task_apply_charging_strategy(edisgo, ctx, *, strategy="dumb", - charging_park_ids=None): +def task_apply_charging_strategy( + edisgo, ctx, *, strategy="dumb", charging_park_ids=None +): """ Apply a charging strategy to the already-imported EV fleet. @@ -212,8 +273,9 @@ def task_apply_charging_strategy(edisgo, ctx, *, strategy="dumb", @register_task("apply_heat_pump_strategy") -def task_apply_heat_pump_strategy(edisgo, ctx, *, strategy="uncontrolled", - heat_pump_names=None): +def task_apply_heat_pump_strategy( + edisgo, ctx, *, strategy="uncontrolled", heat_pump_names=None +): """ Apply a heat-pump operating strategy. @@ -239,10 +301,7 @@ def task_apply_heat_pump_strategy(edisgo, ctx, *, strategy="uncontrolled", """ if not ctx.flags.get("has_heat_pumps"): - ctx.logger.info( - "Skipping 'apply_heat_pump_strategy': no heat pumps " - "present." - ) + ctx.logger.info("Skipping 'apply_heat_pump_strategy': no heat pumps present.") return edisgo edisgo.apply_heat_pump_operating_strategy( strategy=strategy, heat_pump_names=heat_pump_names diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py index d4a6c8a30..bbe776398 100644 --- a/edisgo/run/tasks/timeseries.py +++ b/edisgo/run/tasks/timeseries.py @@ -1,3 +1,13 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later """ Time-series tasks — set active/reactive power profiles on EDisGo. @@ -8,7 +18,10 @@ 1. Set the time index and active-power profiles with one of :func:`task_worst_case_ts`, :func:`task_oedb_ts`, :func:`task_manual_ts`, possibly :func:`task_set_timeindex`. -2. Finally call :func:`task_reactive_power` to fix reactive power +2. Optionally reduce the time index to a selected subset with + :func:`task_select_timesteps` (manual before the imports, auto after + ``import_overlying_grid_data``). +3. Finally call :func:`task_reactive_power` to fix reactive power control — this MUST come last because it overwrites whatever reactive power was set by the earlier steps. """ @@ -181,6 +194,20 @@ def task_oedb_ts( freq=timeindex.get("freq", "h"), ) edisgo.set_timeindex(ti_df) + elif edisgo.timeseries.timeindex.empty: + # No explicit timeindex and none set yet (e.g. no manual time series + # earlier): fall back to a full year derived from the scenario, the + # same default the flex imports use. + from edisgo.tools.tools import get_year_based_on_scenario + + year = get_year_based_on_scenario(ctx.scenario) + if year is None: + raise ValueError( + f"Cannot derive a default time index: invalid scenario " + f"{ctx.scenario!r}. Provide a 'timeindex' or a valid scenario " + f"('eGon2035', 'eGon100RE')." + ) + edisgo.set_timeindex(pd.date_range(f"1/1/{year}", periods=8760, freq="h")) dispatchable_df = None if dispatchable is not None: @@ -277,6 +304,283 @@ def _as_df(obj): return edisgo +def _set_default_full_year_timeindex(edisgo, ctx): + """ + Set a full-year hourly time index derived from the scenario. + + Used as a fallback so time-index-dependent imports (notably the EV + flexibility bands built in ``import_electromobility``) run on an hourly, + full-year index rather than their raw source resolution. The year is only a + label — the DB imports fetch scenario-correct data regardless — and the index + can be overridden later (e.g. by ``oedb_ts`` or the auto ``select_timesteps`` + step). + """ + from edisgo.tools.tools import get_year_based_on_scenario + + year = get_year_based_on_scenario(ctx.scenario) or 2011 + edisgo.set_timeindex(pd.date_range(f"1/1/{year}", periods=8760, freq="h")) + ctx.logger.info( + f"select_timesteps: no time index set; using default full year " + f"{year} (8760 h) so imports build hourly full-year data." + ) + + +@register_task("select_timesteps", provides={"timeseries"}, ts_altering=True) +def task_select_timesteps(edisgo, ctx, **overrides): + """ + Select the time steps the grid is analyzed/optimized for. + + Reduces the time index to a configurable subset. Configuration is + read from the top-level ``timeseries_selection:`` config block (so + eGo can inject it the same way it injects ``overlying_grid``); + inline step params override individual keys of that block. Two + modes: + + ``manual`` + Reduce to an explicit set of time steps. Positioned *before* + the data-import tasks so ``import_heat_pumps`` / ``import_dsm`` + download only the requested steps. The selected index is + stashed in ``ctx.flags['selected_timeindex']`` for those + imports to pick up. + + ``auto`` + Determine the two most critical time intervals and reduce to + them. Must be positioned *after* ``import_overlying_grid_data`` + (needs all active-power time series) and *before* + ``reactive_power``. Two ``method`` options: + + * ``power_flow`` (default) — score intervals via a power flow + (:func:`~.tools.temporal_complexity_reduction.get_most_critical_time_intervals`). + A reactive-power series is set internally to run the scoring + power flow, but ``ctx.flags['reactive_power_set']`` is left + unset so the pipeline's own ``reactive_power`` step still runs + on the reduced index. + * ``residual_load`` — no power flow. The overlying-grid dispatch + is distributed onto the components and the residual load is + ranked over the whole year; intervals are centered on the + highest (load case) and lowest (feed-in case) residual-load + steps. Requires overlying-grid data to be present. + + Both methods delegate to + :func:`~.tools.temporal_complexity_reduction.get_most_critical_time_intervals` + (via its ``by`` parameter) and reduce to a non-overlapping pair + chosen by + :func:`~.tools.temporal_complexity_reduction.select_two_intervals`. + + The auto mode normally yields two disconnected intervals (one for + overloading, one for voltage issues). These are kept separate in the + resulting time index (there is a gap between them). If they overlap, + a non-overlapping pair is chosen if possible, otherwise they are + concatenated into one interval. The intervals themselves are not + stored — a later ``optimize`` step can detect the gap in the time + index and run separate optimizations per interval. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Reads ``ctx.raw_config['timeseries_selection']``. + Sets ``ctx.flags['selected_timeindex']`` (manual) and + ``ctx.flags['timesteps_selected'] = True``. + **overrides + Inline step params overriding keys of the + ``timeseries_selection`` block. Recognized keys: + ``position`` (``"pre_import"`` | ``"post_grid"``, optional) — the + step only acts when the configured ``mode`` matches this + position (``pre_import`` ↔ ``manual``, ``post_grid`` ↔ ``auto``), + otherwise it is a no-op; this lets one pipeline carry both a + pre-import and a post-grid ``select_timesteps`` step and support + either mode via config. When omitted, the step always acts. + ``mode`` (``"manual"`` | ``"auto"``); + for manual: ``timestamps`` (list) or ``start`` / + ``periods`` / ``end`` / ``freq``; + for auto: ``method`` (``"power_flow"`` (default) | + ``"residual_load"``), ``time_steps_per_time_interval``; + for ``method="power_flow"`` additionally ``percentage``, + ``time_step_day_start`` (default 4), ``save_steps`` (default + True; CSV written to ``ctx.results_dir``), + ``use_troubleshooting_mode``, ``overloading_factor``, + ``voltage_deviation_factor``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + Raises + ------ + ValueError + If ``mode`` is missing/unknown, if manual mode has neither + ``timestamps`` nor a range, or if auto mode runs before active + power time series are set. + """ + from edisgo.tools.temporal_complexity_reduction import ( + get_most_critical_time_intervals, + select_two_intervals, + ) + from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex + + cfg = {**ctx.raw_config.get("timeseries_selection", {}), **overrides} + mode = cfg.get("mode") + if mode not in ("manual", "auto"): + raise ValueError( + f"select_timesteps needs mode 'manual' or 'auto', got {mode!r}." + ) + + # A pipeline may include two select_timesteps steps — one before the + # imports (``position: pre_import``, where manual selection belongs) and + # one after import_overlying_grid_data (``position: post_grid``, where auto + # selection belongs) — so the same preset supports both modes. Each step + # only acts when the configured mode matches its position; otherwise it is + # a no-op. When ``position`` is omitted (single-step usage) the step always + # acts. + position = overrides.get("position") + expected_mode = {"pre_import": "manual", "post_grid": "auto"} + if position is not None: + if position not in expected_mode: + raise ValueError( + f"select_timesteps 'position' must be 'pre_import' or " + f"'post_grid', got {position!r}." + ) + if mode != expected_mode[position]: + # This positioned step is not the active selector. If it is the + # pre-import step and no time index has been set yet (i.e. manual + # selection is not driving the index), establish a full-year default + # so the following imports build their time-index-dependent data + # (e.g. EV flexibility bands) on an hourly full-year index. Only a + # label — DB imports fetch scenario-correct data regardless — and it + # is overridden later by oedb_ts / the auto select_timesteps step. + if position == "pre_import" and edisgo.timeseries.timeindex.empty: + _set_default_full_year_timeindex(edisgo, ctx) + ctx.logger.debug( + f"select_timesteps at position {position!r} is a no-op for " + f"mode {mode!r}." + ) + return edisgo + + if mode == "manual": + timestamps = cfg.get("timestamps") + if timestamps is not None: + timeindex = pd.DatetimeIndex(pd.to_datetime(list(timestamps))) + elif cfg.get("end") is not None: + timeindex = pd.date_range( + start=cfg["start"], end=cfg["end"], freq=cfg.get("freq", "h") + ) + elif cfg.get("periods") is not None: + timeindex = pd.date_range( + start=cfg["start"], + periods=cfg["periods"], + freq=cfg.get("freq", "h"), + ) + else: + raise ValueError( + "select_timesteps manual mode needs 'timestamps' or a " + "'start' plus 'periods'/'end' range." + ) + timeindex = timeindex.sort_values().unique() + if not edisgo.timeseries.timeindex.empty: + # A time index is already set (manual selection reducing an existing + # full time series): align the user-supplied timestamps to that + # index's year so date-based slicing matches even if the user wrote + # them in a different (e.g. scenario) year than the internally used + # reference year. + year_diff = edisgo.timeseries.timeindex[0].year - timeindex[0].year + if year_diff != 0: + timeindex = timeindex + pd.DateOffset(years=year_diff) + ctx.flags["selected_timeindex"] = timeindex + if edisgo.timeseries.timeindex.empty: + # positioned before imports: just set the index so HP/DSM + # imports restrict their downloads to it + edisgo.set_timeindex(timeindex) + else: + reduce_timeseries_data_to_given_timeindex(edisgo, timeindex) + ctx.logger.info( + f"select_timesteps (manual): selected {len(timeindex)} time steps." + ) + ctx.flags["timesteps_selected"] = True + return edisgo + + # auto mode + if not ctx.flags.get("timeseries_set"): + raise ValueError( + "select_timesteps mode 'auto' needs active-power time series to " + "be set first (e.g. run oedb_ts before it)." + ) + + method = cfg.get("method", "power_flow") + if method not in ("power_flow", "residual_load"): + raise ValueError( + f"select_timesteps auto 'method' must be 'power_flow' or " + f"'residual_load', got {method!r}." + ) + tsp = cfg.get("time_steps_per_time_interval", 168) + + if method == "residual_load": + # residual-load selection requires overlying-grid data (the dispatch + # distributed onto the components); guard here (mode selection) before + # delegating the computation to the tools function. + og = edisgo.overlying_grid + if all( + s.empty + for s in ( + og.electromobility_active_power, + og.storage_units_active_power, + og.heat_pump_central_active_power, + og.heat_pump_decentral_active_power, + og.dsm_active_power, + og.renewables_curtailment, + ) + ): + raise ValueError( + "select_timesteps method 'residual_load' needs overlying-grid " + "data to be present (run import_overlying_grid_data before it)." + ) + col_a, col_b = "time_steps_load_case", "time_steps_feedin_case" + else: # power_flow + # throwaway reactive power so the scoring power flow yields meaningful + # voltages; do NOT mark reactive_power_set — the pipeline's own + # reactive_power step runs afterwards on the reduced index. + edisgo.set_time_series_reactive_power_control(control="fixed_cosphi") + col_a, col_b = "time_steps_overloading", "time_steps_voltage_issues" + + intervals_df = get_most_critical_time_intervals( + edisgo, + by=method, + percentage=cfg.get("percentage", 1.0), + time_steps_per_time_interval=tsp, + time_step_day_start=cfg.get("time_step_day_start", 4), + save_steps=cfg.get("save_steps", True), + path=str(ctx.results_dir) if ctx.results_dir is not None else "", + use_troubleshooting_mode=cfg.get("use_troubleshooting_mode", True), + overloading_factor=cfg.get("overloading_factor", 0.95), + voltage_deviation_factor=cfg.get("voltage_deviation_factor", 0.95), + ) + intervals = select_two_intervals( + list(intervals_df.get(col_a, [])), + list(intervals_df.get(col_b, [])), + ) + + if not intervals: + raise ValueError( + "select_timesteps mode 'auto' found no critical time intervals; " + "cannot reduce the time index." + ) + + timeindex = intervals[0] + for interval in intervals[1:]: + timeindex = timeindex.union(interval) + timeindex = timeindex.sort_values() + + reduce_timeseries_data_to_given_timeindex(edisgo, timeindex) + ctx.logger.info( + f"select_timesteps (auto): selected {len(intervals)} interval(s), " + f"{len(timeindex)} time steps total." + ) + ctx.flags["timesteps_selected"] = True + return edisgo + + @register_task("reactive_power") def task_reactive_power( edisgo, diff --git a/edisgo/tools/temporal_complexity_reduction.py b/edisgo/tools/temporal_complexity_reduction.py index 2adc269f9..df4bb0730 100644 --- a/edisgo/tools/temporal_complexity_reduction.py +++ b/edisgo/tools/temporal_complexity_reduction.py @@ -647,6 +647,189 @@ def _troubleshooting_mode( return edisgo_obj +def intervals_overlap(a, b): + """ + Return True if two contiguous time-step intervals overlap. + + Each interval is a :pandas:`pandas.DatetimeIndex` of + contiguous, sorted time steps. Overlap is checked on the closed + ``[min, max]`` ranges, so intervals that merely touch (share an end point) + count as overlapping. + """ + return (a.min() <= b.max()) and (b.min() <= a.max()) + + +def select_two_intervals(load_case, feedin_case): + """ + Pick the time intervals to analyze from two ranked candidate lists. + + Used to reduce the ranked most-critical intervals (e.g. from + :func:`get_most_critical_time_intervals`) to the intervals actually analyzed: + + * Start from the most critical interval of each list (index 0). + * If they do not overlap, both are kept — two disconnected intervals are + returned (a later optimization can detect the gap and optimize each + interval separately). + * If they overlap, walk down the second list to the highest ranked interval + that does not overlap the top interval of the first list, and keep that + pair instead. + * If no non-overlapping pair exists, the two most critical intervals are + concatenated into a single contiguous interval and only that one is + returned. + + Parameters + ---------- + load_case : list of pandas.DatetimeIndex + First ranked list of intervals (most critical first). May be empty. + feedin_case : list of pandas.DatetimeIndex + Second ranked list of intervals (most critical first). May be empty. + + Returns + ------- + list of pandas.DatetimeIndex + One or two contiguous, non-overlapping intervals. Empty if both input + lists are empty. + """ + if not load_case and not feedin_case: + return [] + if not load_case: + return [feedin_case[0]] + if not feedin_case: + return [load_case[0]] + + top = load_case[0] + for cand in feedin_case: + if not intervals_overlap(top, cand): + return [top, cand] + + # no non-overlapping pair -> concatenate the two most critical intervals + merged = top.union(feedin_case[0]).sort_values() + start, end = merged.min(), merged.max() + freq = pd.infer_freq(top) or "H" + return [pd.date_range(start=start, end=end, freq=freq)] + + +def _build_centered_interval( + timestep, timeindex, time_steps_per_time_interval, time_step_day_start +): + """ + Build a contiguous interval centered on a critical time step. + + The interval has ``time_steps_per_time_interval`` steps, is centered on + ``timestep``, and its start is snapped to the ``time_step_day_start`` hour of + day (so intervals begin on that hour). The interval is clipped to lie within + ``timeindex``; if centering would run past either end, it is shifted inward. + Centering guarantees the critical step is not the last step of the interval + (where a storage state-of-charge-end constraint would force zero power). + + Parameters + ---------- + timestep : pandas.Timestamp + Critical time step to center on. + timeindex : pandas.DatetimeIndex + The full (sorted) time index the interval must lie within. + time_steps_per_time_interval : int + Interval length in steps. + time_step_day_start : int + Hour of day the interval should start on. + + Returns + ------- + pandas.DatetimeIndex + The contiguous interval. + """ + n = int(time_steps_per_time_interval) + step = timeindex[1] - timeindex[0] + half = (n // 2) * step + # center on the critical step, then snap the start back to the day-start hour + start = timestep - half + while start.hour != int(time_step_day_start): + start = start - step + end = start + (n - 1) * step + # keep the interval within the available time index; shift inward if needed + if start < timeindex[0]: + start = timeindex[0] + end = start + (n - 1) * step + if end > timeindex[-1]: + end = timeindex[-1] + start = end - (n - 1) * step + if start < timeindex[0]: + start = timeindex[0] + return pd.date_range(start=start, end=end, freq=step) + + +def _most_critical_time_intervals_residual_load( + edisgo_obj, + num_time_intervals=None, + percentage=1.0, + time_steps_per_time_interval=168, + time_step_day_start=0, + save_steps=False, + path="", +): + """ + Determine the most critical time intervals from the residual load. + + Ranks the critical single time steps by residual load (via + :func:`get_most_critical_time_steps` with ``by="residual_load"``) and wraps + each into an interval centered on the step and snapped to the + ``time_step_day_start`` hour (see :func:`_build_centered_interval`). Returns a + DataFrame ranked by residual magnitude with per-case columns + ``time_steps_load_case`` (highest residual) and ``time_steps_feedin_case`` + (lowest residual). Overlaps between intervals are allowed (mirroring the + power-flow interval selection); a downstream :func:`select_two_intervals` + picks a non-overlapping pair. + """ + timeindex = edisgo_obj.timeseries.timeindex + + # number of ranked intervals per case + if num_time_intervals is None: + num_time_intervals = int(np.ceil(len(timeindex) * percentage)) + + from edisgo.network.overlying_grid import ( + distribute_overlying_grid_requirements, + ) + + distributed = distribute_overlying_grid_requirements(edisgo_obj) + residual = distributed.timeseries.residual_load + + load_steps = residual.sort_values(ascending=False).index[:num_time_intervals] + feedin_steps = residual.sort_values(ascending=True).index[:num_time_intervals] + + load_intervals = [ + _build_centered_interval( + t, timeindex, time_steps_per_time_interval, time_step_day_start + ) + for t in load_steps + ] + feedin_intervals = [ + _build_centered_interval( + t, timeindex, time_steps_per_time_interval, time_step_day_start + ) + for t in feedin_steps + ] + + steps = pd.DataFrame( + { + "time_steps_load_case": load_intervals, + "time_steps_feedin_case": feedin_intervals, + } + ) + if len(steps) == 0: + logger.info("No critical steps detected. No network expansion required.") + + if save_steps: + abs_path = os.path.abspath(path) + steps.to_csv( + os.path.join( + abs_path, + f"{edisgo_obj.topology.id}_t_{time_steps_per_time_interval}" + f"_residual_load.csv", + ) + ) + return steps + + def get_most_critical_time_intervals( edisgo_obj, num_time_intervals=None, @@ -659,6 +842,7 @@ def get_most_critical_time_intervals( overloading_factor=0.95, voltage_deviation_factor=0.95, weight_by_costs=True, + by="power_flow", ): """ Get time intervals sorted by severity of overloadings as well as voltage issues. @@ -747,15 +931,32 @@ def get_most_critical_time_intervals( time intervals. Default: True. + by : str + Criticality measure used to determine the intervals. Options: + + * "power_flow" (default): run a power flow and score rolling windows by + overloading and voltage violations. Returns columns + ``time_steps_overloading`` / ``time_steps_voltage_issues``. + * "residual_load": no power flow — rank the critical single steps by + residual load (see :func:`get_most_critical_time_steps` with + ``by="residual_load"``) and center an interval on each, snapped to the + ``time_step_day_start`` hour. Returns columns ``time_steps_load_case`` + (highest residual) / ``time_steps_feedin_case`` (lowest residual). + Overlaps between intervals are allowed. + + Default: "power_flow". Returns -------- :pandas:`pandas.DataFrame` - Contains time intervals in which grid expansion needs due to overloading and - voltage issues are detected. The time intervals are determined independently - for overloading and voltage issues and sorted descending by the expected - cumulated grid expansion costs, so that the time intervals with the highest - expected costs correspond to index 0. + Contains time intervals in which grid expansion needs are detected, + ranked most-critical first. Column names depend on ``by`` (see above): + ``time_steps_overloading``/``time_steps_voltage_issues`` for + ``power_flow``, ``time_steps_load_case``/``time_steps_feedin_case`` for + ``residual_load``. For ``power_flow`` the intervals are determined + independently for overloading and voltage issues and sorted descending by + the expected cumulated grid expansion costs, so that the time intervals + with the highest expected costs correspond to index 0. In case of overloading, the time steps in the respective time interval are given in column "time_steps_overloading" and the share of components for which the maximum overloading is reached during the time interval is given in column @@ -766,6 +967,28 @@ def get_most_critical_time_intervals( "percentage_buses_max_voltage_deviation". """ + if by not in ("power_flow", "residual_load"): + raise ValueError( + f"get_most_critical_time_intervals: 'by' must be 'power_flow' or " + f"'residual_load', got {by!r}." + ) + + if by == "residual_load": + # No power flow: rank the critical single steps by residual load (via + # get_most_critical_time_steps(by="residual_load")) and wrap each into an + # interval centered on the step and snapped to the time_step_day_start + # block. Returns per-case columns time_steps_load_case / + # time_steps_feedin_case (ranked, overlaps allowed). + return _most_critical_time_intervals_residual_load( + edisgo_obj, + num_time_intervals=num_time_intervals, + percentage=percentage, + time_steps_per_time_interval=time_steps_per_time_interval, + time_step_day_start=time_step_day_start, + save_steps=save_steps, + path=path, + ) + # check frequency of time series data timeindex = edisgo_obj.timeseries.timeindex timedelta = timeindex[1] - timeindex[0] @@ -845,6 +1068,64 @@ def get_most_critical_time_intervals( return steps +def _most_critical_time_steps_residual_load( + edisgo_obj, + num_steps_loading=None, + num_steps_voltage=None, + percentage=1.0, +): + """ + Rank time steps by residual load, without running a power flow. + + Distributes the overlying-grid dispatch onto the grid components (via + :func:`~.network.overlying_grid.distribute_overlying_grid_requirements`) and + evaluates the residual load (load minus generation minus storage) over the + whole time index. The load-case steps are those with the highest residual + load, the feed-in-case steps those with the lowest (most negative). + + Parameters + ---------- + edisgo_obj : :class:`~.EDisGo` + num_steps_loading : int or None + Number of highest-residual (load-case) steps to select. If None, + ``percentage`` of all steps is used. + num_steps_voltage : int or None + Number of lowest-residual (feed-in-case) steps to select. If None, + ``percentage`` of all steps is used. + percentage : float + Fraction of all time steps to select per case when the corresponding + ``num_steps_*`` is None. Default: 1.0. + + Returns + ------- + :pandas:`pandas.DatetimeIndex` + Unique union of the selected load-case and feed-in-case time steps. + """ + from edisgo.network.overlying_grid import ( + distribute_overlying_grid_requirements, + ) + + distributed = distribute_overlying_grid_requirements(edisgo_obj) + residual = distributed.timeseries.residual_load + + n = len(residual) + if num_steps_loading is None: + num_steps_loading = int(n * percentage) + if num_steps_voltage is None: + num_steps_voltage = int(n * percentage) + num_steps_loading = min(num_steps_loading, n) + num_steps_voltage = min(num_steps_voltage, n) + + # highest residual = worst load case; lowest residual = worst feed-in case + load_case = residual.sort_values(ascending=False).index[:num_steps_loading] + feedin_case = residual.sort_values(ascending=True).index[:num_steps_voltage] + + steps = load_case.append(feedin_case) + if len(steps) == 0: + logger.warning("No critical steps detected. No network expansion required.") + return pd.DatetimeIndex(steps.unique()) + + def get_most_critical_time_steps( edisgo_obj: EDisGo, mode=None, @@ -857,6 +1138,7 @@ def get_most_critical_time_steps( use_troubleshooting_mode=True, run_initial_analyze=True, weight_by_costs=True, + by="power_flow", ) -> pd.DatetimeIndex: """ Get the time steps with the most critical overloading and voltage issues. @@ -926,14 +1208,51 @@ def get_most_critical_time_steps( If False, only the relative overloading is used. Default: True. + by : str + Criticality measure used to rank time steps. Options: + + * "power_flow" (default): run a power flow and score steps by overloading + and voltage violations (the parameters `mode`, `timesteps`, + `lv_grid_id`, `scale_timeseries`, `use_troubleshooting_mode`, + `run_initial_analyze`, `weight_by_costs` apply to this measure). + * "residual_load": no power flow — rank steps by the residual load + (load minus generation minus storage) after distributing the + overlying-grid dispatch onto the components. The highest residual + steps are the critical load cases, the lowest (most negative) the + critical feed-in cases. `num_steps_loading` / `num_steps_voltage` / + `percentage` control how many of each are selected; the power-flow + parameters are ignored. + + Default: "power_flow". Returns -------- :pandas:`pandas.DatetimeIndex` Time index with unique time steps where maximum overloading or maximum - voltage deviation is reached for at least one component respectively bus. + voltage deviation is reached for at least one component respectively bus + (``by="power_flow"``), or with the highest/lowest residual load + (``by="residual_load"``). """ + if by not in ("power_flow", "residual_load"): + raise ValueError( + f"get_most_critical_time_steps: 'by' must be 'power_flow' or " + f"'residual_load', got {by!r}." + ) + + if by == "residual_load": + # No power flow needed: rank time steps by the residual load (load minus + # generation minus storage) after distributing the overlying-grid + # dispatch onto the components. The most critical load-case steps have + # the highest residual load, the most critical feed-in-case steps the + # lowest (most negative). Returns the union of both, deduplicated. + return _most_critical_time_steps_residual_load( + edisgo_obj, + num_steps_loading=num_steps_loading, + num_steps_voltage=num_steps_voltage, + percentage=percentage, + ) + # Run power flow if run_initial_analyze: if use_troubleshooting_mode: diff --git a/edisgo/tools/tools.py b/edisgo/tools/tools.py index 8bbce99aa..ead0219e5 100644 --- a/edisgo/tools/tools.py +++ b/edisgo/tools/tools.py @@ -1228,6 +1228,25 @@ def reduce_timeseries_data_to_given_timeindex( ) # Battery electric vehicle timeseries if electromobility: + # The EV flexibility bands are built in import_electromobility from the + # raw SimBEV grid (typically 15-min and in the reference year 2011), + # independently of the analysis time index. Before slicing by datetime, + # align them to the target index: first resample to its frequency + # (Electromobility.resample uses the correct per-band aggregation — + # mean for power, max for energy), then shift the year and reindex via + # align_series_to_timeindex so datetime .loc lookups below succeed. + _bands = edisgo_obj.electromobility.flexibility_bands + _band0 = next((b for b in _bands.values() if not b.empty), None) + if _band0 is not None and len(_band0.index) > 1: + band_freq = _band0.index[1] - _band0.index[0] + if band_freq != frequency: + edisgo_obj.electromobility.resample(freq=frequency) + # year-align every (now correctly-sampled) band onto the timeindex + for key, df in edisgo_obj.electromobility.flexibility_bands.items(): + if not df.empty: + edisgo_obj.electromobility.flexibility_bands[key] = ( + align_series_to_timeindex(df, timeindex) + ) if save_ev_soc_initial: # timestep EV SOC from timestep before if possible ts_before = timeindex[0] - frequency @@ -1423,7 +1442,7 @@ def reduce_memory_usage(df: pd.DataFrame, show_reduction: bool = False) -> pd.Da for col in df.columns: col_type = df[col].dtype - if col_type != object and str(col_type) != "category": + if not pd.api.types.is_object_dtype(col_type) and str(col_type) != "category": c_min = df[col].min() c_max = df[col].max() diff --git a/run_example_05.py b/run_example_05.py new file mode 100644 index 000000000..552082d3d --- /dev/null +++ b/run_example_05.py @@ -0,0 +1,39 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Runner für uc5_select_timesteps.yaml — einfach ``python run_example_05.py``.""" + +import logging + +from edisgo.run.runner import run_edisgo + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(name)s %(levelname)s: %(message)s", +) + +# edisgo = run_edisgo("/storage/JoDa/ego/edisgo_run_edisgo/eDisGo/edisgo/run/presets/uc4_example_MS.yaml") # noqa: E501 +edisgo = run_edisgo( + { + "extends": "uc5_select_timesteps.yaml", + # "grid": {"ding0_path": "/home/gurobi/.ding0/run_hetzner_59763_2023_04_06/ding0_grids/32355"} # noqa: E501 + "grid": { + "ding0_path": "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" # noqa: E501 + }, + # OG path must be the leaf dir for THIS grid (like ding0_path), not the parent. + "overlying_grid": { + "path": "/storage/JoDa/edisgo_playground/overlying_grid_data/32377" + }, + } +) + +print("\n=== Fertig ===") +print("Ausbaukosten:\n", edisgo.results.grid_expansion_costs) +print("\nUngelöste Probleme:\n", edisgo.results.unresolved_issues) diff --git a/tests/opf/test_powermodels_opf.py b/tests/opf/test_powermodels_opf.py index c79cacb06..9b4300e6d 100644 --- a/tests/opf/test_powermodels_opf.py +++ b/tests/opf/test_powermodels_opf.py @@ -339,3 +339,194 @@ def test_pm_optimize(self): ) ) ) + + +class TestContiguousIntervals: + def test_contiguous_index_is_one_interval(self): + from edisgo.opf.powermodels_opf import _contiguous_intervals + + ti = pd.date_range("2035-01-01", periods=48, freq="h") + result = _contiguous_intervals(ti) + assert len(result) == 1 and result[0].equals(ti) + + def test_two_disconnected_intervals(self): + from edisgo.opf.powermodels_opf import _contiguous_intervals + + a = pd.date_range("2035-01-01", periods=24, freq="h") + b = pd.date_range("2035-07-01", periods=24, freq="h") + result = _contiguous_intervals(a.union(b)) + assert len(result) == 2 + assert result[0].equals(a) and result[1].equals(b) + + def test_freq_restored_on_freqless_index(self): + from edisgo.opf.powermodels_opf import _contiguous_intervals + + idx = pd.DatetimeIndex(pd.date_range("2035-01-01", periods=48, freq="h").values) + assert idx.freq is None + result = _contiguous_intervals(idx) + assert len(result) == 1 and result[0].freq is not None + + def test_single_and_empty(self): + from edisgo.opf.powermodels_opf import _contiguous_intervals + + assert len(_contiguous_intervals(pd.date_range("2035-01-01", periods=1))) == 1 + assert _contiguous_intervals(pd.DatetimeIndex([])) == [] + + +class TestMergeOpfTimeFrames: + @staticmethod + def _empty_snapshot(opf, slack_generator_t): + return { + "slack_generator_t": slack_generator_t, + "hv_requirement_slacks_t": pd.DataFrame(), + "lines_t": {k: pd.DataFrame() for k in opf.lines_t._attributes()}, + "heat_storage_t": { + k: pd.DataFrame() for k in opf.heat_storage_t._attributes() + }, + "grid_slacks_t": { + k: pd.DataFrame() for k in opf.grid_slacks_t._attributes() + }, + "battery_storage_t": { + k: pd.DataFrame() for k in opf.battery_storage_t._attributes() + }, + } + + def test_flat_frame_concatenated_and_sorted(self): + from edisgo.opf.powermodels_opf import _merge_opf_time_frames + from edisgo.opf.results.opf_result_class import OPFResults + + opf = OPFResults() + a = pd.DataFrame({"x": [1.0]}, index=pd.date_range("2035-01-01", periods=1)) + b = pd.DataFrame({"x": [2.0]}, index=pd.date_range("2035-07-01", periods=1)) + _merge_opf_time_frames( + opf, [self._empty_snapshot(opf, a), self._empty_snapshot(opf, b)] + ) + assert len(opf.slack_generator_t) == 2 + assert list(opf.slack_generator_t["x"]) == [1.0, 2.0] + + +class TestPmOptimizeIntervalSplit: + """pm_optimize's multi-interval split, with the single-interval OPF stubbed + (no Julia/DB). Patches powermodels_opf._pm_optimize_single.""" + + @pytest.fixture + def edisgo_obj(self): + e = EDisGo(ding0_grid=pytest.ding0_test_network_path) + e.set_timeindex(pd.date_range("2035-01-01", periods=24, freq="h")) + return e + + def test_single_interval_calls_once_with_freq(self, edisgo_obj, monkeypatch): + import edisgo.opf.powermodels_opf as pmo + + ti = pd.DatetimeIndex(pd.date_range("2035-01-01", periods=24, freq="h").values) + assert ti.freq is None + edisgo_obj.set_timeindex(ti) + seen = [] + monkeypatch.setattr( + pmo, + "_pm_optimize_single", + lambda e, **kw: seen.append(e.timeseries.timeindex.freq), + ) + pmo.pm_optimize(edisgo_obj) + assert seen == [pd.tseries.frequencies.to_offset("h")] + + def test_two_intervals_run_separately_and_restore(self, edisgo_obj, monkeypatch): + import edisgo.opf.powermodels_opf as pmo + + a = pd.date_range("2035-01-01", periods=24, freq="h") + b = pd.date_range("2035-07-01", periods=24, freq="h") + full = a.union(b) + edisgo_obj.set_timeindex(full) + seen = [] + + def fake_single(e, **kw): + seen.append(e.timeseries.timeindex) + e.opf_results.status = "OPTIMAL" + e.opf_results.solver = "Gurobi" + e.opf_results.solution_time = 1.0 + + monkeypatch.setattr(pmo, "_pm_optimize_single", fake_single) + pmo.pm_optimize(edisgo_obj) + assert len(seen) == 2 and seen[0].equals(a) and seen[1].equals(b) + assert edisgo_obj.timeseries.timeindex.equals(full) + assert len(edisgo_obj.opf_results.interval_results) == 2 + assert edisgo_obj.opf_results.solution_time == 2.0 + assert edisgo_obj.opf_results.status == "OPTIMAL" + + def test_overlying_grid_state_restored(self, edisgo_obj, monkeypatch): + import edisgo.opf.powermodels_opf as pmo + + a = pd.date_range("2035-01-01", periods=24, freq="h") + b = pd.date_range("2035-07-01", periods=24, freq="h") + full = a.union(b) + edisgo_obj.set_timeindex(full) + edisgo_obj.overlying_grid.storage_units_soc = pd.Series(1.0, index=full) + seen_types = [] + + def fake_single(e, **kw): + og = e.overlying_grid + seen_types.append(type(og.storage_units_soc).__name__) + og.storage_units_soc = pd.DataFrame( + 0.0, index=e.timeseries.timeindex, columns=["s1"] + ) + e.opf_results.status = "OPTIMAL" + + monkeypatch.setattr(pmo, "_pm_optimize_single", fake_single) + pmo.pm_optimize(edisgo_obj) + assert seen_types == ["Series", "Series"] + assert isinstance(edisgo_obj.overlying_grid.storage_units_soc, pd.Series) + + def test_reactive_power_restored(self, edisgo_obj, monkeypatch): + import edisgo.opf.powermodels_opf as pmo + + a = pd.date_range("2035-01-01", periods=24, freq="h") + b = pd.date_range("2035-07-01", periods=24, freq="h") + full = a.union(b) + edisgo_obj.set_timeindex(full) + gen = edisgo_obj.topology.generators_df.index[0] + edisgo_obj.timeseries._generators_reactive_power = pd.DataFrame( + 0.0, index=full, columns=[gen] + ) + seen_ok = [] + + def fake_single(e, **kw): + ti = e.timeseries.timeindex + q = e.timeseries.generators_reactive_power + seen_ok.append(not q.empty and q.index.equals(ti)) + e.timeseries._generators_reactive_power = pd.DataFrame( + 0.0, index=ti, columns=[gen] + ) + e.opf_results.status = "OPTIMAL" + + monkeypatch.setattr(pmo, "_pm_optimize_single", fake_single) + pmo.pm_optimize(edisgo_obj) + assert seen_ok == [True, True] + assert edisgo_obj.timeseries._generators_reactive_power.index.equals(full) + + def test_infeasible_interval_stores_report_and_raises( + self, edisgo_obj, monkeypatch + ): + import edisgo.opf.powermodels_opf as pmo + + from edisgo.flex_opt.exceptions import InfeasibleModelError + + a = pd.date_range("2035-01-01", periods=24, freq="h") + b = pd.date_range("2035-07-01", periods=24, freq="h") + full = a.union(b) + edisgo_obj.set_timeindex(full) + + def fake_single(e, **kw): + if e.timeseries.timeindex[0] == a[0]: + e.opf_results.status = "OPTIMAL" + e.opf_results.solution_time = 1.0 + else: + raise InfeasibleModelError("stub") + + monkeypatch.setattr(pmo, "_pm_optimize_single", fake_single) + with pytest.raises(InfeasibleModelError): + pmo.pm_optimize(edisgo_obj) + report = edisgo_obj.opf_results.interval_results + assert len(report) == 2 + assert report[0]["status"] == "OPTIMAL" + assert report[1]["status"] == "infeasible" + assert edisgo_obj.timeseries.timeindex.equals(full) diff --git a/tests/run/test_tasks.py b/tests/run/test_tasks.py index c6c12631e..1f1db2237 100644 --- a/tests/run/test_tasks.py +++ b/tests/run/test_tasks.py @@ -7,6 +7,7 @@ enough, and the DB-free branches of ``import_overlying_grid_data`` are exercised directly. """ + import glob import os @@ -18,9 +19,17 @@ from edisgo.edisgo import EDisGo from edisgo.run.config import load_config from edisgo.run.context import RunContext +from edisgo.run.tasks.analysis import task_optimize from edisgo.run.tasks.io import task_import_overlying_grid_data -from edisgo.run.tasks.timeseries import task_manual_ts +from edisgo.run.tasks.timeseries import ( + task_manual_ts, + task_select_timesteps, +) from edisgo.run.validator import validate +from edisgo.tools.temporal_complexity_reduction import ( + intervals_overlap, + select_two_intervals, +) @pytest.fixture @@ -69,8 +78,7 @@ def test_unknown_source_warns(self, edisgo_obj, caplog): assert "unknown source" in caplog.text def test_etrago_without_data_warns(self, edisgo_obj, caplog): - ctx = self._ctx({"enabled": True, "source": "etrago"}, - overlying_grid_data=None) + ctx = self._ctx({"enabled": True, "source": "etrago"}, overlying_grid_data=None) result = task_import_overlying_grid_data(edisgo_obj, ctx) assert result is edisgo_obj assert "no" in caplog.text.lower() @@ -80,8 +88,7 @@ def test_etrago_empty_data_does_not_crash(self, edisgo_obj): A partial/empty etrago dict must not raise (regression: the task used to call .empty on dict.get() results that were None). """ - ctx = self._ctx({"enabled": True, "source": "etrago"}, - overlying_grid_data={}) + ctx = self._ctx({"enabled": True, "source": "etrago"}, overlying_grid_data={}) # must simply return without AttributeError assert task_import_overlying_grid_data(edisgo_obj, ctx) is edisgo_obj @@ -92,6 +99,236 @@ def test_csv_without_path_warns(self, edisgo_obj, caplog): assert "path" in caplog.text.lower() +class TestSelectTimestepsHelpers: + """Pure helpers for auto interval selection — no DB, no power flow.""" + + @staticmethod + def _week(start): + return pd.date_range(start=start, periods=168, freq="h") + + def test_overlap_detection(self): + a = self._week("2035-01-01") + assert intervals_overlap(a, self._week("2035-01-04")) # overlaps + assert not intervals_overlap(a, self._week("2035-06-01")) # disjoint + + def test_disjoint_top_intervals_kept_as_two(self): + load = [self._week("2035-01-01")] + volt = [self._week("2035-06-01")] + result = select_two_intervals(load, volt) + assert len(result) == 2 + assert not intervals_overlap(result[0], result[1]) + + def test_overlap_falls_to_next_ranked_voltage(self): + load = [self._week("2035-01-01")] + # first voltage candidate overlaps the top loading interval, second does not + volt = [self._week("2035-01-03"), self._week("2035-09-01")] + result = select_two_intervals(load, volt) + assert len(result) == 2 + assert result[1].equals(volt[1]) + + def test_all_overlap_concatenates_to_one(self): + load = [self._week("2035-01-01")] + volt = [self._week("2035-01-03")] # only candidate, overlaps + result = select_two_intervals(load, volt) + assert len(result) == 1 + merged = result[0] + # merged interval is contiguous and spans both inputs + assert merged.min() == load[0].min() + assert merged.max() == volt[0].max() + assert (merged[1:] - merged[:-1]).nunique() == 1 # regular spacing + + def test_single_side_only(self): + week = self._week("2035-01-01") + for result in ( + select_two_intervals([week], []), + select_two_intervals([], [week]), + ): + assert len(result) == 1 + assert result[0].equals(week) + assert select_two_intervals([], []) == [] + + +class TestSelectTimestepsManual: + def test_manual_explicit_timestamps_before_imports(self, edisgo_obj): + """ + Manual mode with an empty timeseries (positioned before imports) sets + the index and stashes it for HP/DSM imports. + """ + # start from an empty time index to mimic the pre-import position + edisgo_obj.set_timeindex(pd.DatetimeIndex([])) + ts = ["2011-01-01 00:00", "2011-01-01 02:00"] + ctx = RunContext( + raw_config={"timeseries_selection": {"mode": "manual", "timestamps": ts}} + ) + result = task_select_timesteps(edisgo_obj, ctx) + assert list(result.timeseries.timeindex) == list(pd.to_datetime(ts)) + assert list(ctx.flags["selected_timeindex"]) == list(pd.to_datetime(ts)) + assert ctx.flags["timesteps_selected"] is True + + def test_manual_range_reduces_existing_timeseries(self, edisgo_obj): + """Manual range with an existing 3-step index reduces to the range.""" + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "manual", + "start": "2011-01-01 00:00", + "periods": 2, + "freq": "h", + } + } + ) + result = task_select_timesteps(edisgo_obj, ctx) + assert len(result.timeseries.timeindex) == 2 + + def test_missing_mode_raises(self, edisgo_obj): + with pytest.raises(ValueError, match="mode 'manual' or 'auto'"): + task_select_timesteps(edisgo_obj, RunContext(raw_config={})) + + def test_auto_without_active_power_raises(self, edisgo_obj): + ctx = RunContext(raw_config={"timeseries_selection": {"mode": "auto"}}) + with pytest.raises(ValueError, match="active-power time series"): + task_select_timesteps(edisgo_obj, ctx) + + def test_auto_unknown_method_raises(self, edisgo_obj): + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "auto", + "method": "bogus", + } + } + ) + ctx.flags["timeseries_set"] = True + with pytest.raises(ValueError, match="power_flow.*residual_load"): + task_select_timesteps(edisgo_obj, ctx) + + def test_residual_load_requires_overlying_grid(self, edisgo_obj): + """residual_load method must raise when no overlying-grid data is set.""" + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "auto", + "method": "residual_load", + } + } + ) + ctx.flags["timeseries_set"] = True + with pytest.raises(ValueError, match="overlying-grid data"): + task_select_timesteps(edisgo_obj, ctx) + + +class TestSelectTimestepsPosition: + """The `position` param lets one pipeline carry both a pre-import and a + post-grid select_timesteps step; each no-ops off its mode.""" + + def test_pre_import_noops_in_auto_mode(self, edisgo_obj): + """ + A pre_import step in auto mode must be a no-op — crucially it must NOT + hit the auto guard (no active power set yet), it just returns. + """ + before = edisgo_obj.timeseries.timeindex + ctx = RunContext(raw_config={"timeseries_selection": {"mode": "auto"}}) + result = task_select_timesteps(edisgo_obj, ctx, position="pre_import") + assert result is edisgo_obj + assert result.timeseries.timeindex.equals(before) + assert "timesteps_selected" not in ctx.flags + + def test_post_grid_noops_in_manual_mode(self, edisgo_obj): + """A post_grid step in manual mode must be a no-op (manual already ran + earlier at pre_import).""" + before = edisgo_obj.timeseries.timeindex + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "manual", + "timestamps": ["2011-01-01 00:00"], + } + } + ) + result = task_select_timesteps(edisgo_obj, ctx, position="post_grid") + assert result is edisgo_obj + assert result.timeseries.timeindex.equals(before) + + def test_pre_import_acts_in_manual_mode(self, edisgo_obj): + edisgo_obj.set_timeindex(pd.DatetimeIndex([])) + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "manual", + "timestamps": ["2011-01-01 00:00"], + } + } + ) + task_select_timesteps(edisgo_obj, ctx, position="pre_import") + assert len(edisgo_obj.timeseries.timeindex) == 1 + assert ctx.flags["timesteps_selected"] is True + + def test_bad_position_raises(self, edisgo_obj): + ctx = RunContext(raw_config={"timeseries_selection": {"mode": "manual"}}) + with pytest.raises(ValueError, match="position"): + task_select_timesteps(edisgo_obj, ctx, position="bogus") + + def test_pre_import_sets_default_index_when_empty_in_auto(self, edisgo_obj): + """ + A pre_import step in auto mode with no time index set must establish a + full-year default (so later imports build hourly full-year data), even + though it otherwise no-ops. + """ + edisgo_obj.set_timeindex(pd.DatetimeIndex([])) + ctx = RunContext( + scenario="eGon2035", + raw_config={"timeseries_selection": {"mode": "auto"}}, + ) + task_select_timesteps(edisgo_obj, ctx, position="pre_import") + ti = edisgo_obj.timeseries.timeindex + assert len(ti) == 8760 + assert ti[0].year == 2035 + # still a no-op for actual selection + assert "timesteps_selected" not in ctx.flags + + def test_manual_shifts_user_timestamps_to_timeseries_year(self, edisgo_obj): + """ + Manual mode reducing an existing (differently-yeared) time series shifts + the user timestamps to the time-series year so slicing matches. + """ + # existing time series in 2011 + edisgo_obj.set_timeindex(pd.date_range("2011-06-01", periods=5, freq="h")) + # user selects timestamps written in the scenario year 2035 + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "manual", + "timestamps": ["2035-06-01 01:00", "2035-06-01 03:00"], + } + } + ) + task_select_timesteps(edisgo_obj, ctx) + ti = edisgo_obj.timeseries.timeindex + assert list(ti) == [ + pd.Timestamp("2011-06-01 01:00"), + pd.Timestamp("2011-06-01 03:00"), + ] + + +class TestOptimizeTaskDelegation: + """task_optimize is thin: expand the `flexible` shortcut and call + edisgo.pm_optimize. The multi-interval split lives in pm_optimize and is + tested in tests/opf/test_powermodels_opf.py.""" + + def test_expands_flexible_shortcut_and_calls_pm_optimize(self, edisgo_obj): + edisgo_obj.set_timeindex(pd.date_range("2035-01-01", periods=24, freq="h")) + captured = {} + + def fake_pm_optimize(**kw): + captured.update(kw) + + edisgo_obj.pm_optimize = fake_pm_optimize + task_optimize(edisgo_obj, RunContext(), flexible=["heat_pumps", "storage"]) + # shortcut expanded to explicit name lists (empty ok if grid lacks type) + assert "flexible_hps" in captured and "flexible_storage_units" in captured + assert isinstance(captured["flexible_hps"], list) + + def test_all_bundled_presets_validate(): """ Every bundled preset must pass the (metadata-driven) validator — this diff --git a/tests/run/test_validator.py b/tests/run/test_validator.py index 1d375f10d..9e830e0bc 100644 --- a/tests/run/test_validator.py +++ b/tests/run/test_validator.py @@ -5,6 +5,7 @@ without TS, optimize without flex, flex import without grid, and the stage-level ``load_from`` constraints. """ + import pytest from edisgo.run.validator import validate @@ -30,8 +31,9 @@ def _wrap(pipeline): def test_valid_pipeline(): """A well-formed pipeline must pass validation without raising.""" - validate(_wrap(["setup_grid", "worst_case_ts", "reactive_power", - "reinforce", "save"])) + validate( + _wrap(["setup_grid", "worst_case_ts", "reactive_power", "reinforce", "save"]) + ) def test_unknown_task_rejected(): @@ -46,6 +48,33 @@ def test_reactive_before_ts_rejected(): validate(_wrap(["setup_grid", "reactive_power", "worst_case_ts"])) +def test_select_timesteps_after_reactive_rejected(): + """ + select_timesteps is ts_altering (it reduces the time index), so it must + not appear after reactive_power. + """ + with pytest.raises(ValueError, match="reactive_power"): + validate( + _wrap(["setup_grid", "worst_case_ts", "reactive_power", "select_timesteps"]) + ) + + +def test_select_timesteps_before_reactive_ok(): + """select_timesteps before reactive_power is the intended ordering.""" + validate( + _wrap( + [ + "setup_grid", + "worst_case_ts", + "select_timesteps", + "reactive_power", + "reinforce", + "save", + ] + ) + ) + + def test_reinforce_without_ts_rejected(): """reinforce without any prior time-series step must fail.""" with pytest.raises(ValueError, match="time series"): @@ -66,37 +95,48 @@ def test_flex_import_before_grid_rejected(): def test_stage_load_from_missing_rejected(): """``load_from: X`` where X has not run must fail.""" - cfg = {"stages": [ - {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", - "reinforce"]}, - {"name": "b", "load_from": "nonexistent", - "pipeline": ["reinforce"]}, - ]} + cfg = { + "stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", "reinforce"]}, + {"name": "b", "load_from": "nonexistent", "pipeline": ["reinforce"]}, + ] + } with pytest.raises(ValueError, match="load_from"): validate(cfg) def test_stage_load_from_requires_save_in_source(): """A stage consumed by ``load_from`` must itself end with ``save``.""" - cfg = {"stages": [ - {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", - "reinforce"]}, # no save - {"name": "b", "load_from": "a", "pipeline": ["reinforce"]}, - ]} + cfg = { + "stages": [ + { + "name": "a", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + }, # no save + {"name": "b", "load_from": "a", "pipeline": ["reinforce"]}, + ] + } with pytest.raises(ValueError, match="load_from"): validate(cfg) def test_stage_load_from_with_save_ok(): """Stage chain with a save in the source must validate successfully.""" - cfg = {"stages": [ - {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", - "reinforce", "save"]}, - # load_from reloads the grid with import_timeseries=False, so the - # consuming stage must set time series itself before reinforce. - {"name": "b", "load_from": "a", - "pipeline": ["worst_case_ts", "reinforce", "save"]}, - ]} + cfg = { + "stages": [ + { + "name": "a", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce", "save"], + }, + # load_from reloads the grid with import_timeseries=False, so the + # consuming stage must set time series itself before reinforce. + { + "name": "b", + "load_from": "a", + "pipeline": ["worst_case_ts", "reinforce", "save"], + }, + ] + } validate(cfg) @@ -106,10 +146,14 @@ def test_stage_load_from_without_ts_rejected(): reloaded with import_timeseries=False, so reinforce after a bare load_from (no time-series task in the stage) must be rejected. """ - cfg = {"stages": [ - {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", - "reinforce", "save"]}, - {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, - ]} + cfg = { + "stages": [ + { + "name": "a", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce", "save"], + }, + {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, + ] + } with pytest.raises(ValueError, match="requires time series"): validate(cfg) diff --git a/tests/tools/test_temporal_complexity_reduction.py b/tests/tools/test_temporal_complexity_reduction.py index 1063093f4..4c32daa01 100644 --- a/tests/tools/test_temporal_complexity_reduction.py +++ b/tests/tools/test_temporal_complexity_reduction.py @@ -183,3 +183,92 @@ def test_get_most_critical_time_intervals(self): steps.loc[0, "time_steps_voltage_issues"] == pd.date_range("1/1/2018", periods=24, freq="H") ).all() + + +class TestIntervalHelpers: + """Relocated pure helpers + residual_load selection (no DB / no power flow).""" + + @staticmethod + def _week(start): + return pd.date_range(start=start, periods=168, freq="h") + + def test_intervals_overlap(self): + a = self._week("2035-01-01") + assert temp_red.intervals_overlap(a, self._week("2035-01-04")) + assert not temp_red.intervals_overlap(a, self._week("2035-06-01")) + + def test_select_two_intervals_disjoint(self): + result = temp_red.select_two_intervals( + [self._week("2035-01-01")], [self._week("2035-06-01")] + ) + assert len(result) == 2 + assert not temp_red.intervals_overlap(result[0], result[1]) + + def test_select_two_intervals_next_ranked(self): + load = [self._week("2035-01-01")] + volt = [self._week("2035-01-03"), self._week("2035-09-01")] + result = temp_red.select_two_intervals(load, volt) + assert len(result) == 2 and result[1].equals(volt[1]) + + def test_select_two_intervals_concatenate(self): + result = temp_red.select_two_intervals( + [self._week("2035-01-01")], [self._week("2035-01-03")] + ) + assert len(result) == 1 + assert (result[0][1:] - result[0][:-1]).nunique() == 1 # contiguous + + def test_select_two_intervals_single_and_empty(self): + week = self._week("2035-01-01") + assert temp_red.select_two_intervals([week], [])[0].equals(week) + assert temp_red.select_two_intervals([], [week])[0].equals(week) + assert temp_red.select_two_intervals([], []) == [] + + def test_build_centered_interval(self): + idx = pd.date_range("2035-01-01 00:00", periods=8760, freq="h") + t = pd.Timestamp("2035-02-10 12:00") + iv = temp_red._build_centered_interval(t, idx, 168, 4) + assert len(iv) == 168 + assert iv[0].hour == 4 # starts on the day-start hour + assert t in iv # critical step contained + assert iv[-1] != t # centered -> not the last step + + def test_residual_load_steps_and_intervals(self, monkeypatch): + import types + + idx = pd.date_range("2035-01-01 00:00", periods=8760, freq="h") + residual = pd.Series(range(8760), index=idx, dtype=float) + fake = types.SimpleNamespace( + timeseries=types.SimpleNamespace(residual_load=residual) + ) + monkeypatch.setattr( + "edisgo.network.overlying_grid.distribute_overlying_grid_requirements", + lambda e: fake, + ) + # steps: top-3 highest + bottom-2 lowest residual + steps = temp_red.get_most_critical_time_steps( + object(), by="residual_load", num_steps_loading=3, num_steps_voltage=2 + ) + assert idx[-1] in steps and idx[0] in steps and len(steps) == 5 + + # intervals: per-case columns, centered on the residual max/min steps + e = types.SimpleNamespace( + timeseries=types.SimpleNamespace(timeindex=idx), + topology=types.SimpleNamespace(id="g"), + ) + df = temp_red.get_most_critical_time_intervals( + e, + by="residual_load", + num_time_intervals=2, + time_steps_per_time_interval=168, + time_step_day_start=4, + ) + assert list(df.columns) == ["time_steps_load_case", "time_steps_feedin_case"] + assert len(df) == 2 + # top load-case interval is centered on the global max residual step + assert idx[-1] in df.loc[0, "time_steps_load_case"] + + def test_bad_by_raises(self): + with pytest.raises(ValueError, match="power_flow.*residual_load"): + temp_red.get_most_critical_time_steps(object(), by="bogus") + with pytest.raises(ValueError, match="power_flow.*residual_load"): + temp_red.get_most_critical_time_intervals(object(), by="bogus") From 9a2124901f49f78410a10db77aeb652009ef80da Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 10 Jul 2026 11:19:23 +0200 Subject: [PATCH 42/66] Allow using ding0 grids downloaded from zenodo (#690) --- edisgo/run/tasks/grid.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py index 45e050076..a8474c142 100644 --- a/edisgo/run/tasks/grid.py +++ b/edisgo/run/tasks/grid.py @@ -17,6 +17,7 @@ from edisgo.run.registry import register_task +import pandas as pd @register_task("setup_grid", provides={"grid"}) def task_setup_grid( @@ -73,6 +74,7 @@ def task_setup_grid( """ from edisgo import EDisGo + import os grid_cfg = ctx.raw_config.get("grid", {}) ding0_path = ding0_path or grid_cfg.get("ding0_path") @@ -85,8 +87,15 @@ def task_setup_grid( legacy_ding0_grids = grid_cfg.get("legacy_ding0_grids", False) if edisgo is None: + # Check if topology-subfolder is part of ding0 grids + ding0_path_str = str(ding0_path) + if not os.path.exists(os.path.join(ding0_path_str, "buses.csv")): + topology_path = os.path.join(ding0_path_str, "topology") + if os.path.exists(os.path.join(topology_path, "buses.csv")): + ding0_path_str = topology_path + edisgo = EDisGo( - ding0_grid=str(ding0_path), + ding0_grid=ding0_path_str, legacy_ding0_grids=legacy_ding0_grids, ) else: From 276107483a111a0552e1ac220b1e2d44a80ff9eb Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Mon, 13 Jul 2026 10:34:29 +0200 Subject: [PATCH 43/66] Use overlying_grid_data with adjusted timeindex to set generators_active_power --- edisgo/run/tasks/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 6ffba2ee8..045b824e3 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -296,8 +296,8 @@ def _to_edisgo_timeindex(ts, extra_step=False): # --- 3) set dispatchable/fluctuating generator time series --- if source == "etrago": - disp_ts = overlying_grid_data.get("dispatchable_generators_active_power") - pot_ts = overlying_grid_data.get("renewables_potential") + disp_ts = edisgo.overlying_grid.dispatchable_generators_active_power + pot_ts = edisgo.overlying_grid.renewables_potential if disp_ts is not None and not disp_ts.empty: edisgo.set_time_series_active_power_predefined( dispatchable_generators_ts=disp_ts, From 7f6905b0ff167327cc4989aee08e12f906e1041f Mon Sep 17 00:00:00 2001 From: "Moritz.Schloesser" Date: Tue, 14 Jul 2026 11:43:18 +0000 Subject: [PATCH 44/66] Change database from OEP to local --- edisgo/run/presets/uc5_select_timesteps.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edisgo/run/presets/uc5_select_timesteps.yaml b/edisgo/run/presets/uc5_select_timesteps.yaml index 426ce1231..82431bc7d 100644 --- a/edisgo/run/presets/uc5_select_timesteps.yaml +++ b/edisgo/run/presets/uc5_select_timesteps.yaml @@ -46,8 +46,8 @@ grid: legacy_ding0_grids: false database: - ssh: - enabled: false + source: local + # No explicit base time index is set. oedb_ts falls back to a full year derived # from the scenario when none is given, which is what auto interval selection From 5a09fc3d536b714a2cdfdfdb603357178756a4d4 Mon Sep 17 00:00:00 2001 From: "Moritz.Schloesser" Date: Wed, 15 Jul 2026 13:36:39 +0000 Subject: [PATCH 45/66] feat: spatial complexity reduction for the run pipeline Adds spatial_reduce/spatial_restore tasks bracketing optimize, so the OPF can run on a spatially-reduced grid while reinforcement still runs on the full topology. New core function apply_reduced_results_to_full_grid maps optimized flexible-component dispatch (charging points, heat pumps, DSM loads, storage) back onto the full grid, by name or disaggregated onto old_name members per time step (weighted by each member's pre-OPF flexibility envelope) when aggregation_mode=True. Reactive power is recomputed on restore, mirroring how pm_optimize itself handles it. optimize now also records its flexible-component name lists on ctx.flags and declares provides={"optimized_dispatch"}, so the validator's existing requires/provides system enforces spatial_restore running after both spatial_reduce and optimize. --- edisgo/edisgo.py | 63 ++++- edisgo/run/context.py | 6 + edisgo/run/tasks/__init__.py | 10 +- edisgo/run/tasks/analysis.py | 15 +- edisgo/run/tasks/spatial.py | 150 +++++++++++ edisgo/tools/spatial_complexity_reduction.py | 231 ++++++++++++++++ .../test_spatial_complexity_reduction.py | 253 ++++++++++++++++++ 7 files changed, 724 insertions(+), 4 deletions(-) create mode 100644 edisgo/run/tasks/spatial.py diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 140e290ae..eb71f0b37 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -64,7 +64,10 @@ from edisgo.tools import plots, tools from edisgo.tools.config import Config from edisgo.tools.geo import find_nearest_bus -from edisgo.tools.spatial_complexity_reduction import spatial_complexity_reduction +from edisgo.tools.spatial_complexity_reduction import ( + apply_reduced_results_to_full_grid, + spatial_complexity_reduction, +) from edisgo.tools.tools import ( determine_grid_integration_voltage_level, get_path_length_to_station, @@ -3578,6 +3581,64 @@ def spatial_complexity_reduction( ) return edisgo_obj, busmap_df, linemap_df + def map_reduced_results_to_full_grid( + self, + reduced_grid: EDisGo, + flexible_cps: list | None = None, + flexible_hps: list | None = None, + flexible_loads: list | None = None, + flexible_storage_units: list | None = None, + ) -> EDisGo: + """ + Writes optimized flexible-component dispatch from a spatially-reduced + grid back onto this (full) grid. + + Counterpart to :meth:`spatial_complexity_reduction`: where that + method shrinks this grid for a faster OPF, this method maps the OPF's + active-power results from ``reduced_grid`` back onto ``self`` so + reinforcement can run on the full topology. Only components the OPF + actually rewrites are touched — flexible charging points, heat pumps, + DSM loads, and storage units. Inflexible loads/generators are + untouched, since the OPF never changed their series and ``self`` + already holds the correct values for them. + + See :func:`~.tools.spatial_complexity_reduction.apply_reduced_results_to_full_grid` + for the full matching/disaggregation rules and the reactive-power + recompute this method triggers as a side effect. + + Parameters + ---------- + reduced_grid : :class:`~.EDisGo` + The spatially-reduced EDisGo instance the OPF ran on. Supplies + the optimized active-power series and, if aggregated, the + ``old_name`` provenance for disaggregation. + flexible_cps : list of str, optional + Names of flexible charging points in ``reduced_grid`` to map + back. + flexible_hps : list of str, optional + Names of flexible heat-pump loads in ``reduced_grid`` to map + back. + flexible_loads : list of str, optional + Names of flexible DSM loads in ``reduced_grid`` to map back. + flexible_storage_units : list of str, optional + Names of flexible storage units in ``reduced_grid`` to map back. + + Returns + ------- + :class:`~.EDisGo` + ``self``, with active power written for the given flexible + components and reactive power recomputed. + + """ + return apply_reduced_results_to_full_grid( + full_grid=self, + reduced_grid=reduced_grid, + flexible_cps=flexible_cps, + flexible_hps=flexible_hps, + flexible_loads=flexible_loads, + flexible_storage_units=flexible_storage_units, + ) + def check_integrity(self): """ Method to check the integrity of the EDisGo object. diff --git a/edisgo/run/context.py b/edisgo/run/context.py index e41a07fc9..9042284dd 100644 --- a/edisgo/run/context.py +++ b/edisgo/run/context.py @@ -67,6 +67,11 @@ class RunContext: ``overlying_grid_data=`` argument of :func:`edisgo.run.run_edisgo`. Consumed by the ``import_overlying_grid_data`` task when ``overlying_grid.source == "etrago"``. + full_grid_stash : edisgo.EDisGo or None + The pre-reduction :class:`~edisgo.EDisGo` instance, deepcopied and + stashed by the ``spatial_reduce`` task before it spatially reduces + the working object. Consumed (and cleared back to ``None``) by + ``spatial_restore``. ``None`` when spatial reduction is not in use. """ @@ -81,6 +86,7 @@ class RunContext: current_stage: str | None = None raw_config: dict[str, Any] = field(default_factory=dict) overlying_grid_data: Any = None + full_grid_stash: Any = None def ensure_engine(self): """ diff --git a/edisgo/run/tasks/__init__.py b/edisgo/run/tasks/__init__.py index 9a1be82a8..a0ef5d22c 100644 --- a/edisgo/run/tasks/__init__.py +++ b/edisgo/run/tasks/__init__.py @@ -26,6 +26,7 @@ * :mod:`.analysis` — ``check_integrity``, ``analyze``, ``reinforce``, ``base_reinforce``, ``optimize`` * :mod:`.io` — ``save``, ``load_charging_from_files`` +* :mod:`.spatial` — ``spatial_reduce``, ``spatial_restore`` Task signature convention: ``(edisgo, ctx, **params)``. A task may mutate ``edisgo`` in place and/or return a new EDisGo instance (the @@ -33,4 +34,11 @@ loop). """ -from edisgo.run.tasks import analysis, flex, grid, io, timeseries # noqa: F401 +from edisgo.run.tasks import ( # noqa: F401 + analysis, + flex, + grid, + io, + spatial, + timeseries, +) diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py index 7bbb4ba3f..9f8ec57ec 100644 --- a/edisgo/run/tasks/analysis.py +++ b/edisgo/run/tasks/analysis.py @@ -268,7 +268,9 @@ def task_base_reinforce( return edisgo -@register_task("optimize", requires={"timeseries", "flex"}) +@register_task( + "optimize", requires={"timeseries", "flex"}, provides={"optimized_dispatch"} +) def task_optimize( edisgo, ctx, @@ -307,7 +309,11 @@ def task_optimize( EDisGo instance to optimize. ctx : RunContext Run context. Used for logging and, for multi-interval runs, nothing - else is required from it. + else is required from it. The resolved ``flexible_*`` name lists are + written to ``ctx.flags['flexible_cps']`` / ``ctx.flags['flexible_hps']`` + / ``ctx.flags['flexible_loads']`` / ``ctx.flags['flexible_storage_units']`` + so a later ``spatial_restore`` step knows which components' dispatch + needs mapping back onto the full grid. flexible : list of str, optional High-level selector, subset of ``{"heat_pumps", "charging_points", "storage"}``. If ``None``, nothing is @@ -359,6 +365,11 @@ def task_optimize( if flexible_storage_units is None: flexible_storage_units = [] + ctx.flags["flexible_cps"] = flexible_cps + ctx.flags["flexible_hps"] = flexible_hps + ctx.flags["flexible_loads"] = flexible_loads + ctx.flags["flexible_storage_units"] = flexible_storage_units + # pm_optimize handles a non-contiguous (reduced) time index internally: # it runs one OPF per contiguous interval and merges the results. edisgo.pm_optimize( diff --git a/edisgo/run/tasks/spatial.py b/edisgo/run/tasks/spatial.py new file mode 100644 index 000000000..ff0683144 --- /dev/null +++ b/edisgo/run/tasks/spatial.py @@ -0,0 +1,150 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +""" +Spatial complexity reduction tasks bracketing ``optimize``. + +* :func:`task_spatial_reduce` (``spatial_reduce``) — stashes a deepcopy of + the full grid on ``ctx`` and spatially reduces the working object so + ``optimize`` runs on a smaller grid. +* :func:`task_spatial_restore` (``spatial_restore``) — writes the optimized + flexible-component dispatch back onto the stashed full grid and makes it + the active object again, so ``reinforce`` runs on the full topology. + +Both are no-ops when ``spatial_reduction.enabled`` is false (the default), +so a pipeline that carries this bracket behaves exactly like one that +doesn't when spatial reduction is turned off. +""" + +from __future__ import annotations + +import copy + +from edisgo.run.registry import register_task + + +@register_task("spatial_reduce", requires={"grid"}, provides={"reduced_grid"}) +def task_spatial_reduce(edisgo, ctx, **overrides): + """ + Deepcopy and stash the full grid, then spatially reduce the working + object. + + Configuration is read from the top-level ``spatial_reduction:`` config + block (so eGo can inject it the same way it injects + ``timeseries_selection``); inline step params override individual keys + of that block. + + A no-op when ``enabled`` is not true — ``edisgo`` is returned unchanged + and ``ctx.full_grid_stash`` is left ``None``, so a downstream + ``spatial_restore`` also no-ops (see its docstring) and ``optimize``/ + ``reinforce`` run on the same, unreduced grid as if this task were + absent from the pipeline. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to spatially reduce in place. + ctx : RunContext + Run context. Reads ``ctx.raw_config['spatial_reduction']``. Sets + ``ctx.full_grid_stash`` to the pre-reduction deepcopy. + **overrides + Inline step params overriding keys of the ``spatial_reduction`` + block. Recognized keys: ``enabled`` (bool, default ``False``), + ``mode``, ``cluster_area``, ``reduction_factor``, + ``reduction_factor_not_focused``, ``aggregation_mode``, and the + aggregation sub-modes ``load_aggregation_mode`` / + ``generator_aggregation_mode`` — forwarded to + :meth:`~.EDisGo.spatial_complexity_reduction`. + + Returns + ------- + edisgo.EDisGo + The (possibly) spatially-reduced EDisGo instance. + + """ + cfg = {**ctx.raw_config.get("spatial_reduction", {}), **overrides} + if not cfg.get("enabled", False): + return edisgo + + ctx.full_grid_stash = copy.deepcopy(edisgo) + + kwargs = { + k: v + for k, v in cfg.items() + if k + in ( + "mode", + "cluster_area", + "reduction_factor", + "reduction_factor_not_focused", + "apply_pseudo_coordinates", + "aggregation_mode", + "load_aggregation_mode", + "generator_aggregation_mode", + "line_naming_convention", + "mv_pseudo_coordinates", + ) + } + edisgo.spatial_complexity_reduction(copy_edisgo=False, **kwargs) + return edisgo + + +@register_task( + "spatial_restore", requires={"reduced_grid", "optimized_dispatch"} +) +def task_spatial_restore(edisgo, ctx, **overrides): + """ + Write optimized flexible-component dispatch back onto the stashed full + grid, and make it the active object again. + + Reads the flexible-component name lists ``optimize`` wrote to + ``ctx.flags`` and passes them, together with ``edisgo`` (the reduced, + just-optimized grid) and ``ctx.full_grid_stash`` (the pre-reduction + grid), to :meth:`~.EDisGo.map_reduced_results_to_full_grid`. See that + method (and the core function it wraps, + :func:`~.tools.spatial_complexity_reduction.apply_reduced_results_to_full_grid`) + for the matching/disaggregation rules. + + A no-op when ``ctx.full_grid_stash`` is ``None`` — i.e. when + ``spatial_reduce`` did not run or ran disabled — so ``edisgo`` (the + grid ``optimize`` already ran on) is returned unchanged. + + Parameters + ---------- + edisgo : edisgo.EDisGo + The reduced EDisGo instance ``optimize`` ran on. + ctx : RunContext + Run context. Reads ``ctx.full_grid_stash`` and the + ``flexible_cps`` / ``flexible_hps`` / ``flexible_loads`` / + ``flexible_storage_units`` flags ``optimize`` set. Clears + ``ctx.full_grid_stash`` back to ``None`` after restoring. + **overrides + Unused; accepted for signature consistency with other tasks. + + Returns + ------- + edisgo.EDisGo + The full-grid EDisGo instance with flexible dispatch restored, or + ``edisgo`` unchanged if there is no stash to restore from. + + """ + full_grid = ctx.full_grid_stash + if full_grid is None: + return edisgo + + full_grid.map_reduced_results_to_full_grid( + reduced_grid=edisgo, + flexible_cps=ctx.flags.get("flexible_cps"), + flexible_hps=ctx.flags.get("flexible_hps"), + flexible_loads=ctx.flags.get("flexible_loads"), + flexible_storage_units=ctx.flags.get("flexible_storage_units"), + ) + ctx.full_grid_stash = None + return full_grid diff --git a/edisgo/tools/spatial_complexity_reduction.py b/edisgo/tools/spatial_complexity_reduction.py index 21e8f8665..ad9a51fad 100644 --- a/edisgo/tools/spatial_complexity_reduction.py +++ b/edisgo/tools/spatial_complexity_reduction.py @@ -1916,6 +1916,237 @@ def spatial_complexity_reduction( return busmap_df, linemap_df +def apply_reduced_results_to_full_grid( + full_grid: EDisGo, + reduced_grid: EDisGo, + *, + flexible_cps: list | None = None, + flexible_hps: list | None = None, + flexible_loads: list | None = None, + flexible_storage_units: list | None = None, +) -> EDisGo: + """ + Write optimized flexible-component dispatch from a spatially-reduced grid + back onto the full grid. + + Counterpart to :func:`spatial_complexity_reduction`: where that function + shrinks a grid for a faster OPF, this function maps the OPF's active-power + results back onto the pre-reduction grid so reinforcement can run on the + full topology. Only components the OPF actually rewrites are touched — + flexible charging points, heat pumps, DSM loads, and storage units. + Inflexible loads/generators are untouched: the OPF never changed their + series, so ``full_grid`` already holds the correct values for them. + + ``full_grid`` and ``reduced_grid`` are matched by name for storage units + (never aggregated by :func:`spatial_complexity_reduction`, so their names + are unchanged) and, for the other three flexibility types, by the + ``old_name`` column that :func:`spatial_complexity_reduction` writes onto + ``reduced_grid.topology.loads_df`` when ``aggregation_mode=True``. A + member listed in ``old_name`` is a load whose active-power series was + merged into one representative row; when ``aggregation_mode=False`` (or a + given member was not merged), ``old_name`` is absent and the member's own + name is used directly — i.e. a plain by-name write-back. + + For a merged representative, the representative's optimized series is + disaggregated onto its ``old_name`` members **per time step**, weighted by + each member's own pre-OPF flexibility envelope (a known input, never the + optimized result): + + * charging points — ``upper_power(t)`` from + ``electromobility.flexibility_bands`` (a charging point with no + connected vehicle has ``upper_power(t) == 0``, so it receives none of + the representative's dispatch that time step); + * heat pumps — ``min(heat_demand(t) / cop(t), p_set)``, i.e. the + electrical-equivalent heat demand capped at the heat pump's own rated + power, mirroring how a charging point's ``upper_power(t)`` is already a + capped bound rather than raw uncapped demand; + * DSM loads — ``p_max(t)`` from :attr:`~.network.dsm.DSM.p_max`. + + Weights always sum back to the representative's value exactly at every + time step; a time step where every member's weight is 0 falls back to an + equal split. + + Reactive power is not read from ``reduced_grid``. After writing active + power, this function calls + :meth:`~.EDisGo.set_time_series_reactive_power_control` on ``full_grid`` + with its defaults, mirroring how the OPF itself derives reactive power + for the components it just optimized (see + :func:`~.io.powermodels_io.from_powermodels`) — reactive power is always + a function of whatever active power is currently set, regardless of + whether that active power came from a default, worst case, or the OPF. + + Parameters + ---------- + full_grid : :class:`~.EDisGo` + The pre-reduction EDisGo instance to write dispatch onto, modified in + place. Must contain every component named in ``flexible_cps`` / + ``flexible_hps`` / ``flexible_loads`` / ``flexible_storage_units`` and + (for merged components) every name listed in ``reduced_grid``'s + ``old_name`` columns. + reduced_grid : :class:`~.EDisGo` + The spatially-reduced EDisGo instance the OPF ran on. Supplies the + optimized active-power series and, if aggregated, the ``old_name`` + provenance. + flexible_cps : list of str, optional + Names of flexible charging points in ``reduced_grid`` to map back. + flexible_hps : list of str, optional + Names of flexible heat-pump loads in ``reduced_grid`` to map back. + flexible_loads : list of str, optional + Names of flexible DSM loads in ``reduced_grid`` to map back. + flexible_storage_units : list of str, optional + Names of flexible storage units in ``reduced_grid`` to map back. + + Returns + ------- + :class:`~.EDisGo` + ``full_grid``, with active power written for the given flexible + components and reactive power recomputed. + + """ + # NOTE: "x or []" is unsafe here - callers may pass a numpy array (e.g. + # task_optimize derives flexible_loads as + # edisgo.dsm.p_min.columns.values), and "array or []" raises + # ValueError ("truth value of an array... is ambiguous") for any array + # with more than one element. "is None" is the correct emptiness check + # for an optional list-like argument. + flexible_cps = list(flexible_cps) if flexible_cps is not None else [] + flexible_hps = list(flexible_hps) if flexible_hps is not None else [] + flexible_loads = list(flexible_loads) if flexible_loads is not None else [] + flexible_storage_units = ( + list(flexible_storage_units) if flexible_storage_units is not None else [] + ) + + def _require_full_timeindex(envelope: DataFrame, envelope_name: str) -> None: + """Raise a clear error if ``envelope`` doesn't cover the full grid's + active time index, instead of a bare ``KeyError`` deep inside a + ``.loc`` lookup. + + This can only happen if ``full_grid``'s flexibility-band/DSM/heat-pump + attributes were never trimmed to the same time index as + ``full_grid.timeseries.timeindex`` - i.e. if the pre-reduction stash + was taken before the run's time index was finalized. + """ + ti = full_grid.timeseries.timeindex + missing = ti.difference(envelope.index) + if len(missing) > 0: + raise ValueError( + f"apply_reduced_results_to_full_grid: full_grid's " + f"{envelope_name} does not cover {len(missing)} of " + f"full_grid.timeseries.timeindex's time steps (e.g. " + f"{missing[0]!r}). This usually means the full-grid stash " + f"was taken before the time index was finalized - run " + f"time-index selection (e.g. select_timesteps) before " + f"spatial_reduce." + ) + + def _old_name_map(loads_df: DataFrame, names: list) -> dict: + """Map each representative name in ``names`` to its member names. + + A name absent from ``old_name`` (not merged, or + ``aggregation_mode=False``) maps to itself. + """ + name_map = {} + for name in names: + old_name = loads_df.at[name, "old_name"] if "old_name" in loads_df else None + name_map[name] = old_name if isinstance(old_name, list) else [name] + return name_map + + def _write_by_name(active_power: DataFrame, names: list, target: DataFrame) -> None: + ti = full_grid.timeseries.timeindex + target.loc[ti, names] = active_power.loc[ti, names].values + + def _disaggregate( + active_power: DataFrame, + name_map: dict, + envelope: DataFrame, + target: DataFrame, + ) -> None: + """Split each representative's series onto its members per time step. + + ``envelope`` holds each member's pre-OPF flexibility envelope + (columns = member names, index = time index); members missing from + ``envelope`` are treated as having an all-zero envelope (equal-split + fallback). + """ + ti = full_grid.timeseries.timeindex + for representative, members in name_map.items(): + if len(members) == 1 and members[0] == representative: + target.loc[ti, representative] = active_power.loc[ti, representative] + continue + weights = pd.DataFrame(index=ti, columns=members, dtype=float) + for member in members: + weights[member] = ( + envelope.loc[ti, member] if member in envelope.columns else 0.0 + ) + weight_sum = weights.sum(axis="columns") + zero_envelope = weight_sum == 0 + shares = weights.div(weight_sum.replace(0, np.nan), axis="index") + shares.loc[zero_envelope, :] = 1.0 / len(members) + representative_power = active_power.loc[ti, representative] + for member in members: + target.loc[ti, member] = shares[member] * representative_power + + reduced_loads_df = reduced_grid.topology.loads_df + full_loads_df = full_grid.topology.loads_df + + # Always routed through _disaggregate (never the by-name fast path): under + # aggregation_mode=True, spatial_complexity_reduction renames EVERY group's + # representative row, including singleton groups (a bus with exactly one + # flexible load of a given type/sector) - so a representative's own name + # can differ from its single old_name member's name. _disaggregate already + # handles that case correctly (a singleton's one weight, whether zero or + # not, always resolves its share to the representative's full value), so + # there is no correct case left for a by-name fast path to shortcut. + if flexible_cps: + name_map = _old_name_map(reduced_loads_df, flexible_cps) + envelope = reduced_grid.electromobility.flexibility_bands["upper_power"] + _require_full_timeindex(envelope, "electromobility.flexibility_bands") + _disaggregate( + reduced_grid.timeseries.loads_active_power, + name_map, + envelope, + full_grid.timeseries._loads_active_power, + ) + + if flexible_hps: + name_map = _old_name_map(reduced_loads_df, flexible_hps) + members_flat = [m for members in name_map.values() for m in members] + heat_demand = full_grid.heat_pump.heat_demand_df[members_flat] + cop = full_grid.heat_pump.cop_df[members_flat] + p_set = full_loads_df.p_set[members_flat] + envelope = (heat_demand / cop).clip(upper=p_set, axis="columns") + _require_full_timeindex(envelope, "heat_pump.heat_demand_df/cop_df") + _disaggregate( + reduced_grid.timeseries.loads_active_power, + name_map, + envelope, + full_grid.timeseries._loads_active_power, + ) + + if flexible_loads: + name_map = _old_name_map(reduced_loads_df, flexible_loads) + _require_full_timeindex(full_grid.dsm.p_max, "dsm.p_max") + _disaggregate( + reduced_grid.timeseries.loads_active_power, + name_map, + full_grid.dsm.p_max, + full_grid.timeseries._loads_active_power, + ) + + if flexible_storage_units: + # Storage units are never aggregated by spatial_complexity_reduction + # (only bus-relabeled), so this is always a plain by-name write-back. + _write_by_name( + reduced_grid.timeseries.storage_units_active_power, + flexible_storage_units, + full_grid.timeseries._storage_units_active_power, + ) + + full_grid.set_time_series_reactive_power_control() + + return full_grid + + def compare_voltage( edisgo_unreduced: EDisGo, edisgo_reduced: EDisGo, diff --git a/tests/tools/test_spatial_complexity_reduction.py b/tests/tools/test_spatial_complexity_reduction.py index 1747e85ae..256c42330 100644 --- a/tests/tools/test_spatial_complexity_reduction.py +++ b/tests/tools/test_spatial_complexity_reduction.py @@ -3,6 +3,7 @@ from contextlib import nullcontext as does_not_raise import numpy as np +import pandas as pd import pytest from edisgo import EDisGo @@ -396,3 +397,255 @@ def test_remove_short_end_lines(self, test_edisgo_obj): # assert len(edisgo_root.topology.lines_df) - 1 == len( # edisgo_clean.topology.lines_df # ) + + +class TestApplyReducedResultsToFullGrid: + """ + Tests for :func:`~.tools.spatial_complexity_reduction.apply_reduced_results_to_full_grid`. + + Uses stub OPF results (directly writing to + ``reduced_grid.timeseries._loads_active_power`` / + ``_storage_units_active_power``) rather than running a real OPF, since + what is under test is the map-back/disaggregation logic, not + ``pm_optimize`` itself. + """ + + @pytest.fixture(autouse=True) + def test_edisgo_obj(self): + edisgo_root = EDisGo(ding0_grid=pytest.ding0_test_network_path) + edisgo_root.set_time_series_worst_case_analysis() + make_pseudo_coordinates(edisgo_root) + return edisgo_root + + @pytest.fixture + def full_and_reduced(self, test_edisgo_obj): + full_grid = copy.deepcopy(test_edisgo_obj) + reduced_grid, _, _ = full_grid.spatial_complexity_reduction( + copy_edisgo=True, + mode="kmeansdijkstra", + cluster_area="feeder", + reduction_factor=0.1, + aggregation_mode=True, + load_aggregation_mode="bus", + ) + return full_grid, reduced_grid + + def _first_representative_with(self, reduced_grid, n_members): + loads_df = reduced_grid.topology.loads_df + candidates = loads_df[ + loads_df["old_name"].apply( + lambda v: isinstance(v, list) and len(v) == n_members + ) + ] + assert not candidates.empty, ( + f"fixture grid has no aggregated load representative with " + f"exactly {n_members} old_name member(s); adjust the fixture " + f"or reduction_factor." + ) + return candidates.index[0] + + def test_by_name_write_back_aggregation_mode_false(self, test_edisgo_obj): + # aggregation_mode=False: no merging, so restore is a plain by-name + # write-back for every flexibility type. + full_grid = copy.deepcopy(test_edisgo_obj) + reduced_grid, _, _ = full_grid.spatial_complexity_reduction( + copy_edisgo=True, + mode="kmeansdijkstra", + cluster_area="feeder", + reduction_factor=0.1, + aggregation_mode=False, + ) + ti = full_grid.timeseries.timeindex + load_name = reduced_grid.topology.loads_df.index[0] + storage_name = reduced_grid.topology.storage_units_df.index[0] + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[load_name]) + + reduced_grid.timeseries._loads_active_power.loc[ti, load_name] = [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + reduced_grid.timeseries._storage_units_active_power.loc[ti, storage_name] = [ + 5.0, + 6.0, + 7.0, + 8.0, + ] + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, + reduced_grid=reduced_grid, + flexible_loads=[load_name], + flexible_storage_units=[storage_name], + ) + + assert result.timeseries.loads_active_power.loc[ti, load_name].tolist() == [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + assert result.timeseries.storage_units_active_power.loc[ + ti, storage_name + ].tolist() == [5.0, 6.0, 7.0, 8.0] + + def test_accepts_numpy_array_flexible_component_lists(self, test_edisgo_obj): + # Regression test: task_optimize derives flexible_loads as + # edisgo.dsm.p_min.columns.values (a numpy array), unlike the other + # three flexible_* lists which are built with .tolist(). "x or []" + # raises ValueError ("truth value of an array... is ambiguous") for + # any such array with more than one element - a real crash hit on + # the first end-to-end pipeline run using aggregation_mode=False. + full_grid = copy.deepcopy(test_edisgo_obj) + reduced_grid, _, _ = full_grid.spatial_complexity_reduction( + copy_edisgo=True, + mode="kmeansdijkstra", + cluster_area="feeder", + reduction_factor=0.1, + aggregation_mode=False, + ) + ti = full_grid.timeseries.timeindex + load_name = reduced_grid.topology.loads_df.index[0] + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[load_name]) + reduced_grid.timeseries._loads_active_power.loc[ti, load_name] = [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, + reduced_grid=reduced_grid, + flexible_loads=np.array([load_name]), + ) + + assert result.timeseries.loads_active_power.loc[ti, load_name].tolist() == [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + + def test_disaggregation_multi_member_sums_to_representative( + self, full_and_reduced + ): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=2) + members = reduced_grid.topology.loads_df.at[rep, "old_name"] + + p_max = pd.DataFrame(0.0, index=ti, columns=members) + for i, member in enumerate(members): + p_max[member] = [0.1 * (i + 1), 0.0, 0.2 * (i + 1), 0.05 * (i + 1)] + full_grid.dsm.p_max = p_max + + rep_power = pd.Series([1.0, 2.0, 0.0, 3.0], index=ti) + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = rep_power.values + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + total = result.timeseries.loads_active_power.loc[ti, members].sum(axis=1) + assert np.allclose(total.values, rep_power.values) + # Member with zero envelope at t0/t2/t3 gets none of the dispatch; + # both members zero at t1 falls back to an equal split. + assert result.timeseries.loads_active_power.at[ti[1], members[0]] == pytest.approx( + rep_power.iloc[1] / 2 + ) + + def test_disaggregation_singleton_renamed_representative(self, full_and_reduced): + # Regression test: under aggregation_mode=True, spatial_complexity_ + # reduction renames every group's representative, including + # singleton groups (a bus with exactly one flexible load of a given + # type/sector) - so the representative's name can differ from its + # one old_name member's name. A by-name write-back using the + # representative's name would silently miss the real target column + # on full_grid (which only has the original, un-renamed name) and + # create a phantom column instead - this must not happen. + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=1) + member = reduced_grid.topology.loads_df.at[rep, "old_name"][0] + assert rep != member, "fixture assumption: representative was renamed" + + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[member]) + rep_power = pd.Series([7.0, 8.0, 9.0, 10.0], index=ti) + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = rep_power.values + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + assert result.timeseries.loads_active_power.loc[ti, member].tolist() == ( + rep_power.tolist() + ) + assert rep not in result.timeseries.loads_active_power.columns + + def test_raises_clear_error_on_time_index_mismatch(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=1) + member = reduced_grid.topology.loads_df.at[rep, "old_name"][0] + + # dsm.p_max missing the last time step of full_grid's active index. + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti[:-1], columns=[member]) + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + + with pytest.raises(ValueError, match="does not cover"): + spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + def test_storage_units_never_aggregated_always_by_name(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + storage_name = full_grid.topology.storage_units_df.index[0] + assert storage_name in reduced_grid.topology.storage_units_df.index + assert "old_name" not in reduced_grid.topology.storage_units_df.columns + + reduced_grid.timeseries._storage_units_active_power.loc[ti, storage_name] = [ + 1.0, + 1.0, + 1.0, + 1.0, + ] + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, + reduced_grid=reduced_grid, + flexible_storage_units=[storage_name], + ) + assert result.timeseries.storage_units_active_power.loc[ + ti, storage_name + ].tolist() == [1.0, 1.0, 1.0, 1.0] + + def test_reactive_power_recomputed_after_restore(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=1) + member = reduced_grid.topology.loads_df.at[rep, "old_name"][0] + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[member]) + + reactive_before = full_grid.timeseries.loads_reactive_power.loc[ + ti, member + ].copy() + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = [ + 50.0, + 50.0, + 50.0, + 50.0, + ] + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + reactive_after = result.timeseries.loads_reactive_power.loc[ti, member] + assert not reactive_after.equals(reactive_before) From 3456018e9ac0ed80554877187259308837bc8268 Mon Sep 17 00:00:00 2001 From: "Moritz.Schloesser" Date: Wed, 15 Jul 2026 13:36:48 +0000 Subject: [PATCH 46/66] fix: trim electromobility flexibility bands to the active time index get_flexibility_bands only resamples bands to the active time-series FREQUENCY, not the active date range - so after a manual/short select_timesteps window, flexibility_bands kept spanning the full underlying SimBEV data range with no later step ever trimming it down. Any consumer indexing flexibility_bands by edisgo.timeseries.timeindex (e.g. the OPF's charging-point constraint builder) could then hit missing time steps. build_flexibility_bands now trims electromobility data to the active time index right after building the bands. --- edisgo/run/tasks/flex.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py index 35cce7dce..9c3edab80 100644 --- a/edisgo/run/tasks/flex.py +++ b/edisgo/run/tasks/flex.py @@ -216,6 +216,14 @@ def task_build_flexibility_bands(edisgo, ctx, *, use_case=None): ``import_heat_pumps``, and is more efficient than building bands over a non-final index. + ``get_flexibility_bands`` only resamples to the active *frequency* - the + bands still span whatever date range the underlying SimBEV charging- + process data covers, which is not necessarily the same range as + ``edisgo.timeseries.timeindex`` (e.g. a manually-selected window). This + task additionally trims the bands down to that exact index, so + ``electromobility.flexibility_bands`` always matches + ``edisgo.timeseries.timeindex`` after this step runs. + Parameters ---------- edisgo : edisgo.EDisGo @@ -232,9 +240,20 @@ def task_build_flexibility_bands(edisgo, ctx, *, use_case=None): edisgo.EDisGo The modified EDisGo instance. """ + from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex + if use_case is None: use_case = ["home", "work", "public", "hpc"] edisgo.electromobility.get_flexibility_bands(edisgo, use_case=use_case) + reduce_timeseries_data_to_given_timeindex( + edisgo, + edisgo.timeseries.timeindex, + timeseries=False, + electromobility=True, + heat_pump=False, + dsm=False, + overlying_grid=False, + ) return edisgo From b9c3d8c2f38fa3d5dcc2c30cc38c7d8beb41d0ee Mon Sep 17 00:00:00 2001 From: "Moritz.Schloesser" Date: Wed, 15 Jul 2026 13:36:57 +0000 Subject: [PATCH 47/66] Add uc6_spatial_reduction preset and example runner Standalone preset wiring spatial_reduce/spatial_restore around optimize, disabled by default via spatial_reduction.enabled. run_example_06.py runs it against a real ding0 grid. --- edisgo/run/presets/uc6_spatial_reduction.yaml | 149 ++++++++++++++++++ run_example_06.py | 39 +++++ 2 files changed, 188 insertions(+) create mode 100644 edisgo/run/presets/uc6_spatial_reduction.yaml create mode 100644 run_example_06.py diff --git a/edisgo/run/presets/uc6_spatial_reduction.yaml b/edisgo/run/presets/uc6_spatial_reduction.yaml new file mode 100644 index 000000000..514a84589 --- /dev/null +++ b/edisgo/run/presets/uc6_spatial_reduction.yaml @@ -0,0 +1,149 @@ +_comment: | + UC5 — OPF with configurable timestep selection (manual OR auto), PLUS spatial + complexity reduction bracketing the OPF step. Standalone copy of + uc5_select_timesteps.yaml (not an `extends` overlay) with the spatial_reduce / + spatial_restore bracket added around `optimize`. + + The pipeline carries TWO select_timesteps steps, each with a `position`: + - position: pre_import (before import_heat_pumps) — acts only in MANUAL + mode. It sets the explicit time index, which the heat-pump/DSM imports + and oedb_ts then use to fetch only the selected steps (cheap). + - position: post_grid (after import_overlying_grid_data, before + reactive_power) — acts only in AUTO mode. It needs all active-power + time series (incl. overlying-grid generation) set to run the scoring + power flow via get_most_critical_time_intervals. + Whichever mode is configured, the other positioned step is a no-op. + + Auto mode normally yields two disconnected intervals (one overloading, + one voltage). They are kept separate (a gap in the time index); if they + overlap, a non-overlapping pair is chosen if possible, otherwise they are + concatenated into one interval. A later optimize step can detect the gap + and run separate optimizations per interval. + + Spatial complexity reduction (spatial_reduce / spatial_restore) brackets + `optimize` only: + - spatial_reduce (before optimize): deepcopies and stashes the full grid on + ctx, then spatially reduces the working object so `optimize` runs on a + smaller grid. + - spatial_restore (after optimize): writes the optimized flexible-component + dispatch back onto the stashed full grid (by name, or disaggregated onto + `old_name` members if aggregation_mode is true), recomputes reactive power + for those components, and makes the full grid active again for `reinforce`. + Both are no-ops when `spatial_reduction.enabled` is false (default), so + `reinforce` then runs on the same grid `optimize` used, same as + uc5_select_timesteps.yaml. `reactive_power` (pre-OPF, full time series) stays + where it already was, unaffected by the spatial bracket. + Reinforcement always runs on the full topology, regardless of the flag. + +_workflow: + - setup_grid: load ding0 topology + - import_generators / import_home_batteries + - select_timesteps (pre_import): manual only — set explicit time index + - import_heat_pumps / import_dsm: fetch only selected steps (manual) + - import_electromobility: dumb charging, flex bands + - oedb_ts: real wind/solar + load time series + - apply_charging_strategy / apply_heat_pump_strategy + - build_flexibility_bands: EV bands on the fixed hourly index + - import_overlying_grid_data: HV constraints from CSV dir + - select_timesteps (post_grid): auto only — reduce to critical intervals + - reactive_power: fixed cosphi on the reduced index + - spatial_reduce: no-op unless spatial_reduction.enabled — stash full grid, + reduce working object + - optimize: pm_optimize with flex assets, on the (possibly) reduced grid + - spatial_restore: no-op unless spatial_reduction.enabled — write dispatch + back onto the stashed full grid, recompute reactive power + - reinforce / save: always on the full topology + +# Self-contained (no `extends`): everything uc4_example provided is inlined +# below, so this preset can be run on its own via +# run_edisgo({"extends": "uc5_spatial_reduction", "grid": {"ding0_path": ...}}) +scenario: eGon2035 + +grid: + ding0_path: "/path/to/ding0_grid" + legacy_ding0_grids: false + +database: + source: local + + +# No explicit base time index is set. oedb_ts falls back to a full year derived +# from the scenario when none is given, which is what auto interval selection +# needs (week-long critical intervals to pick from). For manual selection the +# pre-import select_timesteps step sets the index instead. + +overlying_grid: + enabled: true # set true to activate import_overlying_grid_data + source: csv # "csv" (load from path) or "etrago" (kwarg) + path: "/storage/JoDa/edisgo_playground/overlying_grid_data" + +results: + directory: results/uc5_spatial_reduction + +# Top-level block read by the select_timesteps task via ctx.raw_config. +# eGo can inject this block the same way it injects overlying_grid. +# Set `mode` (and its parameters) here or override it in the run script. +timeseries_selection: + mode: manual + # auto method: "power_flow" (default, scores intervals via a power flow) or + # "residual_load" (no power flow — the weeks ending at the max/min residual-load + # time steps; requires overlying-grid data). + method: residual_load + # --- shared auto parameters (both methods) --- + time_steps_per_time_interval: 168 # one week (must be a multiple of 24) + time_step_day_start: 4 # hour of day the intervals start/end on + # --- power_flow method parameters --- + percentage: 1.0 + save_steps: true # write selected intervals CSV to results_dir + use_troubleshooting_mode: true # handle power-flow non-convergence + overloading_factor: 0.95 + voltage_deviation_factor: 0.95 + # --- manual parameters (used when mode: manual) --- + # timestamps: ["2035-01-15 08:00", "2035-01-15 9:00", "2035-01-15 10:00"] + # or a range instead of `timestamps`: + start: "2035-01-15 00:00" + periods: 24 + freq: h + +# Top-level block read by the spatial_reduce/spatial_restore tasks via +# ctx.raw_config. eGo can inject this block the same way it injects +# overlying_grid/timeseries_selection (global `spatial_reduction` default + +# per-grid `spatial_reduction_per_grid` override keyed by mv_grid_id). +spatial_reduction: + enabled: true # set true to activate spatial_reduce/spatial_restore + mode: kmeansdijkstra # clustering mode for spatial_complexity_reduction + cluster_area: feeder + reduction_factor: 0.3 + reduction_factor_not_focused: False + aggregation_mode: flase # start with false; true enables load/generator merging + +pipeline: + - setup_grid + - import_generators + - import_home_batteries + - select_timesteps: {position: pre_import} # acts in manual mode only + - import_heat_pumps + - import_dsm + - import_electromobility: + charging_strategy: null + # flexibility bands are built later (build_flexibility_bands), once the + # analysis time index is fixed, so they are resampled to it + - oedb_ts: + dispatchable: {other: 0.7} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - build_flexibility_bands # hourly bands on the 2035 index + - import_overlying_grid_data + - select_timesteps: {position: post_grid} # acts in auto mode only + - reactive_power + - spatial_reduce # no-op unless spatial_reduction.enabled + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - spatial_restore # no-op unless spatial_reduction.enabled + - reinforce: + catch_convergence_problems: true + - save: + archive: true + save_opf_results: true diff --git a/run_example_06.py b/run_example_06.py new file mode 100644 index 000000000..16425a1dc --- /dev/null +++ b/run_example_06.py @@ -0,0 +1,39 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Runner für uc6_spatial_reduction.yaml — einfach ``python run_example_05.py``.""" + +import logging + +from edisgo.run.runner import run_edisgo + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(name)s %(levelname)s: %(message)s", +) + +# edisgo = run_edisgo("/storage/JoDa/ego/edisgo_run_edisgo/eDisGo/edisgo/run/presets/uc4_example_MS.yaml") # noqa: E501 +edisgo = run_edisgo( + { + "extends": "uc6_spatial_reduction.yaml", + # "grid": {"ding0_path": "/home/gurobi/.ding0/run_hetzner_59763_2023_04_06/ding0_grids/32355"} # noqa: E501 + "grid": { + "ding0_path": "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" # noqa: E501 + }, + # OG path must be the leaf dir for THIS grid (like ding0_path), not the parent. + "overlying_grid": { + "path": "/home/gurobi/.edisgo_input/overlying_grid/32377" + }, + } +) + +print("\n=== Fertig ===") +print("Ausbaukosten:\n", edisgo.results.grid_expansion_costs) +print("\nUngelöste Probleme:\n", edisgo.results.unresolved_issues) From fe3a15661f60fed91a0fc6089cbaf593df11e239 Mon Sep 17 00:00:00 2001 From: "Moritz.Schloesser" Date: Wed, 15 Jul 2026 13:37:26 +0000 Subject: [PATCH 48/66] docs: spatial reduction glossary and grilling-session notes CONTEXT.md gains a glossary section for spatial reduction terms (busmap, disaggregation rule, config surface, ordering). docs_notes/ carries the full design-session record plus two follow-up issue write-ups: the flexibility-bands time-index gap (fixed) and the aggregation_mode=True + charging-points gap (not yet fixed, tracked for later). --- CONTEXT.md | 92 +++++ ...n_mode_flexibility_bands_not_aggregated.md | 83 +++++ ...ue_temporal_reduction_flexibility_bands.md | 81 +++++ .../spatial_reduction_grilling_session.md | 332 ++++++++++++++++++ 4 files changed, 588 insertions(+) create mode 100644 CONTEXT.md create mode 100644 docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md create mode 100644 docs_notes/issue_temporal_reduction_flexibility_bands.md create mode 100644 docs_notes/spatial_reduction_grilling_session.md diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 000000000..7e923e900 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,92 @@ +# Context / Glossary + +Domain vocabulary for the eDisGo run pipeline. Glossary only — no implementation +details, no decisions (those live in `docs/adr/`). + +## Complexity reduction + +- **Spatial complexity reduction** — merging nearby buses into a smaller set of + representative buses (clustering *along the grid*, keeping it radial) to shrink the + grid for faster power flow / optimization. Implemented by + `EDisGo.spatial_complexity_reduction`, which builds a *busmap* + *linemap* and mutates + the Topology. Used only to accelerate the optimization; reinforcement runs on the + **full grid**. +- **Temporal complexity reduction** — keeping only grid-critical time steps/intervals + instead of the full year. See the `select_timesteps` task. +- **Busmap** — DataFrame mapping each original bus to its clustered *new_bus* (with new + coordinates). Index = original bus names. +- **Linemap** — DataFrame mapping original line names to *new_line_name* after lines are + recalculated/merged. +- **Reduced grid** — the spatially-reduced EDisGo object the OPF runs on. +- **Full grid** — the original, unreduced EDisGo object; reinforcement always runs here. +- **Map-back / restore** — writing the OPF flexibility dispatch from the reduced grid + onto the full grid. Only the components the OPF *rewrites* are mapped back — flexible + charging points, heat pumps, DSM loads, and storage. Inflexible loads/generators are + **skipped** (the OPF does not change their series; the full grid already holds them + correctly). Implemented by core function + `tools/spatial_complexity_reduction.py::apply_reduced_results_to_full_grid(full_grid, + reduced_grid, *, flexible_cps=None, flexible_hps=None, flexible_loads=None, + flexible_storage_units=None)` + thin wrapper `EDisGo.map_reduced_results_to_full_grid`. + Provenance for disaggregation comes solely from `old_name` on the reduced grid's + `loads_df`/`generators_df` — no busmap/linemap stash needed on `ctx`. + - `aggregation_mode=False`: components keep their names → write back **by component + name**. + - `aggregation_mode=True`: loads/generators may be merged into a representative + (originals recorded in `old_name`; storage is never aggregated). The representative's + optimized series is **disaggregated** onto its `old_name` members. +- **Disaggregation rule** — split a merged representative's optimized series onto its + original members **per time step**, weighted by each member's own *pre-OPF flexibility + envelope* (a known input, never the optimized result): `upper_power(t)` band for + charging points, heat-demand/thermal envelope for heat pumps, `p_max(t)` band for DSM. + Per-step weighting means a charging point only receives power at steps where it has a + connected vehicle (`upper_power(t) > 0`). Sums back to the representative exactly at + every step; equal split as the zero-envelope fallback. +- **Reactive power on restore** — `spatial_restore` writes active power only, then calls + `EDisGo.set_time_series_reactive_power_control()` itself (plain fixed-cosphi default), + mirroring exactly how `pm_optimize`'s own results-writer + (`io/powermodels_io.py::from_powermodels`) handles it: write P, then blanket-recompute Q. + No bespoke reactive-power logic — proportional-split and recompute are mathematically + identical under fixed-cosphi since all `old_name` members of one representative share + the same `power_factor`. + +## Pipeline tasks (spatial reduction) + +- **spatial_reduce** — task run *before* `optimize`: deepcopy the full grid, stash it, + and spatially reduce the working object so `optimize` runs on the reduced grid. +- **spatial_restore** — task run *after* `optimize`: write the optimized dispatch time + series back onto the stashed full grid and make it active again. **Optional** — a run + may legitimately stop after the reduced-grid optimization when only the derived time + series matter. +- **Full-grid stash** — the deepcopied full grid kept in memory on `ctx` between + `spatial_reduce` and `spatial_restore`. Held in-memory (matching legacy eGo, which kept + both full and reduced grids resident during optimize). Persisting it to a disk artifact + to lower peak memory is a possible later optimization, not the initial design. + +## Config surface (spatial reduction) + +- Top-level YAML block `spatial_reduction:` (mirrors `timeseries_selection:`), holding + `mode`, `cluster_area`, `reduction_factor`, `reduction_factor_not_focused`, + `aggregation_mode`, and aggregation sub-modes. Read inside the `spatial_reduce` task via + `ctx.raw_config.get("spatial_reduction", {})`. +- eGo injects it exactly like `timeseries_selection`: a global `spatial_reduction` default + plus a `spatial_reduction_per_grid` dict keyed by `str(mv_grid_id)`, whole-block + replacement (not a field-level merge). If neither is set the key is omitted entirely and + the eDisGo preset's own default applies. + +## Ordering (spatial reduction) + +Pipeline order: `select_timesteps` → `spatial_reduce` → `optimize` → `spatial_restore` +→ `reinforce`. + +- Spatial reduction touches only **topology**; temporal reduction touches only the + **time index** — orthogonal operations that commute as *mechanisms*. Spatial reduction + works on any time index, including full-year (clustering depends on coordinates/graph + distance, not the time series; the `reduction_factor_not_focused` worst-case power flow + runs on an internal deepcopy and does not disturb the working series). +- But the **pipeline** pins `select_timesteps` before `spatial_reduce`: the full-grid + stash inherits whatever time index exists at deepcopy time, and reinforce runs on that + stash. Reducing timesteps first means the stash (and therefore reinforce) carries the + reduced index — full **topology**, reduced **time index**. Reversing the two would + force separately re-reducing the stash's index. +- `spatial_restore` does **no time-index surgery** — it only writes flexible-component + dispatch back onto the stashed full grid. diff --git a/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md b/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md new file mode 100644 index 000000000..37be52abe --- /dev/null +++ b/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md @@ -0,0 +1,83 @@ +# `spatial_complexity_reduction(aggregation_mode=True)` doesn't aggregate `electromobility.flexibility_bands`, breaking `optimize` on merged charging points + +**Type:** bug +**Found:** 2026-07-15, running the real `uc6_spatial_reduction.yaml` pipeline +on grid 32377 with `spatial_reduction.aggregation_mode: true`. +**Affects:** `spatial_complexity_reduction`/`apply_busmap` +(`edisgo/tools/spatial_complexity_reduction.py`), specifically the load +aggregation step; consumed by `_build_electromobility` +(`edisgo/io/powermodels_io.py`). + +## Problem + +When `aggregation_mode=True`, `apply_busmap` merges loads at the same bus +(`aggregate_loads_df`, `spatial_complexity_reduction.py:1587-1599`) and +correctly aggregates their **time series** via `aggregate_timeseries` +(`spatial_complexity_reduction.py:1718`, called for `loads_active_power` / +`loads_reactive_power`). This works fine for DSM loads and heat pumps, whose +OPF constraints are read directly from `loads_df`/`dsm.p_max`/ +`heat_pump.heat_demand_df` keyed by whatever name is currently in `loads_df`. + +Charging points are different: the OPF's own constraint builder, +`_build_electromobility` (`edisgo/io/powermodels_io.py:1212`), does **not** +read its upper-power bound from `loads_df` — it reads +`electromobility.flexibility_bands["upper_power"][cp_name]`, a separate +DataFrame keyed by charging-point name. `spatial_complexity_reduction` never +touches `flexibility_bands` during aggregation, so it still only has entries +for the *original*, pre-merge charging-point names. + +When a bus has 2+ charging points and gets merged into one representative +load (e.g. `Load_Bus_mvgd_32377_F2_B2_charging_point_hpc`), that +representative name has: +- a correctly-aggregated `loads_active_power` entry (summed from the + originals), but +- **no** entry at all in `flexibility_bands["upper_power"]`. + +`task_optimize` derives `flexible_cps` from `loads_df` (filtering +`type == "charging_point"`), so it passes the representative's name into +`pm_optimize` → `to_powermodels` → `_build_electromobility`, which then does +`flex_bands_df["upper_power"][emob_df.index[cp_i]]` and raises `KeyError` +for the representative name. + +## Reproduction + +Run `uc6_spatial_reduction.yaml` (or any preset with the spatial bracket) on +a grid with 2+ charging points sharing a bus, with +`spatial_reduction: {enabled: true, aggregation_mode: true, +load_aggregation_mode: bus}` and `optimize: {flexible: [..., +charging_points, ...]}`. Crashes inside `optimize`, before `spatial_restore` +ever runs: + +``` +KeyError: 'Load_Bus_mvgd_32377_F2_B2_charging_point_hpc' + .../edisgo/io/powermodels_io.py:1212, in _build_electromobility + * flex_bands_df["upper_power"][emob_df.index[cp_i]].iloc[0] +``` + +## Scope note + +This is independent of the spatial-reduction pipeline-integration work +(`spatial_reduce`/`spatial_restore`/`apply_reduced_results_to_full_grid`) — +it's a gap in `spatial_complexity_reduction` itself (the reduction side), +already reachable via `EDisGo.spatial_complexity_reduction` + +`EDisGo.pm_optimize` directly, with no pipeline involved. It only manifests +under `aggregation_mode=True` with charging points present; `aggregation_ +mode=False` is unaffected since every component keeps its original name +there, and `flexibility_bands` stays valid. + +Also worth checking as part of the same fix: whether heat pumps have an +analogous issue if the OPF ever reads a heat-pump-specific band keyed +differently from `loads_df` (current understanding, from the spatial- +reduction design session, is that heat pumps use `heat_demand_df`/`cop_df`/ +`loads_df.p_set` directly, which DO get aggregated correctly — but worth +double-checking with a merged multi-heat-pump bus once this issue is picked +up, in case some other heat-pump-specific structure has the same gap). + +## Suggested fix + +Extend `apply_busmap`'s load-aggregation step to also aggregate +`electromobility.flexibility_bands` (`upper_power`, `lower_energy`, +`upper_energy`) onto the same representative name, using the same +`old_name`-based grouping/summing already used for `aggregate_timeseries` +— summing `upper_power` per merged group is the natural analogue of summing +`p_set`/active power for the representative. diff --git a/docs_notes/issue_temporal_reduction_flexibility_bands.md b/docs_notes/issue_temporal_reduction_flexibility_bands.md new file mode 100644 index 000000000..5aac8caca --- /dev/null +++ b/docs_notes/issue_temporal_reduction_flexibility_bands.md @@ -0,0 +1,81 @@ +# Flexibility bands should be built scoped to the selected time index, not built-then-trimmed + +**Type:** design gap / bug +**Found:** 2026-07-15, while testing spatial complexity reduction against a +manual-mode `select_timesteps` run (`uc6_spatial_reduction.yaml`). +**Affects:** `task_build_flexibility_bands` (`edisgo/run/tasks/flex.py`), +`Electromobility.get_flexibility_bands` (`edisgo/network/electromobility.py`), +and by extension any other per-component band/envelope built after +`select_timesteps` (heat pump `heat_demand_df`/`cop_df`, DSM `p_max`/`p_min`). + +## Problem + +`select_timesteps` (manual mode, `position: pre_import`) fixes +`edisgo.timeseries.timeindex` to an arbitrary, possibly short window (e.g. 24 +hours starting `2035-01-15`) early in the pipeline, before electromobility +data is even imported. + +`build_flexibility_bands` runs later and calls +`Electromobility.get_flexibility_bands(edisgo, use_case=...)`. Per that +method's own docstring, `resample=True` only "resamples the bands to the same +**frequency** as time series data in the `TimeSeries` object" — it does +*not* clip the bands to the same *date range*. The bands are built from the +raw SimBEV charging-process data and keep whatever date range that data +spans, matching the target row spacing (e.g. hourly) but not the target +window. + +Nothing downstream re-trims `electromobility.flexibility_bands` to the +selected 24-hour window afterward. The mismatch went unnoticed until a piece +of code tried to actually index `flexibility_bands` using +`edisgo.timeseries.timeindex` and hit a `KeyError`-shaped failure (missing +time steps) — in this case, the new +`apply_reduced_results_to_full_grid`/`spatial_restore` disaggregation logic, +which reads `flexibility_bands["upper_power"]` as a per-charging-point, +per-time-step weighting envelope. + +This is a **pre-existing gap**, not something introduced by spatial +reduction — it already exists in `uc5_select_timesteps.yaml` (the preset +`uc6_spatial_reduction.yaml`/`uc5_spatial_reduction.yaml` were both derived +from). It simply had no consumer that indexed `flexibility_bands` by the +active time index before now. + +## Immediate fix applied (unblocks spatial-reduction testing) + +`task_build_flexibility_bands` now calls +`reduce_timeseries_data_to_given_timeindex(edisgo, edisgo.timeseries.timeindex, +electromobility=True, timeseries=False, heat_pump=False, dsm=False, +overlying_grid=False)` right after `get_flexibility_bands`, trimming +`flexibility_bands` down to the active index. See +`edisgo/run/tasks/flex.py::task_build_flexibility_bands`. + +This is a workaround (build full, then trim), not the better design below. + +## Suggested proper fix (not yet implemented — this issue) + +Build flexibility bands (and, likely, heat-pump/DSM bands) scoped to the +already-selected time index from the start, rather than building over the +full/native data range and trimming afterward: + +- `Electromobility.get_flexibility_bands` (or its caller) should accept/use + the target time index *before* running the difference-array band + construction, so the SimBEV charging-process data outside that window is + never even considered. +- Audit whether `HeatPump`/`DSM` band construction (wherever their + time-varying bounds are first populated — likely in the `import_heat_pumps` + / `import_dsm` tasks or their underlying `edisgo/io/*` importers) has the + same built-on-full-range-then-maybe-trimmed pattern, since + `reduce_timeseries_data_to_given_timeindex` already has `heat_pump=True`/ + `dsm=True` flags suggesting this was anticipated but may not be + consistently invoked at the right point in every pipeline path. +- Consider whether this should be a single, explicit "finalize time index" + pipeline hook that every band-producing task can rely on having already + run, rather than each task needing to remember to trim itself. + +## Reproduction + +Run `uc6_spatial_reduction.yaml` (or `uc5_select_timesteps.yaml`) with +`timeseries_selection: {mode: manual, start: "2035-01-15 00:00", periods: +24, freq: h}` and `overlying_grid.enabled: true`, then inspect +`edisgo.electromobility.flexibility_bands["upper_power"].index` after +`build_flexibility_bands` runs — before the fix above, its date range does +not match `edisgo.timeseries.timeindex`. diff --git a/docs_notes/spatial_reduction_grilling_session.md b/docs_notes/spatial_reduction_grilling_session.md new file mode 100644 index 000000000..b90ddb190 --- /dev/null +++ b/docs_notes/spatial_reduction_grilling_session.md @@ -0,0 +1,332 @@ +# Spatial complexity reduction — pipeline integration design (grilling session) + +**Status:** DESIGN CLOSED. All questions resolved 2026-07-14. Next: implement, and decide +the two ADR candidates below. +**Date:** 2026-07-09 (started), closed 2026-07-14. +**Repo/branch:** `/storage/MS/ego/eDisGo`, branch `edisgo_run_edisgo`. +**Companion files:** `CONTEXT.md` (glossary, at repo root), this file. + +--- + +## Goal + +Integrate eDisGo's **spatial complexity reduction** into the run pipeline, as a +counterpart to the temporal complexity reduction (timestep selection) already added. + +Spatial reduction merges nearby buses into representative buses to shrink the grid so +the **optimization (OPF)** runs faster. **Reinforcement must run on the FULL grid** +(full topology) — but on the *reduced* time index (temporal reduction still applies). + +--- + +## How spatial reduction works (established from code) + +- `EDisGo.spatial_complexity_reduction()` (edisgo.py:3439) wraps + `tools/spatial_complexity_reduction.py::spatial_complexity_reduction()` (line 1830). +- It builds a **busmap** (original bus → clustered `new_bus`) + **linemap**, then mutates + the Topology in place (or on a copy if `copy_edisgo=True`). Returns `(edisgo, busmap_df, + linemap_df)`. +- Clustering uses coordinates / grid-graph distance — **independent of the time index**. + Works on a full-year grid too. `apply_pseudo_coordinates=True` (default) fills missing + coords. +- **Storage is never aggregated** (only bus-relabeled) — keeps its name/identity even + with `aggregation_mode=True` (spatial_complexity_reduction.py:1756–1760). +- **Loads/generators ARE merged** when `aggregation_mode=True` (lines 1699–1754), grouped + by bus(+type+sector). Originals recorded in an **`old_name`** column on the reduced + component rows; their series summed via `aggregate_timeseries`. +- `reduction_factor_not_focused` uses `find_buses_of_interest` which runs a worst-case + power flow — but on an **internal deepcopy** (line 93), so it does NOT disturb the + working object's time series. + +### Legacy eGo reference (the pattern we are re-implementing cleanly) +`eGo/ego/tools/edisgo_integration.py::_run_edisgo_task_optimisation` (~line 1632): +- `edisgo_copy = deepcopy(edisgo_grid)` (full grid stays in `edisgo_grid`) +- temporal-reduce copy → spatial-reduce copy → `pm_optimize` on copy +- write dispatch back onto the full grid **by component name** (lines 1763–1795): + loads/generators/storage active+reactive power, sliced by `time_steps` +- `edisgo_grid.timeseries.timeindex = timeindex` (union of optimized intervals) +- reinforce runs on the full grid. Both grids were resident in memory simultaneously. + +--- + +## Decisions made (confirmed with user) + +1. **Two bracketing tasks (Option A), NOT inside `pm_optimize`.** + - `spatial_reduce` (before `optimize`) and `spatial_restore` (after `optimize`). + - Rationale: reduce/restore are topology operations, not OPF concerns; must be + visible/toggleable in YAML; enables stopping after the reduced-grid OPF when only + the derived time series matter. + - This deliberately breaks the "all logic inside pm_optimize" principle we used for + the multi-interval split — justified by the stop-early capability. **ADR candidate.** + +2. **`spatial_restore` is OPTIONAL** — a run may stop after optimizing on the reduced + grid. (But when present it must follow optimize; see validator note.) + +3. **Full-grid stash kept IN MEMORY on `ctx`** (matches legacy, which held both grids + resident). Disk-artifact persistence to cut peak memory is a deferred optimization, + not v1. (Runner does support disk reload via `save` + `stage_artifacts` + `load_from`, + but we are not using it here.) + +4. **Aggregation support:** implement `aggregation_mode=False` FIRST, then design so + `aggregation_mode=True` follows. + +5. **Map-back only touches components the OPF rewrites:** flexible charging points, heat + pumps, DSM loads, and storage. **Inflexible loads/generators are SKIPPED** — the OPF + doesn't change them; the full grid already holds their correct series. + - `aggregation_mode=False`: write back **by component name**. + - `aggregation_mode=True`: disaggregate the representative's series onto its `old_name` + members. + +6. **Disaggregation rule (aggregation_mode=True):** split the representative's optimized + series onto original members **per time step**, weighted by each member's own **pre-OPF + flexibility envelope** (a known input, never the optimized result): + - charging points → `electromobility.flexibility_bands["upper_power"][cp_name]` (a CP + with no connected vehicle has `upper_power(t)=0`, so it receives no charge that step + — physically correct); + - heat pumps → `weight(t) = min(heat_demand_df[hp_name][t] / cop_df[hp_name][t], + loads_df.p_set[hp_name])`. **Refined during implementation (2026-07-14):** the + original "heat-demand/thermal envelope" phrasing was underspecified. Investigated + `edisgo/io/powermodels_io.py::_build_heatpump` (~lines 1259-1282): the OPF's actual + per-unit electrical cap is the CONSTANT rated power `loads_df.p_set`, not a + time-varying series — the genuinely time-varying pre-OPF quantity is + `heat_demand_df` (thermal, MW) divided by `cop_df` (electrical-equivalent demand). + Capping that ratio at each member's own `p_set` mirrors charging points exactly (a + CP's `upper_power(t)` is already a capped bound, not raw uncapped vehicle demand) and + ensures no member is ever assigned a share exceeding what it could physically draw; + - DSM → `dsm.p_max[load_name][t]` band (`edisgo/network/dsm.py:63`). + - All three sources share the same shape: rows = timestamps matching + `TimeSeries.timeindex`, columns = component names matching `Topology.loads_df.index`. + - Sums back to the representative exactly at each step; equal split as zero-envelope + fallback. (User explicitly preferred a time-series-based split over a static scalar, + because these envelopes are known pre-OPF and reflect actual flexibility-relevant + events, e.g. a connected vehicle or nonzero heat demand.) + +7. **Ordering:** `select_timesteps` → `spatial_reduce` → `optimize` → `spatial_restore` + → `reinforce`. + - The two reduction *mechanisms* commute (orthogonal: topology vs time index), BUT the + pipeline pins `select_timesteps` before `spatial_reduce` so the stashed full grid + (hence reinforce) inherits the **reduced** time index. Result: reinforce on full + **topology** × reduced **time index**. + - `spatial_restore` does **no time-index surgery** — only writes flexible dispatch back. + +8. **Tasks stay thin; computation lives in eDisGo core** (same principle as the + timestep-selection refactor). See open question for how this applies to restore. + +--- + +## Where we paused — OPEN QUESTION (resume here) + +Applying "tasks are thin wrappers" to the restore half. Established: +- **Reduction half is already correct:** `spatial_complexity_reduction()` is already a + self-contained core function + EDisGo method. `spatial_reduce` task just needs to + deepcopy+stash the full grid and call it. Nothing to extract. +- **Restore half is the gap:** there is **NO** existing core function that maps reduced + OPF results back onto a full grid (the legacy logic lived inline in eGo's private + method; `_restore_pristine_inputs` in powermodels_opf.py:225 is unrelated — it's the + multi-interval snapshot/restore). + +**Proposal put to the user (awaiting confirmation):** +- (a) Create a NEW core function, e.g. + `tools/spatial_complexity_reduction.py::apply_reduced_results_to_full_grid(full_grid, + reduced_grid, *, flexible_cps, flexible_hps, flexible_loads, flexible_storage_units)` + + a thin `EDisGo` method wrapper (mirroring `spatial_complexity_reduction`). The task + `spatial_restore` just reads the stashed full grid + flexible sets from `ctx` and calls + it. Used **outside** the pipeline, a caller passes `full_grid`, `reduced_grid`, and the + flexible sets directly (no `ctx`). This gives symmetry: both halves = core fn + method + wrapper + thin task; both usable standalone; disaggregation rule lives/tested in core. +- (b) **`old_name`** carried on the reduced grid's `loads_df`/`generators_df` is + sufficient provenance for disaggregation — the reduced grid self-describes its origins, + so **no busmap needs stashing**. Full grid needed as the write target (holds individual + members + their pre-OPF weighting envelopes); reduced grid supplies optimized series + + `old_name`. Both grids are required args. + +**User's last message (the prompt to answer):** agrees restore logic should NOT live in +the task and should become its own eDisGo function; flexible-component names stored in +`ctx` and passed to the function, or passed differently when used outside the pipeline; +reduction is already an independent eDisGo function. + +→ So (a) and (b) are essentially aligned with the user's view; next step is to CONFIRM the +signature details (both grids as args; `old_name` sufficient, no busmap) and then move on. + +**RESOLVED (2026-07-14):** +- Core function signature: + `apply_reduced_results_to_full_grid(full_grid, reduced_grid, *, flexible_cps=None, + flexible_hps=None, flexible_loads=None, flexible_storage_units=None)` — four separate + kwargs, one per flexible-component type, each defaulting to `None`/skip. +- `EDisGo` method wrapper name: `map_reduced_results_to_full_grid` (full symmetry with the + core function name, no abbreviation). +- `old_name` on the reduced grid's `loads_df`/`generators_df` is CONFIRMED sufficient + provenance for disaggregation. No busmap/linemap stash on `ctx`. + +--- + +## Remaining questions still to grill (not yet discussed) + +- ~~Validator ordering~~ — **RESOLVED (2026-07-14).** Investigated + `edisgo/run/validator.py` (`validate()`, lines 47-139) + `edisgo/run/registry.py` + (`TaskMeta`, `register_task()`): ordering today is capability-based (`requires`/ + `provides` sets accumulated linearly across the pipeline, `validator.py:96,118-130`), + NOT a dependency graph and NOT named task-to-task precedence. The one existing hardcoded + exception is `reactive_power` must be last among `ts_altering` tasks + (`validator.py:110-116`). `select_timesteps`'s optional dual-position behavior + (`timeseries.py:328-403`) is NOT validator-enforced — it's a runtime-only check inside + the task, so it was not usable as a precedent. + - **Decision:** extend the existing capability system rather than add a new validator + concept or fall back to runtime-only checking (matches the pipeline's existing + mechanism everywhere else): + - `spatial_reduce` declares `provides={"reduced_grid"}`. + - `optimize` declares `provides={"optimized_dispatch", ...}` (in addition to its + existing provides). + - `spatial_restore` declares `requires={"reduced_grid", "optimized_dispatch"}`. + - This closes the gap where presence-only capability accumulation would otherwise let + `spatial_restore` validate successfully even if placed before `optimize` (both + `reduced_grid`-derived requirements would already be "satisfied" from + `spatial_reduce` alone) — requiring `optimized_dispatch` too means `spatial_restore` + cannot pass validation until `optimize` has actually appeared earlier in the + pipeline. + - **Correction (2026-07-14, during implementation):** `register_task`'s `requires`/ + `provides` (`edisgo/run/registry.py:52-58`) are fixed at decoration time (module + load), NOT evaluated per-run — so "optimize requires `reduced_grid` only when spatial + reduction is configured for THIS run" is not expressible and was dropped. + `optimize`'s `requires` stays exactly `{"timeseries", "flex"}`, unchanged — it does + not need to know spatial reduction exists. Ordering is fully enforced from + `spatial_restore`'s side alone; adding `reduced_grid` to `optimize`'s `requires` + unconditionally would have broken every existing preset that runs `optimize` without + `spatial_reduce` (uc2, uc4, uc5_select_timesteps). +- ~~YAML config surface~~ — **RESOLVED (2026-07-14).** Top-level `spatial_reduction:` + block, mirroring `timeseries_selection:`. Read via + `ctx.raw_config.get("spatial_reduction", {})` inside the `spatial_reduce` task, same + pattern as `select_timesteps` (`timeseries.py:415`). Holds `mode`, `cluster_area`, + `reduction_factor`, `reduction_factor_not_focused`, `aggregation_mode`, aggregation + sub-modes. +- ~~eGo injection~~ — **RESOLVED (2026-07-14).** Verified `timeseries_selection`'s actual + injection in `EDisGoNetworks._build_run_edisgo_config()`, + `eGo/ego/tools/edisgo_integration.py:675-685`: global `timeseries_selection` default + + `timeseries_selection_per_grid` dict keyed by `str(mv_grid_id)` + (`edisgo_integration.py:681-684`), **whole-block replacement** (not field-level merge), + key omitted from `cfg` entirely if both are unset (line 684: `if ts_selection is not + None`), letting the eDisGo preset's own default apply. Granularity is truly per + individual MV grid (`mv_grid_id`, looped in `run_all`, `edisgo_integration.py:597-626`). + - **Decision:** `spatial_reduction` replicates this exactly — global `spatial_reduction` + default + `spatial_reduction_per_grid` dict keyed by `str(mv_grid_id)`, whole-block + replacement, omitted if unset (same as `timeseries_selection`, not the simpler + `overlying_grid` hardcoded-fallback pattern). +- ~~Reactive power~~ — **RESOLVED (2026-07-14).** Investigated existing convention: + `pm_optimize`'s results-writer (`edisgo/io/powermodels_io.py::from_powermodels`, + lines 283-352) writes ONLY active power for flex components (heat pumps, CPs, DSM, + storage) into `_generators_active_power`/`_loads_active_power`/ + `_storage_units_active_power`; reactive power is untouched there. Immediately after + (line 354-355), it calls the plain `edisgo_object.set_time_series_reactive_power_control()` + — same generic fixed-cosphi default (`network/timeseries.py::fixed_cosphi`, + `flex_opt/q_control.py`) used everywhere else in eDisGo, applied blanket over the whole + object, not scoped to flex components. Confirmed the existing `reactive_power` pipeline + task (`edisgo/run/tasks/timeseries.py:584-629`) is just a thin wrapper around the exact + same call — no special-casing for OPF-derived components anywhere in the codebase today. + - Considered alternative: split reactive power proportionally to each `old_name` + member's share of the representative's active power (mirroring the active-power + disaggregation rule) instead of recomputing. **Verified mathematically equivalent** + under fixed-cosphi: all `old_name` members of one representative share the same + `type` → same `power_factor`, so `Q = P · tan(φ)` per member gives an identical result + whether derived by proportional split or by recomputing from each member's + disaggregated P directly. + - **Decision:** `spatial_restore` writes active power for flexible components onto the + full grid, then calls `set_time_series_reactive_power_control()` itself — mirrors + `pm_optimize`'s own convention exactly (write P, then blanket-recompute Q). Reuses the + existing method with no new reactive-power math anywhere, and makes `spatial_restore` + correct standalone even in pipelines with no downstream `reactive_power` task. +- ~~Testing strategy~~ — **RESOLVED (2026-07-14).** Scope for this first pass: core + function only (`apply_reduced_results_to_full_grid`), NOT pipeline/task-level + integration tests (deferred). Cover both aggregation modes: + - `aggregation_mode=False`: by-name write-back correctness. + - `aggregation_mode=True`: disaggregation math — per-step envelope-weighted split, + exact-sum-back-to-representative check, and the equal-split zero-envelope fallback. + - Stub/fake OPF results as fixtures; no real `pm_optimize` call, no real pipeline run + through `ctx`/validator. +- ~~uc5 preset~~ — **RESOLVED (2026-07-14).** New standalone preset + `edisgo/run/presets/uc5_spatial_reduction.yaml` (full copy of + `uc5_select_timesteps.yaml` + the spatial bracket, NOT an `extends` overlay — tasks + don't exist in code yet so this is documentation/example, and a standalone file matches + `uc5_select_timesteps.yaml`'s own self-contained style). Disable switch: explicit + `spatial_reduction.enabled` flag (mirrors `overlying_grid.enabled`, NOT + `timeseries_selection`'s absent-block-is-the-toggle style) — lets params stay in the + YAML while toggling on/off with one flag. Bracket placement: `spatial_reduce` right + before `optimize`, `spatial_restore` right after; `reactive_power` stays where it already + is (pre-OPF full-series cosphi on the reduced index), unaffected by the spatial bracket. + Final order: `select_timesteps(post_grid) → reactive_power → spatial_reduce → optimize → + spatial_restore → reinforce`. + +--- + +## Implementation (2026-07-14) + +Implemented and tested end-to-end (real venv, python3.10, `pip install -e ".[dev]"`, +real ding0 test grid `tests/data/ding0_test_network_1`): + +- `apply_reduced_results_to_full_grid` + + `EDisGo.map_reduced_results_to_full_grid` — + `edisgo/tools/spatial_complexity_reduction.py`, `edisgo/edisgo.py`. +- `spatial_reduce` / `spatial_restore` tasks — new file `edisgo/run/tasks/spatial.py`, + registered in `edisgo/run/tasks/__init__.py`. +- `RunContext.full_grid_stash` — new field, `edisgo/run/context.py`. +- `task_optimize` writes `flexible_cps`/`flexible_hps`/`flexible_loads`/ + `flexible_storage_units` to `ctx.flags`; `@register_task("optimize", ...)` gained + `provides={"optimized_dispatch"}` — `edisgo/run/tasks/analysis.py`. +- New preset `edisgo/run/presets/uc5_spatial_reduction.yaml` (already covered above). +- New tests `tests/tools/test_spatial_complexity_reduction.py::TestApplyReducedResultsToFullGrid` + (6 tests: by-name write-back, multi-member disaggregation sum check, singleton-rename + regression, time-index-mismatch error, storage-unit by-name path, reactive-power + recompute). Full existing suite (`tests/tools/`, `tests/run/`, `tests/opf/`) reverified + green: 99 passed, 1 unrelated skip. + +**Two real bugs found by a dedicated code-review agent (dispatched because no import- +capable env existed initially) and fixed before landing:** + +1. **Singleton-rename data corruption (serious).** Original code took a `_write_by_name` + fast path whenever a flexible-component set had NO multi-member merged group, + assuming an unmerged representative's name always equals its member's name. FALSE + under `aggregation_mode=True`: `spatial_complexity_reduction` renames **every** + group's representative, including singleton groups (a bus with exactly one flexible + load of a type/sector) — confirmed empirically on the real test grid (11 such + singletons exist in `ding0_test_network_1` alone). `_write_by_name` using the + representative's (renamed) name against `full_grid` (which only has the original, + un-renamed name) silently created a phantom column via pandas' `.loc[]` auto-vivify + behavior, leaving the real target column stale — a silent, hard-to-detect data + corruption, not a crash. **Fix:** removed the `_write_by_name` fast path for CPs/HPs/ + DSM loads entirely; always route through `_disaggregate`, which was proven correct for + every case (matching name, mismatched singleton, multi-member) by direct test. + `_write_by_name` now only serves storage units, which are genuinely never renamed. +2. **`flexible_* or []` crashes on numpy-array input (real, hit on first live + end-to-end run).** `task_optimize` derives `flexible_loads` as + `edisgo.dsm.p_min.columns.values` (`analysis.py:357`) — a numpy array — unlike the + other three `flexible_*` lists, which use `.tolist()`. `array or []` raises + `ValueError: The truth value of an array with more than one element is ambiguous...` + for any such array with 2+ elements. This surfaced immediately on the very first real + pipeline run (`run_example_06.py`, `uc6_spatial_reduction.yaml`, `aggregation_mode: + false`) — the OPF/Gurobi solve completed successfully, `spatial_restore` crashed on + the very next line. **Fix:** replaced `flexible_x = flexible_x or []` with + `flexible_x = list(flexible_x) if flexible_x is not None else []` for all four + parameters in `apply_reduced_results_to_full_grid` — `is None` is the correct + emptiness check for an optional list-like argument that may be a list, tuple, or numpy + array (the codebase's own docstrings elsewhere already document these params as + accepting `numpy.ndarray or None`). Added regression test + `test_accepts_numpy_array_flexible_component_lists`. +3. **Unguarded `KeyError` on time-index mismatch (robustness).** If a flexibility-band/ + DSM/heat-pump attribute on `full_grid` doesn't cover `full_grid.timeseries.timeindex` + (e.g. the full-grid stash was taken before time-index selection ran — a real risk for + any pipeline not following the `select_timesteps` → `spatial_reduce` convention, since + nothing in the registry enforces that order), the disaggregation envelope lookup would + raise a bare `KeyError` deep inside a `.loc` call. **Fix:** added + `_require_full_timeindex`, a defensive check before each envelope lookup that raises a + clear `ValueError` naming the mismatch and the likely cause. Decided against also + adding a validator `requires={"timeseries"}` declaration — the pipeline is already + constructed so a time index is guaranteed set before `spatial_reduce` runs (decision 7, + above), so the defensive check alone is sufficient without touching registry metadata. + +## ADR candidates (offer at end of session) + +1. "Spatial reduction as two bracketing pipeline tasks, restore optional" — hard to + reverse, surprising (breaks the pm_optimize-owns-its-logic principle), real trade-off + (consistency vs stop-early). → write when design settles. +2. Possibly: "Disaggregation by pre-OPF flexibility envelope per time step" — a modeling + choice with alternatives (static scalar, proportional-to-original-series). Borderline; + decide at end. From 4fae4a03901fa3f3f0b08cec299ecba645de6f01 Mon Sep 17 00:00:00 2001 From: "Moritz.Schloesser" Date: Wed, 15 Jul 2026 15:03:53 +0000 Subject: [PATCH 49/66] Change overlying_grid path --- edisgo/run/presets/uc6_spatial_reduction.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/edisgo/run/presets/uc6_spatial_reduction.yaml b/edisgo/run/presets/uc6_spatial_reduction.yaml index 514a84589..66c116221 100644 --- a/edisgo/run/presets/uc6_spatial_reduction.yaml +++ b/edisgo/run/presets/uc6_spatial_reduction.yaml @@ -75,10 +75,10 @@ database: overlying_grid: enabled: true # set true to activate import_overlying_grid_data source: csv # "csv" (load from path) or "etrago" (kwarg) - path: "/storage/JoDa/edisgo_playground/overlying_grid_data" + path: "/home/gurobi/.edisgo_input/overlying_grid" results: - directory: results/uc5_spatial_reduction + directory: results/uc6_spatial_reduction # Top-level block read by the select_timesteps task via ctx.raw_config. # eGo can inject this block the same way it injects overlying_grid. @@ -115,7 +115,7 @@ spatial_reduction: cluster_area: feeder reduction_factor: 0.3 reduction_factor_not_focused: False - aggregation_mode: flase # start with false; true enables load/generator merging + aggregation_mode: false # start with false; true enables load/generator merging pipeline: - setup_grid From a28731d7dd0e257c94fbe1f369115ae7369363f7 Mon Sep 17 00:00:00 2001 From: "Moritz.Schloesser" Date: Wed, 15 Jul 2026 15:18:57 +0000 Subject: [PATCH 50/66] Fix stale UC5 naming in uc6_spatial_reduction.yaml, add issue writeup The preset's header comment and self-contained-run example still said "UC5"/uc5_spatial_reduction from when the file was copied from uc5_select_timesteps.yaml. Also adds a full issue description covering what's done and what's still open (aggregation_mode=True + charging points) for the spatial-reduction pipeline integration. --- ...mplexity_reduction_pipeline_integration.md | 140 ++++++++++++++++++ edisgo/run/presets/uc6_spatial_reduction.yaml | 4 +- 2 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md diff --git a/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md b/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md new file mode 100644 index 000000000..be650dadd --- /dev/null +++ b/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md @@ -0,0 +1,140 @@ +# Integrate spatial complexity reduction into the run pipeline + +## Goal + +Add **spatial complexity reduction** to the eDisGo run pipeline as a +counterpart to the existing temporal complexity reduction (timestep +selection, `select_timesteps`). Spatial reduction merges nearby buses into +representative buses to shrink the grid so the **optimization (OPF)** runs +faster. **Reinforcement must run on the FULL grid** (full topology) — but on +the *reduced* time index (temporal reduction still applies). + +## What's done + +- **Two bracketing pipeline tasks**, `spatial_reduce` (before `optimize`) + and `spatial_restore` (after, optional): `spatial_reduce` deepcopies and + stashes the full grid on the run context, then spatially reduces the + working object so `optimize` runs on a smaller grid; `spatial_restore` + writes the optimized flexible-component dispatch back onto the stashed + full grid and makes it active again for `reinforce`. Both are no-ops when + `spatial_reduction.enabled` is false, so a pipeline carrying the bracket + behaves identically to one without it when disabled. +- **New core function** `apply_reduced_results_to_full_grid(full_grid, + reduced_grid, *, flexible_cps, flexible_hps, flexible_loads, + flexible_storage_units)` in `edisgo/tools/spatial_complexity_reduction.py`, + plus a thin `EDisGo.map_reduced_results_to_full_grid` wrapper — mirroring + how `spatial_complexity_reduction`/`EDisGo.spatial_complexity_reduction` + already pair up for the reduction half. +- **Map-back only touches components the OPF rewrites**: flexible charging + points, heat pumps, DSM loads, and storage units. Inflexible + loads/generators are skipped since the OPF never changes their series. + With `aggregation_mode=False` (see "What's still open" below), every + component keeps its own name, so restore is a plain by-name write-back. +- **Reactive power on restore**: `spatial_restore` writes active power only, + then calls `EDisGo.set_time_series_reactive_power_control()` itself — + mirroring exactly how the OPF's own results-writer + (`io/powermodels_io.py::from_powermodels`) handles it (write P, then + blanket-recompute Q). No new reactive-power logic anywhere. +- **Validator integration**: extended the existing `requires`/`provides` + capability system rather than adding new validator machinery. + `spatial_reduce` provides `reduced_grid`; `optimize` additionally provides + `optimized_dispatch`; `spatial_restore` requires both — so a misordered + pipeline (e.g. `spatial_restore` before `optimize`) fails static + validation instead of crashing mid-run. +- **Config surface**: top-level `spatial_reduction:` YAML block (`enabled`, + `mode`, `cluster_area`, `reduction_factor`, `reduction_factor_not_focused`, + `aggregation_mode`), mirroring `timeseries_selection:` exactly, read via + `ctx.raw_config.get("spatial_reduction", {})`. +- **New preset** `edisgo/run/presets/uc6_spatial_reduction.yaml` wiring the + bracket into a real pipeline (`select_timesteps → reactive_power → + spatial_reduce → optimize → spatial_restore → reinforce → save`), disabled + by default via `spatial_reduction.enabled: false`. +- **eGo integration**: `EDisGoNetworks._build_run_edisgo_config` injects + `spatial_reduction`/`spatial_reduction_per_grid` exactly like + `timeseries_selection`/`timeseries_selection_per_grid` (global default + + per-grid override keyed by MV grid id, whole-block replacement). New + `scenario_setting_uc6_example.json`, plus unit tests for the injection + logic. +- **Tests**: `tests/tools/test_spatial_complexity_reduction.py` + (`TestApplyReducedResultsToFullGrid`) covers by-name write-back, the + numpy-array-input regression, the time-index-mismatch guard, and the + reactive-power recompute, with stub OPF results (no real Julia/Gurobi + dependency in CI). +- **Verified end-to-end** against a real grid (32377) through both eDisGo + directly and through eGo: `spatial_reduce → optimize (Gurobi) → + spatial_restore → reinforce → save` all completed successfully, and a + dedicated notebook (`analyse_spatial_reduction.ipynb`) confirms the + reduced grid's OPF output exactly matches the full grid's post-restore + value for every flexible component (0 mismatches across 388 components on + the test run). +- **Two bugs found and fixed during implementation** (unrelated to the + design, surfaced by actually running the code): a `flexible_* or []` + crash on numpy-array input (`task_optimize` derives `flexible_loads` as an + array, not a list); and `electromobility.flexibility_bands` not being + trimmed to the active time index after `build_flexibility_bands`, causing + a `KeyError` deep inside the OPF's charging-point constraint builder. + +## What's still open + +### `aggregation_mode=True` is not usable with charging points present + +`spatial_complexity_reduction` (`aggregation_mode=True`) merges loads at the +same bus and correctly aggregates their **time series** +(`loads_active_power`/`loads_reactive_power`). This works for DSM loads and +heat pumps, whose OPF constraints are read directly from +`loads_df`/`dsm.p_max`/`heat_pump.heat_demand_df`. + +Charging points are different: the OPF's constraint builder +(`_build_electromobility` in `edisgo/io/powermodels_io.py`) reads its +upper-power bound from `electromobility.flexibility_bands["upper_power"]`, a +separate DataFrame keyed by charging-point name — and +`spatial_complexity_reduction` never aggregates `flexibility_bands` during +load merging. So a merged representative has a correctly-summed +`loads_active_power` entry but **no** entry in `flexibility_bands`, and +`optimize` crashes with a `KeyError` for the representative's name before +`spatial_restore` ever runs. + +This is a gap in the reduction side (`spatial_complexity_reduction`/ +`apply_busmap`), not in `spatial_restore`/`apply_reduced_results_to_full_grid` +— reachable via `EDisGo.spatial_complexity_reduction` + `EDisGo.pm_optimize` +directly, no pipeline involved. It only manifests under +`aggregation_mode=True` with charging points present at a bus with 2+ of +them; `aggregation_mode=False` is unaffected. Full writeup with the exact +traceback and a suggested fix (aggregate `flexibility_bands` in +`apply_busmap`'s load-merging step, the same way `loads_active_power` +already is) is in +`docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md`. + +**Until this is fixed, `aggregation_mode` should stay `False`** — that is +the default in the new preset and the only mode covered by the disaggregation +tests and the end-to-end verification above. + +### Related, deferred design gap (not blocking, tracked separately) + +`build_flexibility_bands` builds bands over whatever date range the +underlying SimBEV data spans and only resamples to the target *frequency*, +not the target *date range* — the fix applied here trims them after the +fact (`reduce_timeseries_data_to_given_timeindex`). Building them +pre-scoped to the selected window in the first place would be cleaner and +more efficient; see +`docs_notes/issue_temporal_reduction_flexibility_bands.md`. + +### Not yet done + +- Reactive-power write-back has no dedicated integration test against a + real OPF run (covered by unit test with stub dispatch only). +- No pipeline/task-level integration test for `spatial_reduce`/ + `spatial_restore` (current test scope is core-function-only, per an + explicit scoping decision — see `docs_notes/spatial_reduction_grilling_session.md`). +- Two ADR candidates flagged during design, not yet written: "spatial + reduction as two bracketing pipeline tasks, restore optional" (breaks the + established "logic lives inside `pm_optimize`" convention, trading + consistency for the ability to stop early after the reduced-grid OPF), and + "disaggregation by pre-OPF flexibility envelope" (a modeling choice with + real alternatives). + +## References + +- Full design session record: `docs_notes/spatial_reduction_grilling_session.md` +- `aggregation_mode=True` + charging points gap: `docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md` +- Flexibility-bands time-index gap (fixed): `docs_notes/issue_temporal_reduction_flexibility_bands.md` diff --git a/edisgo/run/presets/uc6_spatial_reduction.yaml b/edisgo/run/presets/uc6_spatial_reduction.yaml index 66c116221..767327552 100644 --- a/edisgo/run/presets/uc6_spatial_reduction.yaml +++ b/edisgo/run/presets/uc6_spatial_reduction.yaml @@ -1,5 +1,5 @@ _comment: | - UC5 — OPF with configurable timestep selection (manual OR auto), PLUS spatial + UC6 — OPF with configurable timestep selection (manual OR auto), PLUS spatial complexity reduction bracketing the OPF step. Standalone copy of uc5_select_timesteps.yaml (not an `extends` overlay) with the spatial_reduce / spatial_restore bracket added around `optimize`. @@ -56,7 +56,7 @@ _workflow: # Self-contained (no `extends`): everything uc4_example provided is inlined # below, so this preset can be run on its own via -# run_edisgo({"extends": "uc5_spatial_reduction", "grid": {"ding0_path": ...}}) +# run_edisgo({"extends": "uc6_spatial_reduction", "grid": {"ding0_path": ...}}) scenario: eGon2035 grid: From af2c7a024b61538da222763a3abd06113ee96b5f Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Wed, 15 Jul 2026 17:55:16 +0200 Subject: [PATCH 51/66] Spatial complexity reduction for the run pipeline (partial) (#706) * Change database from OEP to local * feat: spatial complexity reduction for the run pipeline Adds spatial_reduce/spatial_restore tasks bracketing optimize, so the OPF can run on a spatially-reduced grid while reinforcement still runs on the full topology. New core function apply_reduced_results_to_full_grid maps optimized flexible-component dispatch (charging points, heat pumps, DSM loads, storage) back onto the full grid, by name or disaggregated onto old_name members per time step (weighted by each member's pre-OPF flexibility envelope) when aggregation_mode=True. Reactive power is recomputed on restore, mirroring how pm_optimize itself handles it. optimize now also records its flexible-component name lists on ctx.flags and declares provides={"optimized_dispatch"}, so the validator's existing requires/provides system enforces spatial_restore running after both spatial_reduce and optimize. * fix: trim electromobility flexibility bands to the active time index get_flexibility_bands only resamples bands to the active time-series FREQUENCY, not the active date range - so after a manual/short select_timesteps window, flexibility_bands kept spanning the full underlying SimBEV data range with no later step ever trimming it down. Any consumer indexing flexibility_bands by edisgo.timeseries.timeindex (e.g. the OPF's charging-point constraint builder) could then hit missing time steps. build_flexibility_bands now trims electromobility data to the active time index right after building the bands. * Add uc6_spatial_reduction preset and example runner Standalone preset wiring spatial_reduce/spatial_restore around optimize, disabled by default via spatial_reduction.enabled. run_example_06.py runs it against a real ding0 grid. * docs: spatial reduction glossary and grilling-session notes CONTEXT.md gains a glossary section for spatial reduction terms (busmap, disaggregation rule, config surface, ordering). docs_notes/ carries the full design-session record plus two follow-up issue write-ups: the flexibility-bands time-index gap (fixed) and the aggregation_mode=True + charging-points gap (not yet fixed, tracked for later). * Change overlying_grid path * Fix stale UC5 naming in uc6_spatial_reduction.yaml, add issue writeup The preset's header comment and self-contained-run example still said "UC5"/uc5_spatial_reduction from when the file was copied from uc5_select_timesteps.yaml. Also adds a full issue description covering what's done and what's still open (aggregation_mode=True + charging points) for the spatial-reduction pipeline integration. * style: apply pre-commit hooks to the spatial-reduction commits Hooks weren't run before the original commits landed. Adds the missing license header to edisgo/run/context.py, applies ruff-format's line collapsing across a few files, and manually wraps two docstring :func: cross-references that were still over the line-length limit after formatting. --------- --- CONTEXT.md | 92 +++++ ...n_mode_flexibility_bands_not_aggregated.md | 83 +++++ ...mplexity_reduction_pipeline_integration.md | 140 ++++++++ ...ue_temporal_reduction_flexibility_bands.md | 81 +++++ .../spatial_reduction_grilling_session.md | 332 ++++++++++++++++++ edisgo/edisgo.py | 68 +++- edisgo/run/context.py | 17 + edisgo/run/presets/uc5_select_timesteps.yaml | 4 +- edisgo/run/presets/uc6_spatial_reduction.yaml | 149 ++++++++ edisgo/run/tasks/__init__.py | 10 +- edisgo/run/tasks/analysis.py | 15 +- edisgo/run/tasks/flex.py | 19 + edisgo/run/tasks/spatial.py | 148 ++++++++ edisgo/tools/spatial_complexity_reduction.py | 231 ++++++++++++ run_example_06.py | 37 ++ .../test_spatial_complexity_reduction.py | 252 +++++++++++++ 16 files changed, 1669 insertions(+), 9 deletions(-) create mode 100644 CONTEXT.md create mode 100644 docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md create mode 100644 docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md create mode 100644 docs_notes/issue_temporal_reduction_flexibility_bands.md create mode 100644 docs_notes/spatial_reduction_grilling_session.md create mode 100644 edisgo/run/presets/uc6_spatial_reduction.yaml create mode 100644 edisgo/run/tasks/spatial.py create mode 100644 run_example_06.py diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 000000000..7e923e900 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,92 @@ +# Context / Glossary + +Domain vocabulary for the eDisGo run pipeline. Glossary only — no implementation +details, no decisions (those live in `docs/adr/`). + +## Complexity reduction + +- **Spatial complexity reduction** — merging nearby buses into a smaller set of + representative buses (clustering *along the grid*, keeping it radial) to shrink the + grid for faster power flow / optimization. Implemented by + `EDisGo.spatial_complexity_reduction`, which builds a *busmap* + *linemap* and mutates + the Topology. Used only to accelerate the optimization; reinforcement runs on the + **full grid**. +- **Temporal complexity reduction** — keeping only grid-critical time steps/intervals + instead of the full year. See the `select_timesteps` task. +- **Busmap** — DataFrame mapping each original bus to its clustered *new_bus* (with new + coordinates). Index = original bus names. +- **Linemap** — DataFrame mapping original line names to *new_line_name* after lines are + recalculated/merged. +- **Reduced grid** — the spatially-reduced EDisGo object the OPF runs on. +- **Full grid** — the original, unreduced EDisGo object; reinforcement always runs here. +- **Map-back / restore** — writing the OPF flexibility dispatch from the reduced grid + onto the full grid. Only the components the OPF *rewrites* are mapped back — flexible + charging points, heat pumps, DSM loads, and storage. Inflexible loads/generators are + **skipped** (the OPF does not change their series; the full grid already holds them + correctly). Implemented by core function + `tools/spatial_complexity_reduction.py::apply_reduced_results_to_full_grid(full_grid, + reduced_grid, *, flexible_cps=None, flexible_hps=None, flexible_loads=None, + flexible_storage_units=None)` + thin wrapper `EDisGo.map_reduced_results_to_full_grid`. + Provenance for disaggregation comes solely from `old_name` on the reduced grid's + `loads_df`/`generators_df` — no busmap/linemap stash needed on `ctx`. + - `aggregation_mode=False`: components keep their names → write back **by component + name**. + - `aggregation_mode=True`: loads/generators may be merged into a representative + (originals recorded in `old_name`; storage is never aggregated). The representative's + optimized series is **disaggregated** onto its `old_name` members. +- **Disaggregation rule** — split a merged representative's optimized series onto its + original members **per time step**, weighted by each member's own *pre-OPF flexibility + envelope* (a known input, never the optimized result): `upper_power(t)` band for + charging points, heat-demand/thermal envelope for heat pumps, `p_max(t)` band for DSM. + Per-step weighting means a charging point only receives power at steps where it has a + connected vehicle (`upper_power(t) > 0`). Sums back to the representative exactly at + every step; equal split as the zero-envelope fallback. +- **Reactive power on restore** — `spatial_restore` writes active power only, then calls + `EDisGo.set_time_series_reactive_power_control()` itself (plain fixed-cosphi default), + mirroring exactly how `pm_optimize`'s own results-writer + (`io/powermodels_io.py::from_powermodels`) handles it: write P, then blanket-recompute Q. + No bespoke reactive-power logic — proportional-split and recompute are mathematically + identical under fixed-cosphi since all `old_name` members of one representative share + the same `power_factor`. + +## Pipeline tasks (spatial reduction) + +- **spatial_reduce** — task run *before* `optimize`: deepcopy the full grid, stash it, + and spatially reduce the working object so `optimize` runs on the reduced grid. +- **spatial_restore** — task run *after* `optimize`: write the optimized dispatch time + series back onto the stashed full grid and make it active again. **Optional** — a run + may legitimately stop after the reduced-grid optimization when only the derived time + series matter. +- **Full-grid stash** — the deepcopied full grid kept in memory on `ctx` between + `spatial_reduce` and `spatial_restore`. Held in-memory (matching legacy eGo, which kept + both full and reduced grids resident during optimize). Persisting it to a disk artifact + to lower peak memory is a possible later optimization, not the initial design. + +## Config surface (spatial reduction) + +- Top-level YAML block `spatial_reduction:` (mirrors `timeseries_selection:`), holding + `mode`, `cluster_area`, `reduction_factor`, `reduction_factor_not_focused`, + `aggregation_mode`, and aggregation sub-modes. Read inside the `spatial_reduce` task via + `ctx.raw_config.get("spatial_reduction", {})`. +- eGo injects it exactly like `timeseries_selection`: a global `spatial_reduction` default + plus a `spatial_reduction_per_grid` dict keyed by `str(mv_grid_id)`, whole-block + replacement (not a field-level merge). If neither is set the key is omitted entirely and + the eDisGo preset's own default applies. + +## Ordering (spatial reduction) + +Pipeline order: `select_timesteps` → `spatial_reduce` → `optimize` → `spatial_restore` +→ `reinforce`. + +- Spatial reduction touches only **topology**; temporal reduction touches only the + **time index** — orthogonal operations that commute as *mechanisms*. Spatial reduction + works on any time index, including full-year (clustering depends on coordinates/graph + distance, not the time series; the `reduction_factor_not_focused` worst-case power flow + runs on an internal deepcopy and does not disturb the working series). +- But the **pipeline** pins `select_timesteps` before `spatial_reduce`: the full-grid + stash inherits whatever time index exists at deepcopy time, and reinforce runs on that + stash. Reducing timesteps first means the stash (and therefore reinforce) carries the + reduced index — full **topology**, reduced **time index**. Reversing the two would + force separately re-reducing the stash's index. +- `spatial_restore` does **no time-index surgery** — it only writes flexible-component + dispatch back onto the stashed full grid. diff --git a/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md b/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md new file mode 100644 index 000000000..37be52abe --- /dev/null +++ b/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md @@ -0,0 +1,83 @@ +# `spatial_complexity_reduction(aggregation_mode=True)` doesn't aggregate `electromobility.flexibility_bands`, breaking `optimize` on merged charging points + +**Type:** bug +**Found:** 2026-07-15, running the real `uc6_spatial_reduction.yaml` pipeline +on grid 32377 with `spatial_reduction.aggregation_mode: true`. +**Affects:** `spatial_complexity_reduction`/`apply_busmap` +(`edisgo/tools/spatial_complexity_reduction.py`), specifically the load +aggregation step; consumed by `_build_electromobility` +(`edisgo/io/powermodels_io.py`). + +## Problem + +When `aggregation_mode=True`, `apply_busmap` merges loads at the same bus +(`aggregate_loads_df`, `spatial_complexity_reduction.py:1587-1599`) and +correctly aggregates their **time series** via `aggregate_timeseries` +(`spatial_complexity_reduction.py:1718`, called for `loads_active_power` / +`loads_reactive_power`). This works fine for DSM loads and heat pumps, whose +OPF constraints are read directly from `loads_df`/`dsm.p_max`/ +`heat_pump.heat_demand_df` keyed by whatever name is currently in `loads_df`. + +Charging points are different: the OPF's own constraint builder, +`_build_electromobility` (`edisgo/io/powermodels_io.py:1212`), does **not** +read its upper-power bound from `loads_df` — it reads +`electromobility.flexibility_bands["upper_power"][cp_name]`, a separate +DataFrame keyed by charging-point name. `spatial_complexity_reduction` never +touches `flexibility_bands` during aggregation, so it still only has entries +for the *original*, pre-merge charging-point names. + +When a bus has 2+ charging points and gets merged into one representative +load (e.g. `Load_Bus_mvgd_32377_F2_B2_charging_point_hpc`), that +representative name has: +- a correctly-aggregated `loads_active_power` entry (summed from the + originals), but +- **no** entry at all in `flexibility_bands["upper_power"]`. + +`task_optimize` derives `flexible_cps` from `loads_df` (filtering +`type == "charging_point"`), so it passes the representative's name into +`pm_optimize` → `to_powermodels` → `_build_electromobility`, which then does +`flex_bands_df["upper_power"][emob_df.index[cp_i]]` and raises `KeyError` +for the representative name. + +## Reproduction + +Run `uc6_spatial_reduction.yaml` (or any preset with the spatial bracket) on +a grid with 2+ charging points sharing a bus, with +`spatial_reduction: {enabled: true, aggregation_mode: true, +load_aggregation_mode: bus}` and `optimize: {flexible: [..., +charging_points, ...]}`. Crashes inside `optimize`, before `spatial_restore` +ever runs: + +``` +KeyError: 'Load_Bus_mvgd_32377_F2_B2_charging_point_hpc' + .../edisgo/io/powermodels_io.py:1212, in _build_electromobility + * flex_bands_df["upper_power"][emob_df.index[cp_i]].iloc[0] +``` + +## Scope note + +This is independent of the spatial-reduction pipeline-integration work +(`spatial_reduce`/`spatial_restore`/`apply_reduced_results_to_full_grid`) — +it's a gap in `spatial_complexity_reduction` itself (the reduction side), +already reachable via `EDisGo.spatial_complexity_reduction` + +`EDisGo.pm_optimize` directly, with no pipeline involved. It only manifests +under `aggregation_mode=True` with charging points present; `aggregation_ +mode=False` is unaffected since every component keeps its original name +there, and `flexibility_bands` stays valid. + +Also worth checking as part of the same fix: whether heat pumps have an +analogous issue if the OPF ever reads a heat-pump-specific band keyed +differently from `loads_df` (current understanding, from the spatial- +reduction design session, is that heat pumps use `heat_demand_df`/`cop_df`/ +`loads_df.p_set` directly, which DO get aggregated correctly — but worth +double-checking with a merged multi-heat-pump bus once this issue is picked +up, in case some other heat-pump-specific structure has the same gap). + +## Suggested fix + +Extend `apply_busmap`'s load-aggregation step to also aggregate +`electromobility.flexibility_bands` (`upper_power`, `lower_energy`, +`upper_energy`) onto the same representative name, using the same +`old_name`-based grouping/summing already used for `aggregate_timeseries` +— summing `upper_power` per merged group is the natural analogue of summing +`p_set`/active power for the representative. diff --git a/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md b/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md new file mode 100644 index 000000000..be650dadd --- /dev/null +++ b/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md @@ -0,0 +1,140 @@ +# Integrate spatial complexity reduction into the run pipeline + +## Goal + +Add **spatial complexity reduction** to the eDisGo run pipeline as a +counterpart to the existing temporal complexity reduction (timestep +selection, `select_timesteps`). Spatial reduction merges nearby buses into +representative buses to shrink the grid so the **optimization (OPF)** runs +faster. **Reinforcement must run on the FULL grid** (full topology) — but on +the *reduced* time index (temporal reduction still applies). + +## What's done + +- **Two bracketing pipeline tasks**, `spatial_reduce` (before `optimize`) + and `spatial_restore` (after, optional): `spatial_reduce` deepcopies and + stashes the full grid on the run context, then spatially reduces the + working object so `optimize` runs on a smaller grid; `spatial_restore` + writes the optimized flexible-component dispatch back onto the stashed + full grid and makes it active again for `reinforce`. Both are no-ops when + `spatial_reduction.enabled` is false, so a pipeline carrying the bracket + behaves identically to one without it when disabled. +- **New core function** `apply_reduced_results_to_full_grid(full_grid, + reduced_grid, *, flexible_cps, flexible_hps, flexible_loads, + flexible_storage_units)` in `edisgo/tools/spatial_complexity_reduction.py`, + plus a thin `EDisGo.map_reduced_results_to_full_grid` wrapper — mirroring + how `spatial_complexity_reduction`/`EDisGo.spatial_complexity_reduction` + already pair up for the reduction half. +- **Map-back only touches components the OPF rewrites**: flexible charging + points, heat pumps, DSM loads, and storage units. Inflexible + loads/generators are skipped since the OPF never changes their series. + With `aggregation_mode=False` (see "What's still open" below), every + component keeps its own name, so restore is a plain by-name write-back. +- **Reactive power on restore**: `spatial_restore` writes active power only, + then calls `EDisGo.set_time_series_reactive_power_control()` itself — + mirroring exactly how the OPF's own results-writer + (`io/powermodels_io.py::from_powermodels`) handles it (write P, then + blanket-recompute Q). No new reactive-power logic anywhere. +- **Validator integration**: extended the existing `requires`/`provides` + capability system rather than adding new validator machinery. + `spatial_reduce` provides `reduced_grid`; `optimize` additionally provides + `optimized_dispatch`; `spatial_restore` requires both — so a misordered + pipeline (e.g. `spatial_restore` before `optimize`) fails static + validation instead of crashing mid-run. +- **Config surface**: top-level `spatial_reduction:` YAML block (`enabled`, + `mode`, `cluster_area`, `reduction_factor`, `reduction_factor_not_focused`, + `aggregation_mode`), mirroring `timeseries_selection:` exactly, read via + `ctx.raw_config.get("spatial_reduction", {})`. +- **New preset** `edisgo/run/presets/uc6_spatial_reduction.yaml` wiring the + bracket into a real pipeline (`select_timesteps → reactive_power → + spatial_reduce → optimize → spatial_restore → reinforce → save`), disabled + by default via `spatial_reduction.enabled: false`. +- **eGo integration**: `EDisGoNetworks._build_run_edisgo_config` injects + `spatial_reduction`/`spatial_reduction_per_grid` exactly like + `timeseries_selection`/`timeseries_selection_per_grid` (global default + + per-grid override keyed by MV grid id, whole-block replacement). New + `scenario_setting_uc6_example.json`, plus unit tests for the injection + logic. +- **Tests**: `tests/tools/test_spatial_complexity_reduction.py` + (`TestApplyReducedResultsToFullGrid`) covers by-name write-back, the + numpy-array-input regression, the time-index-mismatch guard, and the + reactive-power recompute, with stub OPF results (no real Julia/Gurobi + dependency in CI). +- **Verified end-to-end** against a real grid (32377) through both eDisGo + directly and through eGo: `spatial_reduce → optimize (Gurobi) → + spatial_restore → reinforce → save` all completed successfully, and a + dedicated notebook (`analyse_spatial_reduction.ipynb`) confirms the + reduced grid's OPF output exactly matches the full grid's post-restore + value for every flexible component (0 mismatches across 388 components on + the test run). +- **Two bugs found and fixed during implementation** (unrelated to the + design, surfaced by actually running the code): a `flexible_* or []` + crash on numpy-array input (`task_optimize` derives `flexible_loads` as an + array, not a list); and `electromobility.flexibility_bands` not being + trimmed to the active time index after `build_flexibility_bands`, causing + a `KeyError` deep inside the OPF's charging-point constraint builder. + +## What's still open + +### `aggregation_mode=True` is not usable with charging points present + +`spatial_complexity_reduction` (`aggregation_mode=True`) merges loads at the +same bus and correctly aggregates their **time series** +(`loads_active_power`/`loads_reactive_power`). This works for DSM loads and +heat pumps, whose OPF constraints are read directly from +`loads_df`/`dsm.p_max`/`heat_pump.heat_demand_df`. + +Charging points are different: the OPF's constraint builder +(`_build_electromobility` in `edisgo/io/powermodels_io.py`) reads its +upper-power bound from `electromobility.flexibility_bands["upper_power"]`, a +separate DataFrame keyed by charging-point name — and +`spatial_complexity_reduction` never aggregates `flexibility_bands` during +load merging. So a merged representative has a correctly-summed +`loads_active_power` entry but **no** entry in `flexibility_bands`, and +`optimize` crashes with a `KeyError` for the representative's name before +`spatial_restore` ever runs. + +This is a gap in the reduction side (`spatial_complexity_reduction`/ +`apply_busmap`), not in `spatial_restore`/`apply_reduced_results_to_full_grid` +— reachable via `EDisGo.spatial_complexity_reduction` + `EDisGo.pm_optimize` +directly, no pipeline involved. It only manifests under +`aggregation_mode=True` with charging points present at a bus with 2+ of +them; `aggregation_mode=False` is unaffected. Full writeup with the exact +traceback and a suggested fix (aggregate `flexibility_bands` in +`apply_busmap`'s load-merging step, the same way `loads_active_power` +already is) is in +`docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md`. + +**Until this is fixed, `aggregation_mode` should stay `False`** — that is +the default in the new preset and the only mode covered by the disaggregation +tests and the end-to-end verification above. + +### Related, deferred design gap (not blocking, tracked separately) + +`build_flexibility_bands` builds bands over whatever date range the +underlying SimBEV data spans and only resamples to the target *frequency*, +not the target *date range* — the fix applied here trims them after the +fact (`reduce_timeseries_data_to_given_timeindex`). Building them +pre-scoped to the selected window in the first place would be cleaner and +more efficient; see +`docs_notes/issue_temporal_reduction_flexibility_bands.md`. + +### Not yet done + +- Reactive-power write-back has no dedicated integration test against a + real OPF run (covered by unit test with stub dispatch only). +- No pipeline/task-level integration test for `spatial_reduce`/ + `spatial_restore` (current test scope is core-function-only, per an + explicit scoping decision — see `docs_notes/spatial_reduction_grilling_session.md`). +- Two ADR candidates flagged during design, not yet written: "spatial + reduction as two bracketing pipeline tasks, restore optional" (breaks the + established "logic lives inside `pm_optimize`" convention, trading + consistency for the ability to stop early after the reduced-grid OPF), and + "disaggregation by pre-OPF flexibility envelope" (a modeling choice with + real alternatives). + +## References + +- Full design session record: `docs_notes/spatial_reduction_grilling_session.md` +- `aggregation_mode=True` + charging points gap: `docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md` +- Flexibility-bands time-index gap (fixed): `docs_notes/issue_temporal_reduction_flexibility_bands.md` diff --git a/docs_notes/issue_temporal_reduction_flexibility_bands.md b/docs_notes/issue_temporal_reduction_flexibility_bands.md new file mode 100644 index 000000000..5aac8caca --- /dev/null +++ b/docs_notes/issue_temporal_reduction_flexibility_bands.md @@ -0,0 +1,81 @@ +# Flexibility bands should be built scoped to the selected time index, not built-then-trimmed + +**Type:** design gap / bug +**Found:** 2026-07-15, while testing spatial complexity reduction against a +manual-mode `select_timesteps` run (`uc6_spatial_reduction.yaml`). +**Affects:** `task_build_flexibility_bands` (`edisgo/run/tasks/flex.py`), +`Electromobility.get_flexibility_bands` (`edisgo/network/electromobility.py`), +and by extension any other per-component band/envelope built after +`select_timesteps` (heat pump `heat_demand_df`/`cop_df`, DSM `p_max`/`p_min`). + +## Problem + +`select_timesteps` (manual mode, `position: pre_import`) fixes +`edisgo.timeseries.timeindex` to an arbitrary, possibly short window (e.g. 24 +hours starting `2035-01-15`) early in the pipeline, before electromobility +data is even imported. + +`build_flexibility_bands` runs later and calls +`Electromobility.get_flexibility_bands(edisgo, use_case=...)`. Per that +method's own docstring, `resample=True` only "resamples the bands to the same +**frequency** as time series data in the `TimeSeries` object" — it does +*not* clip the bands to the same *date range*. The bands are built from the +raw SimBEV charging-process data and keep whatever date range that data +spans, matching the target row spacing (e.g. hourly) but not the target +window. + +Nothing downstream re-trims `electromobility.flexibility_bands` to the +selected 24-hour window afterward. The mismatch went unnoticed until a piece +of code tried to actually index `flexibility_bands` using +`edisgo.timeseries.timeindex` and hit a `KeyError`-shaped failure (missing +time steps) — in this case, the new +`apply_reduced_results_to_full_grid`/`spatial_restore` disaggregation logic, +which reads `flexibility_bands["upper_power"]` as a per-charging-point, +per-time-step weighting envelope. + +This is a **pre-existing gap**, not something introduced by spatial +reduction — it already exists in `uc5_select_timesteps.yaml` (the preset +`uc6_spatial_reduction.yaml`/`uc5_spatial_reduction.yaml` were both derived +from). It simply had no consumer that indexed `flexibility_bands` by the +active time index before now. + +## Immediate fix applied (unblocks spatial-reduction testing) + +`task_build_flexibility_bands` now calls +`reduce_timeseries_data_to_given_timeindex(edisgo, edisgo.timeseries.timeindex, +electromobility=True, timeseries=False, heat_pump=False, dsm=False, +overlying_grid=False)` right after `get_flexibility_bands`, trimming +`flexibility_bands` down to the active index. See +`edisgo/run/tasks/flex.py::task_build_flexibility_bands`. + +This is a workaround (build full, then trim), not the better design below. + +## Suggested proper fix (not yet implemented — this issue) + +Build flexibility bands (and, likely, heat-pump/DSM bands) scoped to the +already-selected time index from the start, rather than building over the +full/native data range and trimming afterward: + +- `Electromobility.get_flexibility_bands` (or its caller) should accept/use + the target time index *before* running the difference-array band + construction, so the SimBEV charging-process data outside that window is + never even considered. +- Audit whether `HeatPump`/`DSM` band construction (wherever their + time-varying bounds are first populated — likely in the `import_heat_pumps` + / `import_dsm` tasks or their underlying `edisgo/io/*` importers) has the + same built-on-full-range-then-maybe-trimmed pattern, since + `reduce_timeseries_data_to_given_timeindex` already has `heat_pump=True`/ + `dsm=True` flags suggesting this was anticipated but may not be + consistently invoked at the right point in every pipeline path. +- Consider whether this should be a single, explicit "finalize time index" + pipeline hook that every band-producing task can rely on having already + run, rather than each task needing to remember to trim itself. + +## Reproduction + +Run `uc6_spatial_reduction.yaml` (or `uc5_select_timesteps.yaml`) with +`timeseries_selection: {mode: manual, start: "2035-01-15 00:00", periods: +24, freq: h}` and `overlying_grid.enabled: true`, then inspect +`edisgo.electromobility.flexibility_bands["upper_power"].index` after +`build_flexibility_bands` runs — before the fix above, its date range does +not match `edisgo.timeseries.timeindex`. diff --git a/docs_notes/spatial_reduction_grilling_session.md b/docs_notes/spatial_reduction_grilling_session.md new file mode 100644 index 000000000..b90ddb190 --- /dev/null +++ b/docs_notes/spatial_reduction_grilling_session.md @@ -0,0 +1,332 @@ +# Spatial complexity reduction — pipeline integration design (grilling session) + +**Status:** DESIGN CLOSED. All questions resolved 2026-07-14. Next: implement, and decide +the two ADR candidates below. +**Date:** 2026-07-09 (started), closed 2026-07-14. +**Repo/branch:** `/storage/MS/ego/eDisGo`, branch `edisgo_run_edisgo`. +**Companion files:** `CONTEXT.md` (glossary, at repo root), this file. + +--- + +## Goal + +Integrate eDisGo's **spatial complexity reduction** into the run pipeline, as a +counterpart to the temporal complexity reduction (timestep selection) already added. + +Spatial reduction merges nearby buses into representative buses to shrink the grid so +the **optimization (OPF)** runs faster. **Reinforcement must run on the FULL grid** +(full topology) — but on the *reduced* time index (temporal reduction still applies). + +--- + +## How spatial reduction works (established from code) + +- `EDisGo.spatial_complexity_reduction()` (edisgo.py:3439) wraps + `tools/spatial_complexity_reduction.py::spatial_complexity_reduction()` (line 1830). +- It builds a **busmap** (original bus → clustered `new_bus`) + **linemap**, then mutates + the Topology in place (or on a copy if `copy_edisgo=True`). Returns `(edisgo, busmap_df, + linemap_df)`. +- Clustering uses coordinates / grid-graph distance — **independent of the time index**. + Works on a full-year grid too. `apply_pseudo_coordinates=True` (default) fills missing + coords. +- **Storage is never aggregated** (only bus-relabeled) — keeps its name/identity even + with `aggregation_mode=True` (spatial_complexity_reduction.py:1756–1760). +- **Loads/generators ARE merged** when `aggregation_mode=True` (lines 1699–1754), grouped + by bus(+type+sector). Originals recorded in an **`old_name`** column on the reduced + component rows; their series summed via `aggregate_timeseries`. +- `reduction_factor_not_focused` uses `find_buses_of_interest` which runs a worst-case + power flow — but on an **internal deepcopy** (line 93), so it does NOT disturb the + working object's time series. + +### Legacy eGo reference (the pattern we are re-implementing cleanly) +`eGo/ego/tools/edisgo_integration.py::_run_edisgo_task_optimisation` (~line 1632): +- `edisgo_copy = deepcopy(edisgo_grid)` (full grid stays in `edisgo_grid`) +- temporal-reduce copy → spatial-reduce copy → `pm_optimize` on copy +- write dispatch back onto the full grid **by component name** (lines 1763–1795): + loads/generators/storage active+reactive power, sliced by `time_steps` +- `edisgo_grid.timeseries.timeindex = timeindex` (union of optimized intervals) +- reinforce runs on the full grid. Both grids were resident in memory simultaneously. + +--- + +## Decisions made (confirmed with user) + +1. **Two bracketing tasks (Option A), NOT inside `pm_optimize`.** + - `spatial_reduce` (before `optimize`) and `spatial_restore` (after `optimize`). + - Rationale: reduce/restore are topology operations, not OPF concerns; must be + visible/toggleable in YAML; enables stopping after the reduced-grid OPF when only + the derived time series matter. + - This deliberately breaks the "all logic inside pm_optimize" principle we used for + the multi-interval split — justified by the stop-early capability. **ADR candidate.** + +2. **`spatial_restore` is OPTIONAL** — a run may stop after optimizing on the reduced + grid. (But when present it must follow optimize; see validator note.) + +3. **Full-grid stash kept IN MEMORY on `ctx`** (matches legacy, which held both grids + resident). Disk-artifact persistence to cut peak memory is a deferred optimization, + not v1. (Runner does support disk reload via `save` + `stage_artifacts` + `load_from`, + but we are not using it here.) + +4. **Aggregation support:** implement `aggregation_mode=False` FIRST, then design so + `aggregation_mode=True` follows. + +5. **Map-back only touches components the OPF rewrites:** flexible charging points, heat + pumps, DSM loads, and storage. **Inflexible loads/generators are SKIPPED** — the OPF + doesn't change them; the full grid already holds their correct series. + - `aggregation_mode=False`: write back **by component name**. + - `aggregation_mode=True`: disaggregate the representative's series onto its `old_name` + members. + +6. **Disaggregation rule (aggregation_mode=True):** split the representative's optimized + series onto original members **per time step**, weighted by each member's own **pre-OPF + flexibility envelope** (a known input, never the optimized result): + - charging points → `electromobility.flexibility_bands["upper_power"][cp_name]` (a CP + with no connected vehicle has `upper_power(t)=0`, so it receives no charge that step + — physically correct); + - heat pumps → `weight(t) = min(heat_demand_df[hp_name][t] / cop_df[hp_name][t], + loads_df.p_set[hp_name])`. **Refined during implementation (2026-07-14):** the + original "heat-demand/thermal envelope" phrasing was underspecified. Investigated + `edisgo/io/powermodels_io.py::_build_heatpump` (~lines 1259-1282): the OPF's actual + per-unit electrical cap is the CONSTANT rated power `loads_df.p_set`, not a + time-varying series — the genuinely time-varying pre-OPF quantity is + `heat_demand_df` (thermal, MW) divided by `cop_df` (electrical-equivalent demand). + Capping that ratio at each member's own `p_set` mirrors charging points exactly (a + CP's `upper_power(t)` is already a capped bound, not raw uncapped vehicle demand) and + ensures no member is ever assigned a share exceeding what it could physically draw; + - DSM → `dsm.p_max[load_name][t]` band (`edisgo/network/dsm.py:63`). + - All three sources share the same shape: rows = timestamps matching + `TimeSeries.timeindex`, columns = component names matching `Topology.loads_df.index`. + - Sums back to the representative exactly at each step; equal split as zero-envelope + fallback. (User explicitly preferred a time-series-based split over a static scalar, + because these envelopes are known pre-OPF and reflect actual flexibility-relevant + events, e.g. a connected vehicle or nonzero heat demand.) + +7. **Ordering:** `select_timesteps` → `spatial_reduce` → `optimize` → `spatial_restore` + → `reinforce`. + - The two reduction *mechanisms* commute (orthogonal: topology vs time index), BUT the + pipeline pins `select_timesteps` before `spatial_reduce` so the stashed full grid + (hence reinforce) inherits the **reduced** time index. Result: reinforce on full + **topology** × reduced **time index**. + - `spatial_restore` does **no time-index surgery** — only writes flexible dispatch back. + +8. **Tasks stay thin; computation lives in eDisGo core** (same principle as the + timestep-selection refactor). See open question for how this applies to restore. + +--- + +## Where we paused — OPEN QUESTION (resume here) + +Applying "tasks are thin wrappers" to the restore half. Established: +- **Reduction half is already correct:** `spatial_complexity_reduction()` is already a + self-contained core function + EDisGo method. `spatial_reduce` task just needs to + deepcopy+stash the full grid and call it. Nothing to extract. +- **Restore half is the gap:** there is **NO** existing core function that maps reduced + OPF results back onto a full grid (the legacy logic lived inline in eGo's private + method; `_restore_pristine_inputs` in powermodels_opf.py:225 is unrelated — it's the + multi-interval snapshot/restore). + +**Proposal put to the user (awaiting confirmation):** +- (a) Create a NEW core function, e.g. + `tools/spatial_complexity_reduction.py::apply_reduced_results_to_full_grid(full_grid, + reduced_grid, *, flexible_cps, flexible_hps, flexible_loads, flexible_storage_units)` + + a thin `EDisGo` method wrapper (mirroring `spatial_complexity_reduction`). The task + `spatial_restore` just reads the stashed full grid + flexible sets from `ctx` and calls + it. Used **outside** the pipeline, a caller passes `full_grid`, `reduced_grid`, and the + flexible sets directly (no `ctx`). This gives symmetry: both halves = core fn + method + wrapper + thin task; both usable standalone; disaggregation rule lives/tested in core. +- (b) **`old_name`** carried on the reduced grid's `loads_df`/`generators_df` is + sufficient provenance for disaggregation — the reduced grid self-describes its origins, + so **no busmap needs stashing**. Full grid needed as the write target (holds individual + members + their pre-OPF weighting envelopes); reduced grid supplies optimized series + + `old_name`. Both grids are required args. + +**User's last message (the prompt to answer):** agrees restore logic should NOT live in +the task and should become its own eDisGo function; flexible-component names stored in +`ctx` and passed to the function, or passed differently when used outside the pipeline; +reduction is already an independent eDisGo function. + +→ So (a) and (b) are essentially aligned with the user's view; next step is to CONFIRM the +signature details (both grids as args; `old_name` sufficient, no busmap) and then move on. + +**RESOLVED (2026-07-14):** +- Core function signature: + `apply_reduced_results_to_full_grid(full_grid, reduced_grid, *, flexible_cps=None, + flexible_hps=None, flexible_loads=None, flexible_storage_units=None)` — four separate + kwargs, one per flexible-component type, each defaulting to `None`/skip. +- `EDisGo` method wrapper name: `map_reduced_results_to_full_grid` (full symmetry with the + core function name, no abbreviation). +- `old_name` on the reduced grid's `loads_df`/`generators_df` is CONFIRMED sufficient + provenance for disaggregation. No busmap/linemap stash on `ctx`. + +--- + +## Remaining questions still to grill (not yet discussed) + +- ~~Validator ordering~~ — **RESOLVED (2026-07-14).** Investigated + `edisgo/run/validator.py` (`validate()`, lines 47-139) + `edisgo/run/registry.py` + (`TaskMeta`, `register_task()`): ordering today is capability-based (`requires`/ + `provides` sets accumulated linearly across the pipeline, `validator.py:96,118-130`), + NOT a dependency graph and NOT named task-to-task precedence. The one existing hardcoded + exception is `reactive_power` must be last among `ts_altering` tasks + (`validator.py:110-116`). `select_timesteps`'s optional dual-position behavior + (`timeseries.py:328-403`) is NOT validator-enforced — it's a runtime-only check inside + the task, so it was not usable as a precedent. + - **Decision:** extend the existing capability system rather than add a new validator + concept or fall back to runtime-only checking (matches the pipeline's existing + mechanism everywhere else): + - `spatial_reduce` declares `provides={"reduced_grid"}`. + - `optimize` declares `provides={"optimized_dispatch", ...}` (in addition to its + existing provides). + - `spatial_restore` declares `requires={"reduced_grid", "optimized_dispatch"}`. + - This closes the gap where presence-only capability accumulation would otherwise let + `spatial_restore` validate successfully even if placed before `optimize` (both + `reduced_grid`-derived requirements would already be "satisfied" from + `spatial_reduce` alone) — requiring `optimized_dispatch` too means `spatial_restore` + cannot pass validation until `optimize` has actually appeared earlier in the + pipeline. + - **Correction (2026-07-14, during implementation):** `register_task`'s `requires`/ + `provides` (`edisgo/run/registry.py:52-58`) are fixed at decoration time (module + load), NOT evaluated per-run — so "optimize requires `reduced_grid` only when spatial + reduction is configured for THIS run" is not expressible and was dropped. + `optimize`'s `requires` stays exactly `{"timeseries", "flex"}`, unchanged — it does + not need to know spatial reduction exists. Ordering is fully enforced from + `spatial_restore`'s side alone; adding `reduced_grid` to `optimize`'s `requires` + unconditionally would have broken every existing preset that runs `optimize` without + `spatial_reduce` (uc2, uc4, uc5_select_timesteps). +- ~~YAML config surface~~ — **RESOLVED (2026-07-14).** Top-level `spatial_reduction:` + block, mirroring `timeseries_selection:`. Read via + `ctx.raw_config.get("spatial_reduction", {})` inside the `spatial_reduce` task, same + pattern as `select_timesteps` (`timeseries.py:415`). Holds `mode`, `cluster_area`, + `reduction_factor`, `reduction_factor_not_focused`, `aggregation_mode`, aggregation + sub-modes. +- ~~eGo injection~~ — **RESOLVED (2026-07-14).** Verified `timeseries_selection`'s actual + injection in `EDisGoNetworks._build_run_edisgo_config()`, + `eGo/ego/tools/edisgo_integration.py:675-685`: global `timeseries_selection` default + + `timeseries_selection_per_grid` dict keyed by `str(mv_grid_id)` + (`edisgo_integration.py:681-684`), **whole-block replacement** (not field-level merge), + key omitted from `cfg` entirely if both are unset (line 684: `if ts_selection is not + None`), letting the eDisGo preset's own default apply. Granularity is truly per + individual MV grid (`mv_grid_id`, looped in `run_all`, `edisgo_integration.py:597-626`). + - **Decision:** `spatial_reduction` replicates this exactly — global `spatial_reduction` + default + `spatial_reduction_per_grid` dict keyed by `str(mv_grid_id)`, whole-block + replacement, omitted if unset (same as `timeseries_selection`, not the simpler + `overlying_grid` hardcoded-fallback pattern). +- ~~Reactive power~~ — **RESOLVED (2026-07-14).** Investigated existing convention: + `pm_optimize`'s results-writer (`edisgo/io/powermodels_io.py::from_powermodels`, + lines 283-352) writes ONLY active power for flex components (heat pumps, CPs, DSM, + storage) into `_generators_active_power`/`_loads_active_power`/ + `_storage_units_active_power`; reactive power is untouched there. Immediately after + (line 354-355), it calls the plain `edisgo_object.set_time_series_reactive_power_control()` + — same generic fixed-cosphi default (`network/timeseries.py::fixed_cosphi`, + `flex_opt/q_control.py`) used everywhere else in eDisGo, applied blanket over the whole + object, not scoped to flex components. Confirmed the existing `reactive_power` pipeline + task (`edisgo/run/tasks/timeseries.py:584-629`) is just a thin wrapper around the exact + same call — no special-casing for OPF-derived components anywhere in the codebase today. + - Considered alternative: split reactive power proportionally to each `old_name` + member's share of the representative's active power (mirroring the active-power + disaggregation rule) instead of recomputing. **Verified mathematically equivalent** + under fixed-cosphi: all `old_name` members of one representative share the same + `type` → same `power_factor`, so `Q = P · tan(φ)` per member gives an identical result + whether derived by proportional split or by recomputing from each member's + disaggregated P directly. + - **Decision:** `spatial_restore` writes active power for flexible components onto the + full grid, then calls `set_time_series_reactive_power_control()` itself — mirrors + `pm_optimize`'s own convention exactly (write P, then blanket-recompute Q). Reuses the + existing method with no new reactive-power math anywhere, and makes `spatial_restore` + correct standalone even in pipelines with no downstream `reactive_power` task. +- ~~Testing strategy~~ — **RESOLVED (2026-07-14).** Scope for this first pass: core + function only (`apply_reduced_results_to_full_grid`), NOT pipeline/task-level + integration tests (deferred). Cover both aggregation modes: + - `aggregation_mode=False`: by-name write-back correctness. + - `aggregation_mode=True`: disaggregation math — per-step envelope-weighted split, + exact-sum-back-to-representative check, and the equal-split zero-envelope fallback. + - Stub/fake OPF results as fixtures; no real `pm_optimize` call, no real pipeline run + through `ctx`/validator. +- ~~uc5 preset~~ — **RESOLVED (2026-07-14).** New standalone preset + `edisgo/run/presets/uc5_spatial_reduction.yaml` (full copy of + `uc5_select_timesteps.yaml` + the spatial bracket, NOT an `extends` overlay — tasks + don't exist in code yet so this is documentation/example, and a standalone file matches + `uc5_select_timesteps.yaml`'s own self-contained style). Disable switch: explicit + `spatial_reduction.enabled` flag (mirrors `overlying_grid.enabled`, NOT + `timeseries_selection`'s absent-block-is-the-toggle style) — lets params stay in the + YAML while toggling on/off with one flag. Bracket placement: `spatial_reduce` right + before `optimize`, `spatial_restore` right after; `reactive_power` stays where it already + is (pre-OPF full-series cosphi on the reduced index), unaffected by the spatial bracket. + Final order: `select_timesteps(post_grid) → reactive_power → spatial_reduce → optimize → + spatial_restore → reinforce`. + +--- + +## Implementation (2026-07-14) + +Implemented and tested end-to-end (real venv, python3.10, `pip install -e ".[dev]"`, +real ding0 test grid `tests/data/ding0_test_network_1`): + +- `apply_reduced_results_to_full_grid` + + `EDisGo.map_reduced_results_to_full_grid` — + `edisgo/tools/spatial_complexity_reduction.py`, `edisgo/edisgo.py`. +- `spatial_reduce` / `spatial_restore` tasks — new file `edisgo/run/tasks/spatial.py`, + registered in `edisgo/run/tasks/__init__.py`. +- `RunContext.full_grid_stash` — new field, `edisgo/run/context.py`. +- `task_optimize` writes `flexible_cps`/`flexible_hps`/`flexible_loads`/ + `flexible_storage_units` to `ctx.flags`; `@register_task("optimize", ...)` gained + `provides={"optimized_dispatch"}` — `edisgo/run/tasks/analysis.py`. +- New preset `edisgo/run/presets/uc5_spatial_reduction.yaml` (already covered above). +- New tests `tests/tools/test_spatial_complexity_reduction.py::TestApplyReducedResultsToFullGrid` + (6 tests: by-name write-back, multi-member disaggregation sum check, singleton-rename + regression, time-index-mismatch error, storage-unit by-name path, reactive-power + recompute). Full existing suite (`tests/tools/`, `tests/run/`, `tests/opf/`) reverified + green: 99 passed, 1 unrelated skip. + +**Two real bugs found by a dedicated code-review agent (dispatched because no import- +capable env existed initially) and fixed before landing:** + +1. **Singleton-rename data corruption (serious).** Original code took a `_write_by_name` + fast path whenever a flexible-component set had NO multi-member merged group, + assuming an unmerged representative's name always equals its member's name. FALSE + under `aggregation_mode=True`: `spatial_complexity_reduction` renames **every** + group's representative, including singleton groups (a bus with exactly one flexible + load of a type/sector) — confirmed empirically on the real test grid (11 such + singletons exist in `ding0_test_network_1` alone). `_write_by_name` using the + representative's (renamed) name against `full_grid` (which only has the original, + un-renamed name) silently created a phantom column via pandas' `.loc[]` auto-vivify + behavior, leaving the real target column stale — a silent, hard-to-detect data + corruption, not a crash. **Fix:** removed the `_write_by_name` fast path for CPs/HPs/ + DSM loads entirely; always route through `_disaggregate`, which was proven correct for + every case (matching name, mismatched singleton, multi-member) by direct test. + `_write_by_name` now only serves storage units, which are genuinely never renamed. +2. **`flexible_* or []` crashes on numpy-array input (real, hit on first live + end-to-end run).** `task_optimize` derives `flexible_loads` as + `edisgo.dsm.p_min.columns.values` (`analysis.py:357`) — a numpy array — unlike the + other three `flexible_*` lists, which use `.tolist()`. `array or []` raises + `ValueError: The truth value of an array with more than one element is ambiguous...` + for any such array with 2+ elements. This surfaced immediately on the very first real + pipeline run (`run_example_06.py`, `uc6_spatial_reduction.yaml`, `aggregation_mode: + false`) — the OPF/Gurobi solve completed successfully, `spatial_restore` crashed on + the very next line. **Fix:** replaced `flexible_x = flexible_x or []` with + `flexible_x = list(flexible_x) if flexible_x is not None else []` for all four + parameters in `apply_reduced_results_to_full_grid` — `is None` is the correct + emptiness check for an optional list-like argument that may be a list, tuple, or numpy + array (the codebase's own docstrings elsewhere already document these params as + accepting `numpy.ndarray or None`). Added regression test + `test_accepts_numpy_array_flexible_component_lists`. +3. **Unguarded `KeyError` on time-index mismatch (robustness).** If a flexibility-band/ + DSM/heat-pump attribute on `full_grid` doesn't cover `full_grid.timeseries.timeindex` + (e.g. the full-grid stash was taken before time-index selection ran — a real risk for + any pipeline not following the `select_timesteps` → `spatial_reduce` convention, since + nothing in the registry enforces that order), the disaggregation envelope lookup would + raise a bare `KeyError` deep inside a `.loc` call. **Fix:** added + `_require_full_timeindex`, a defensive check before each envelope lookup that raises a + clear `ValueError` naming the mismatch and the likely cause. Decided against also + adding a validator `requires={"timeseries"}` declaration — the pipeline is already + constructed so a time index is guaranteed set before `spatial_reduce` runs (decision 7, + above), so the defensive check alone is sufficient without touching registry metadata. + +## ADR candidates (offer at end of session) + +1. "Spatial reduction as two bracketing pipeline tasks, restore optional" — hard to + reverse, surprising (breaks the pm_optimize-owns-its-logic principle), real trade-off + (consistency vs stop-early). → write when design settles. +2. Possibly: "Disaggregation by pre-OPF flexibility envelope per time step" — a modeling + choice with alternatives (static scalar, proportional-to-original-series). Borderline; + decide at end. diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 140e290ae..e3ca90422 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -64,7 +64,10 @@ from edisgo.tools import plots, tools from edisgo.tools.config import Config from edisgo.tools.geo import find_nearest_bus -from edisgo.tools.spatial_complexity_reduction import spatial_complexity_reduction +from edisgo.tools.spatial_complexity_reduction import ( + apply_reduced_results_to_full_grid, + spatial_complexity_reduction, +) from edisgo.tools.tools import ( determine_grid_integration_voltage_level, get_path_length_to_station, @@ -266,9 +269,7 @@ def run_pipeline(self, config, overlying_grid_data=None): """ from edisgo.run import _run_pipeline_on - return _run_pipeline_on( - self, config, overlying_grid_data=overlying_grid_data - ) + return _run_pipeline_on(self, config, overlying_grid_data=overlying_grid_data) def import_ding0_grid(self, path, legacy_ding0_grids=True): """ @@ -3578,6 +3579,65 @@ def spatial_complexity_reduction( ) return edisgo_obj, busmap_df, linemap_df + def map_reduced_results_to_full_grid( + self, + reduced_grid: EDisGo, + flexible_cps: list | None = None, + flexible_hps: list | None = None, + flexible_loads: list | None = None, + flexible_storage_units: list | None = None, + ) -> EDisGo: + """ + Writes optimized flexible-component dispatch from a spatially-reduced + grid back onto this (full) grid. + + Counterpart to :meth:`spatial_complexity_reduction`: where that + method shrinks this grid for a faster OPF, this method maps the OPF's + active-power results from ``reduced_grid`` back onto ``self`` so + reinforcement can run on the full topology. Only components the OPF + actually rewrites are touched — flexible charging points, heat pumps, + DSM loads, and storage units. Inflexible loads/generators are + untouched, since the OPF never changed their series and ``self`` + already holds the correct values for them. + + See + :func:`~.tools.spatial_complexity_reduction.apply_reduced_results_to_full_grid` + for the full matching/disaggregation rules and the reactive-power + recompute this method triggers as a side effect. + + Parameters + ---------- + reduced_grid : :class:`~.EDisGo` + The spatially-reduced EDisGo instance the OPF ran on. Supplies + the optimized active-power series and, if aggregated, the + ``old_name`` provenance for disaggregation. + flexible_cps : list of str, optional + Names of flexible charging points in ``reduced_grid`` to map + back. + flexible_hps : list of str, optional + Names of flexible heat-pump loads in ``reduced_grid`` to map + back. + flexible_loads : list of str, optional + Names of flexible DSM loads in ``reduced_grid`` to map back. + flexible_storage_units : list of str, optional + Names of flexible storage units in ``reduced_grid`` to map back. + + Returns + ------- + :class:`~.EDisGo` + ``self``, with active power written for the given flexible + components and reactive power recomputed. + + """ + return apply_reduced_results_to_full_grid( + full_grid=self, + reduced_grid=reduced_grid, + flexible_cps=flexible_cps, + flexible_hps=flexible_hps, + flexible_loads=flexible_loads, + flexible_storage_units=flexible_storage_units, + ) + def check_integrity(self): """ Method to check the integrity of the EDisGo object. diff --git a/edisgo/run/context.py b/edisgo/run/context.py index e41a07fc9..88cc18c7a 100644 --- a/edisgo/run/context.py +++ b/edisgo/run/context.py @@ -1,3 +1,13 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later """ Runtime context passed to every task during pipeline execution. @@ -19,6 +29,7 @@ Tasks should treat ``flags`` as advisory — they MAY short-circuit based on a flag but MUST NOT assume a flag is present. """ + from __future__ import annotations import logging @@ -67,6 +78,11 @@ class RunContext: ``overlying_grid_data=`` argument of :func:`edisgo.run.run_edisgo`. Consumed by the ``import_overlying_grid_data`` task when ``overlying_grid.source == "etrago"``. + full_grid_stash : edisgo.EDisGo or None + The pre-reduction :class:`~edisgo.EDisGo` instance, deepcopied and + stashed by the ``spatial_reduce`` task before it spatially reduces + the working object. Consumed (and cleared back to ``None``) by + ``spatial_restore``. ``None`` when spatial reduction is not in use. """ @@ -81,6 +97,7 @@ class RunContext: current_stage: str | None = None raw_config: dict[str, Any] = field(default_factory=dict) overlying_grid_data: Any = None + full_grid_stash: Any = None def ensure_engine(self): """ diff --git a/edisgo/run/presets/uc5_select_timesteps.yaml b/edisgo/run/presets/uc5_select_timesteps.yaml index 426ce1231..82431bc7d 100644 --- a/edisgo/run/presets/uc5_select_timesteps.yaml +++ b/edisgo/run/presets/uc5_select_timesteps.yaml @@ -46,8 +46,8 @@ grid: legacy_ding0_grids: false database: - ssh: - enabled: false + source: local + # No explicit base time index is set. oedb_ts falls back to a full year derived # from the scenario when none is given, which is what auto interval selection diff --git a/edisgo/run/presets/uc6_spatial_reduction.yaml b/edisgo/run/presets/uc6_spatial_reduction.yaml new file mode 100644 index 000000000..767327552 --- /dev/null +++ b/edisgo/run/presets/uc6_spatial_reduction.yaml @@ -0,0 +1,149 @@ +_comment: | + UC6 — OPF with configurable timestep selection (manual OR auto), PLUS spatial + complexity reduction bracketing the OPF step. Standalone copy of + uc5_select_timesteps.yaml (not an `extends` overlay) with the spatial_reduce / + spatial_restore bracket added around `optimize`. + + The pipeline carries TWO select_timesteps steps, each with a `position`: + - position: pre_import (before import_heat_pumps) — acts only in MANUAL + mode. It sets the explicit time index, which the heat-pump/DSM imports + and oedb_ts then use to fetch only the selected steps (cheap). + - position: post_grid (after import_overlying_grid_data, before + reactive_power) — acts only in AUTO mode. It needs all active-power + time series (incl. overlying-grid generation) set to run the scoring + power flow via get_most_critical_time_intervals. + Whichever mode is configured, the other positioned step is a no-op. + + Auto mode normally yields two disconnected intervals (one overloading, + one voltage). They are kept separate (a gap in the time index); if they + overlap, a non-overlapping pair is chosen if possible, otherwise they are + concatenated into one interval. A later optimize step can detect the gap + and run separate optimizations per interval. + + Spatial complexity reduction (spatial_reduce / spatial_restore) brackets + `optimize` only: + - spatial_reduce (before optimize): deepcopies and stashes the full grid on + ctx, then spatially reduces the working object so `optimize` runs on a + smaller grid. + - spatial_restore (after optimize): writes the optimized flexible-component + dispatch back onto the stashed full grid (by name, or disaggregated onto + `old_name` members if aggregation_mode is true), recomputes reactive power + for those components, and makes the full grid active again for `reinforce`. + Both are no-ops when `spatial_reduction.enabled` is false (default), so + `reinforce` then runs on the same grid `optimize` used, same as + uc5_select_timesteps.yaml. `reactive_power` (pre-OPF, full time series) stays + where it already was, unaffected by the spatial bracket. + Reinforcement always runs on the full topology, regardless of the flag. + +_workflow: + - setup_grid: load ding0 topology + - import_generators / import_home_batteries + - select_timesteps (pre_import): manual only — set explicit time index + - import_heat_pumps / import_dsm: fetch only selected steps (manual) + - import_electromobility: dumb charging, flex bands + - oedb_ts: real wind/solar + load time series + - apply_charging_strategy / apply_heat_pump_strategy + - build_flexibility_bands: EV bands on the fixed hourly index + - import_overlying_grid_data: HV constraints from CSV dir + - select_timesteps (post_grid): auto only — reduce to critical intervals + - reactive_power: fixed cosphi on the reduced index + - spatial_reduce: no-op unless spatial_reduction.enabled — stash full grid, + reduce working object + - optimize: pm_optimize with flex assets, on the (possibly) reduced grid + - spatial_restore: no-op unless spatial_reduction.enabled — write dispatch + back onto the stashed full grid, recompute reactive power + - reinforce / save: always on the full topology + +# Self-contained (no `extends`): everything uc4_example provided is inlined +# below, so this preset can be run on its own via +# run_edisgo({"extends": "uc6_spatial_reduction", "grid": {"ding0_path": ...}}) +scenario: eGon2035 + +grid: + ding0_path: "/path/to/ding0_grid" + legacy_ding0_grids: false + +database: + source: local + + +# No explicit base time index is set. oedb_ts falls back to a full year derived +# from the scenario when none is given, which is what auto interval selection +# needs (week-long critical intervals to pick from). For manual selection the +# pre-import select_timesteps step sets the index instead. + +overlying_grid: + enabled: true # set true to activate import_overlying_grid_data + source: csv # "csv" (load from path) or "etrago" (kwarg) + path: "/home/gurobi/.edisgo_input/overlying_grid" + +results: + directory: results/uc6_spatial_reduction + +# Top-level block read by the select_timesteps task via ctx.raw_config. +# eGo can inject this block the same way it injects overlying_grid. +# Set `mode` (and its parameters) here or override it in the run script. +timeseries_selection: + mode: manual + # auto method: "power_flow" (default, scores intervals via a power flow) or + # "residual_load" (no power flow — the weeks ending at the max/min residual-load + # time steps; requires overlying-grid data). + method: residual_load + # --- shared auto parameters (both methods) --- + time_steps_per_time_interval: 168 # one week (must be a multiple of 24) + time_step_day_start: 4 # hour of day the intervals start/end on + # --- power_flow method parameters --- + percentage: 1.0 + save_steps: true # write selected intervals CSV to results_dir + use_troubleshooting_mode: true # handle power-flow non-convergence + overloading_factor: 0.95 + voltage_deviation_factor: 0.95 + # --- manual parameters (used when mode: manual) --- + # timestamps: ["2035-01-15 08:00", "2035-01-15 9:00", "2035-01-15 10:00"] + # or a range instead of `timestamps`: + start: "2035-01-15 00:00" + periods: 24 + freq: h + +# Top-level block read by the spatial_reduce/spatial_restore tasks via +# ctx.raw_config. eGo can inject this block the same way it injects +# overlying_grid/timeseries_selection (global `spatial_reduction` default + +# per-grid `spatial_reduction_per_grid` override keyed by mv_grid_id). +spatial_reduction: + enabled: true # set true to activate spatial_reduce/spatial_restore + mode: kmeansdijkstra # clustering mode for spatial_complexity_reduction + cluster_area: feeder + reduction_factor: 0.3 + reduction_factor_not_focused: False + aggregation_mode: false # start with false; true enables load/generator merging + +pipeline: + - setup_grid + - import_generators + - import_home_batteries + - select_timesteps: {position: pre_import} # acts in manual mode only + - import_heat_pumps + - import_dsm + - import_electromobility: + charging_strategy: null + # flexibility bands are built later (build_flexibility_bands), once the + # analysis time index is fixed, so they are resampled to it + - oedb_ts: + dispatchable: {other: 0.7} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - build_flexibility_bands # hourly bands on the 2035 index + - import_overlying_grid_data + - select_timesteps: {position: post_grid} # acts in auto mode only + - reactive_power + - spatial_reduce # no-op unless spatial_reduction.enabled + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - spatial_restore # no-op unless spatial_reduction.enabled + - reinforce: + catch_convergence_problems: true + - save: + archive: true + save_opf_results: true diff --git a/edisgo/run/tasks/__init__.py b/edisgo/run/tasks/__init__.py index 9a1be82a8..a0ef5d22c 100644 --- a/edisgo/run/tasks/__init__.py +++ b/edisgo/run/tasks/__init__.py @@ -26,6 +26,7 @@ * :mod:`.analysis` — ``check_integrity``, ``analyze``, ``reinforce``, ``base_reinforce``, ``optimize`` * :mod:`.io` — ``save``, ``load_charging_from_files`` +* :mod:`.spatial` — ``spatial_reduce``, ``spatial_restore`` Task signature convention: ``(edisgo, ctx, **params)``. A task may mutate ``edisgo`` in place and/or return a new EDisGo instance (the @@ -33,4 +34,11 @@ loop). """ -from edisgo.run.tasks import analysis, flex, grid, io, timeseries # noqa: F401 +from edisgo.run.tasks import ( # noqa: F401 + analysis, + flex, + grid, + io, + spatial, + timeseries, +) diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py index 7bbb4ba3f..9f8ec57ec 100644 --- a/edisgo/run/tasks/analysis.py +++ b/edisgo/run/tasks/analysis.py @@ -268,7 +268,9 @@ def task_base_reinforce( return edisgo -@register_task("optimize", requires={"timeseries", "flex"}) +@register_task( + "optimize", requires={"timeseries", "flex"}, provides={"optimized_dispatch"} +) def task_optimize( edisgo, ctx, @@ -307,7 +309,11 @@ def task_optimize( EDisGo instance to optimize. ctx : RunContext Run context. Used for logging and, for multi-interval runs, nothing - else is required from it. + else is required from it. The resolved ``flexible_*`` name lists are + written to ``ctx.flags['flexible_cps']`` / ``ctx.flags['flexible_hps']`` + / ``ctx.flags['flexible_loads']`` / ``ctx.flags['flexible_storage_units']`` + so a later ``spatial_restore`` step knows which components' dispatch + needs mapping back onto the full grid. flexible : list of str, optional High-level selector, subset of ``{"heat_pumps", "charging_points", "storage"}``. If ``None``, nothing is @@ -359,6 +365,11 @@ def task_optimize( if flexible_storage_units is None: flexible_storage_units = [] + ctx.flags["flexible_cps"] = flexible_cps + ctx.flags["flexible_hps"] = flexible_hps + ctx.flags["flexible_loads"] = flexible_loads + ctx.flags["flexible_storage_units"] = flexible_storage_units + # pm_optimize handles a non-contiguous (reduced) time index internally: # it runs one OPF per contiguous interval and merges the results. edisgo.pm_optimize( diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py index 35cce7dce..9c3edab80 100644 --- a/edisgo/run/tasks/flex.py +++ b/edisgo/run/tasks/flex.py @@ -216,6 +216,14 @@ def task_build_flexibility_bands(edisgo, ctx, *, use_case=None): ``import_heat_pumps``, and is more efficient than building bands over a non-final index. + ``get_flexibility_bands`` only resamples to the active *frequency* - the + bands still span whatever date range the underlying SimBEV charging- + process data covers, which is not necessarily the same range as + ``edisgo.timeseries.timeindex`` (e.g. a manually-selected window). This + task additionally trims the bands down to that exact index, so + ``electromobility.flexibility_bands`` always matches + ``edisgo.timeseries.timeindex`` after this step runs. + Parameters ---------- edisgo : edisgo.EDisGo @@ -232,9 +240,20 @@ def task_build_flexibility_bands(edisgo, ctx, *, use_case=None): edisgo.EDisGo The modified EDisGo instance. """ + from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex + if use_case is None: use_case = ["home", "work", "public", "hpc"] edisgo.electromobility.get_flexibility_bands(edisgo, use_case=use_case) + reduce_timeseries_data_to_given_timeindex( + edisgo, + edisgo.timeseries.timeindex, + timeseries=False, + electromobility=True, + heat_pump=False, + dsm=False, + overlying_grid=False, + ) return edisgo diff --git a/edisgo/run/tasks/spatial.py b/edisgo/run/tasks/spatial.py new file mode 100644 index 000000000..c7c893b08 --- /dev/null +++ b/edisgo/run/tasks/spatial.py @@ -0,0 +1,148 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +""" +Spatial complexity reduction tasks bracketing ``optimize``. + +* :func:`task_spatial_reduce` (``spatial_reduce``) — stashes a deepcopy of + the full grid on ``ctx`` and spatially reduces the working object so + ``optimize`` runs on a smaller grid. +* :func:`task_spatial_restore` (``spatial_restore``) — writes the optimized + flexible-component dispatch back onto the stashed full grid and makes it + the active object again, so ``reinforce`` runs on the full topology. + +Both are no-ops when ``spatial_reduction.enabled`` is false (the default), +so a pipeline that carries this bracket behaves exactly like one that +doesn't when spatial reduction is turned off. +""" + +from __future__ import annotations + +import copy + +from edisgo.run.registry import register_task + + +@register_task("spatial_reduce", requires={"grid"}, provides={"reduced_grid"}) +def task_spatial_reduce(edisgo, ctx, **overrides): + """ + Deepcopy and stash the full grid, then spatially reduce the working + object. + + Configuration is read from the top-level ``spatial_reduction:`` config + block (so eGo can inject it the same way it injects + ``timeseries_selection``); inline step params override individual keys + of that block. + + A no-op when ``enabled`` is not true — ``edisgo`` is returned unchanged + and ``ctx.full_grid_stash`` is left ``None``, so a downstream + ``spatial_restore`` also no-ops (see its docstring) and ``optimize``/ + ``reinforce`` run on the same, unreduced grid as if this task were + absent from the pipeline. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to spatially reduce in place. + ctx : RunContext + Run context. Reads ``ctx.raw_config['spatial_reduction']``. Sets + ``ctx.full_grid_stash`` to the pre-reduction deepcopy. + **overrides + Inline step params overriding keys of the ``spatial_reduction`` + block. Recognized keys: ``enabled`` (bool, default ``False``), + ``mode``, ``cluster_area``, ``reduction_factor``, + ``reduction_factor_not_focused``, ``aggregation_mode``, and the + aggregation sub-modes ``load_aggregation_mode`` / + ``generator_aggregation_mode`` — forwarded to + :meth:`~.EDisGo.spatial_complexity_reduction`. + + Returns + ------- + edisgo.EDisGo + The (possibly) spatially-reduced EDisGo instance. + + """ + cfg = {**ctx.raw_config.get("spatial_reduction", {}), **overrides} + if not cfg.get("enabled", False): + return edisgo + + ctx.full_grid_stash = copy.deepcopy(edisgo) + + kwargs = { + k: v + for k, v in cfg.items() + if k + in ( + "mode", + "cluster_area", + "reduction_factor", + "reduction_factor_not_focused", + "apply_pseudo_coordinates", + "aggregation_mode", + "load_aggregation_mode", + "generator_aggregation_mode", + "line_naming_convention", + "mv_pseudo_coordinates", + ) + } + edisgo.spatial_complexity_reduction(copy_edisgo=False, **kwargs) + return edisgo + + +@register_task("spatial_restore", requires={"reduced_grid", "optimized_dispatch"}) +def task_spatial_restore(edisgo, ctx, **overrides): + """ + Write optimized flexible-component dispatch back onto the stashed full + grid, and make it the active object again. + + Reads the flexible-component name lists ``optimize`` wrote to + ``ctx.flags`` and passes them, together with ``edisgo`` (the reduced, + just-optimized grid) and ``ctx.full_grid_stash`` (the pre-reduction + grid), to :meth:`~.EDisGo.map_reduced_results_to_full_grid`. See that + method (and the core function it wraps, + :func:`~.tools.spatial_complexity_reduction.apply_reduced_results_to_full_grid`) + for the matching/disaggregation rules. + + A no-op when ``ctx.full_grid_stash`` is ``None`` — i.e. when + ``spatial_reduce`` did not run or ran disabled — so ``edisgo`` (the + grid ``optimize`` already ran on) is returned unchanged. + + Parameters + ---------- + edisgo : edisgo.EDisGo + The reduced EDisGo instance ``optimize`` ran on. + ctx : RunContext + Run context. Reads ``ctx.full_grid_stash`` and the + ``flexible_cps`` / ``flexible_hps`` / ``flexible_loads`` / + ``flexible_storage_units`` flags ``optimize`` set. Clears + ``ctx.full_grid_stash`` back to ``None`` after restoring. + **overrides + Unused; accepted for signature consistency with other tasks. + + Returns + ------- + edisgo.EDisGo + The full-grid EDisGo instance with flexible dispatch restored, or + ``edisgo`` unchanged if there is no stash to restore from. + + """ + full_grid = ctx.full_grid_stash + if full_grid is None: + return edisgo + + full_grid.map_reduced_results_to_full_grid( + reduced_grid=edisgo, + flexible_cps=ctx.flags.get("flexible_cps"), + flexible_hps=ctx.flags.get("flexible_hps"), + flexible_loads=ctx.flags.get("flexible_loads"), + flexible_storage_units=ctx.flags.get("flexible_storage_units"), + ) + ctx.full_grid_stash = None + return full_grid diff --git a/edisgo/tools/spatial_complexity_reduction.py b/edisgo/tools/spatial_complexity_reduction.py index 21e8f8665..ad9a51fad 100644 --- a/edisgo/tools/spatial_complexity_reduction.py +++ b/edisgo/tools/spatial_complexity_reduction.py @@ -1916,6 +1916,237 @@ def spatial_complexity_reduction( return busmap_df, linemap_df +def apply_reduced_results_to_full_grid( + full_grid: EDisGo, + reduced_grid: EDisGo, + *, + flexible_cps: list | None = None, + flexible_hps: list | None = None, + flexible_loads: list | None = None, + flexible_storage_units: list | None = None, +) -> EDisGo: + """ + Write optimized flexible-component dispatch from a spatially-reduced grid + back onto the full grid. + + Counterpart to :func:`spatial_complexity_reduction`: where that function + shrinks a grid for a faster OPF, this function maps the OPF's active-power + results back onto the pre-reduction grid so reinforcement can run on the + full topology. Only components the OPF actually rewrites are touched — + flexible charging points, heat pumps, DSM loads, and storage units. + Inflexible loads/generators are untouched: the OPF never changed their + series, so ``full_grid`` already holds the correct values for them. + + ``full_grid`` and ``reduced_grid`` are matched by name for storage units + (never aggregated by :func:`spatial_complexity_reduction`, so their names + are unchanged) and, for the other three flexibility types, by the + ``old_name`` column that :func:`spatial_complexity_reduction` writes onto + ``reduced_grid.topology.loads_df`` when ``aggregation_mode=True``. A + member listed in ``old_name`` is a load whose active-power series was + merged into one representative row; when ``aggregation_mode=False`` (or a + given member was not merged), ``old_name`` is absent and the member's own + name is used directly — i.e. a plain by-name write-back. + + For a merged representative, the representative's optimized series is + disaggregated onto its ``old_name`` members **per time step**, weighted by + each member's own pre-OPF flexibility envelope (a known input, never the + optimized result): + + * charging points — ``upper_power(t)`` from + ``electromobility.flexibility_bands`` (a charging point with no + connected vehicle has ``upper_power(t) == 0``, so it receives none of + the representative's dispatch that time step); + * heat pumps — ``min(heat_demand(t) / cop(t), p_set)``, i.e. the + electrical-equivalent heat demand capped at the heat pump's own rated + power, mirroring how a charging point's ``upper_power(t)`` is already a + capped bound rather than raw uncapped demand; + * DSM loads — ``p_max(t)`` from :attr:`~.network.dsm.DSM.p_max`. + + Weights always sum back to the representative's value exactly at every + time step; a time step where every member's weight is 0 falls back to an + equal split. + + Reactive power is not read from ``reduced_grid``. After writing active + power, this function calls + :meth:`~.EDisGo.set_time_series_reactive_power_control` on ``full_grid`` + with its defaults, mirroring how the OPF itself derives reactive power + for the components it just optimized (see + :func:`~.io.powermodels_io.from_powermodels`) — reactive power is always + a function of whatever active power is currently set, regardless of + whether that active power came from a default, worst case, or the OPF. + + Parameters + ---------- + full_grid : :class:`~.EDisGo` + The pre-reduction EDisGo instance to write dispatch onto, modified in + place. Must contain every component named in ``flexible_cps`` / + ``flexible_hps`` / ``flexible_loads`` / ``flexible_storage_units`` and + (for merged components) every name listed in ``reduced_grid``'s + ``old_name`` columns. + reduced_grid : :class:`~.EDisGo` + The spatially-reduced EDisGo instance the OPF ran on. Supplies the + optimized active-power series and, if aggregated, the ``old_name`` + provenance. + flexible_cps : list of str, optional + Names of flexible charging points in ``reduced_grid`` to map back. + flexible_hps : list of str, optional + Names of flexible heat-pump loads in ``reduced_grid`` to map back. + flexible_loads : list of str, optional + Names of flexible DSM loads in ``reduced_grid`` to map back. + flexible_storage_units : list of str, optional + Names of flexible storage units in ``reduced_grid`` to map back. + + Returns + ------- + :class:`~.EDisGo` + ``full_grid``, with active power written for the given flexible + components and reactive power recomputed. + + """ + # NOTE: "x or []" is unsafe here - callers may pass a numpy array (e.g. + # task_optimize derives flexible_loads as + # edisgo.dsm.p_min.columns.values), and "array or []" raises + # ValueError ("truth value of an array... is ambiguous") for any array + # with more than one element. "is None" is the correct emptiness check + # for an optional list-like argument. + flexible_cps = list(flexible_cps) if flexible_cps is not None else [] + flexible_hps = list(flexible_hps) if flexible_hps is not None else [] + flexible_loads = list(flexible_loads) if flexible_loads is not None else [] + flexible_storage_units = ( + list(flexible_storage_units) if flexible_storage_units is not None else [] + ) + + def _require_full_timeindex(envelope: DataFrame, envelope_name: str) -> None: + """Raise a clear error if ``envelope`` doesn't cover the full grid's + active time index, instead of a bare ``KeyError`` deep inside a + ``.loc`` lookup. + + This can only happen if ``full_grid``'s flexibility-band/DSM/heat-pump + attributes were never trimmed to the same time index as + ``full_grid.timeseries.timeindex`` - i.e. if the pre-reduction stash + was taken before the run's time index was finalized. + """ + ti = full_grid.timeseries.timeindex + missing = ti.difference(envelope.index) + if len(missing) > 0: + raise ValueError( + f"apply_reduced_results_to_full_grid: full_grid's " + f"{envelope_name} does not cover {len(missing)} of " + f"full_grid.timeseries.timeindex's time steps (e.g. " + f"{missing[0]!r}). This usually means the full-grid stash " + f"was taken before the time index was finalized - run " + f"time-index selection (e.g. select_timesteps) before " + f"spatial_reduce." + ) + + def _old_name_map(loads_df: DataFrame, names: list) -> dict: + """Map each representative name in ``names`` to its member names. + + A name absent from ``old_name`` (not merged, or + ``aggregation_mode=False``) maps to itself. + """ + name_map = {} + for name in names: + old_name = loads_df.at[name, "old_name"] if "old_name" in loads_df else None + name_map[name] = old_name if isinstance(old_name, list) else [name] + return name_map + + def _write_by_name(active_power: DataFrame, names: list, target: DataFrame) -> None: + ti = full_grid.timeseries.timeindex + target.loc[ti, names] = active_power.loc[ti, names].values + + def _disaggregate( + active_power: DataFrame, + name_map: dict, + envelope: DataFrame, + target: DataFrame, + ) -> None: + """Split each representative's series onto its members per time step. + + ``envelope`` holds each member's pre-OPF flexibility envelope + (columns = member names, index = time index); members missing from + ``envelope`` are treated as having an all-zero envelope (equal-split + fallback). + """ + ti = full_grid.timeseries.timeindex + for representative, members in name_map.items(): + if len(members) == 1 and members[0] == representative: + target.loc[ti, representative] = active_power.loc[ti, representative] + continue + weights = pd.DataFrame(index=ti, columns=members, dtype=float) + for member in members: + weights[member] = ( + envelope.loc[ti, member] if member in envelope.columns else 0.0 + ) + weight_sum = weights.sum(axis="columns") + zero_envelope = weight_sum == 0 + shares = weights.div(weight_sum.replace(0, np.nan), axis="index") + shares.loc[zero_envelope, :] = 1.0 / len(members) + representative_power = active_power.loc[ti, representative] + for member in members: + target.loc[ti, member] = shares[member] * representative_power + + reduced_loads_df = reduced_grid.topology.loads_df + full_loads_df = full_grid.topology.loads_df + + # Always routed through _disaggregate (never the by-name fast path): under + # aggregation_mode=True, spatial_complexity_reduction renames EVERY group's + # representative row, including singleton groups (a bus with exactly one + # flexible load of a given type/sector) - so a representative's own name + # can differ from its single old_name member's name. _disaggregate already + # handles that case correctly (a singleton's one weight, whether zero or + # not, always resolves its share to the representative's full value), so + # there is no correct case left for a by-name fast path to shortcut. + if flexible_cps: + name_map = _old_name_map(reduced_loads_df, flexible_cps) + envelope = reduced_grid.electromobility.flexibility_bands["upper_power"] + _require_full_timeindex(envelope, "electromobility.flexibility_bands") + _disaggregate( + reduced_grid.timeseries.loads_active_power, + name_map, + envelope, + full_grid.timeseries._loads_active_power, + ) + + if flexible_hps: + name_map = _old_name_map(reduced_loads_df, flexible_hps) + members_flat = [m for members in name_map.values() for m in members] + heat_demand = full_grid.heat_pump.heat_demand_df[members_flat] + cop = full_grid.heat_pump.cop_df[members_flat] + p_set = full_loads_df.p_set[members_flat] + envelope = (heat_demand / cop).clip(upper=p_set, axis="columns") + _require_full_timeindex(envelope, "heat_pump.heat_demand_df/cop_df") + _disaggregate( + reduced_grid.timeseries.loads_active_power, + name_map, + envelope, + full_grid.timeseries._loads_active_power, + ) + + if flexible_loads: + name_map = _old_name_map(reduced_loads_df, flexible_loads) + _require_full_timeindex(full_grid.dsm.p_max, "dsm.p_max") + _disaggregate( + reduced_grid.timeseries.loads_active_power, + name_map, + full_grid.dsm.p_max, + full_grid.timeseries._loads_active_power, + ) + + if flexible_storage_units: + # Storage units are never aggregated by spatial_complexity_reduction + # (only bus-relabeled), so this is always a plain by-name write-back. + _write_by_name( + reduced_grid.timeseries.storage_units_active_power, + flexible_storage_units, + full_grid.timeseries._storage_units_active_power, + ) + + full_grid.set_time_series_reactive_power_control() + + return full_grid + + def compare_voltage( edisgo_unreduced: EDisGo, edisgo_reduced: EDisGo, diff --git a/run_example_06.py b/run_example_06.py new file mode 100644 index 000000000..fb40baeb0 --- /dev/null +++ b/run_example_06.py @@ -0,0 +1,37 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Runner für uc6_spatial_reduction.yaml — einfach ``python run_example_05.py``.""" + +import logging + +from edisgo.run.runner import run_edisgo + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(name)s %(levelname)s: %(message)s", +) + +# edisgo = run_edisgo("/storage/JoDa/ego/edisgo_run_edisgo/eDisGo/edisgo/run/presets/uc4_example_MS.yaml") # noqa: E501 +edisgo = run_edisgo( + { + "extends": "uc6_spatial_reduction.yaml", + # "grid": {"ding0_path": "/home/gurobi/.ding0/run_hetzner_59763_2023_04_06/ding0_grids/32355"} # noqa: E501 + "grid": { + "ding0_path": "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" # noqa: E501 + }, + # OG path must be the leaf dir for THIS grid (like ding0_path), not the parent. + "overlying_grid": {"path": "/home/gurobi/.edisgo_input/overlying_grid/32377"}, + } +) + +print("\n=== Fertig ===") +print("Ausbaukosten:\n", edisgo.results.grid_expansion_costs) +print("\nUngelöste Probleme:\n", edisgo.results.unresolved_issues) diff --git a/tests/tools/test_spatial_complexity_reduction.py b/tests/tools/test_spatial_complexity_reduction.py index 1747e85ae..425255dee 100644 --- a/tests/tools/test_spatial_complexity_reduction.py +++ b/tests/tools/test_spatial_complexity_reduction.py @@ -3,6 +3,7 @@ from contextlib import nullcontext as does_not_raise import numpy as np +import pandas as pd import pytest from edisgo import EDisGo @@ -396,3 +397,254 @@ def test_remove_short_end_lines(self, test_edisgo_obj): # assert len(edisgo_root.topology.lines_df) - 1 == len( # edisgo_clean.topology.lines_df # ) + + +class TestApplyReducedResultsToFullGrid: + """ + Tests for + :func:`~.tools.spatial_complexity_reduction.apply_reduced_results_to_full_grid`. + + Uses stub OPF results (directly writing to + ``reduced_grid.timeseries._loads_active_power`` / + ``_storage_units_active_power``) rather than running a real OPF, since + what is under test is the map-back/disaggregation logic, not + ``pm_optimize`` itself. + """ + + @pytest.fixture(autouse=True) + def test_edisgo_obj(self): + edisgo_root = EDisGo(ding0_grid=pytest.ding0_test_network_path) + edisgo_root.set_time_series_worst_case_analysis() + make_pseudo_coordinates(edisgo_root) + return edisgo_root + + @pytest.fixture + def full_and_reduced(self, test_edisgo_obj): + full_grid = copy.deepcopy(test_edisgo_obj) + reduced_grid, _, _ = full_grid.spatial_complexity_reduction( + copy_edisgo=True, + mode="kmeansdijkstra", + cluster_area="feeder", + reduction_factor=0.1, + aggregation_mode=True, + load_aggregation_mode="bus", + ) + return full_grid, reduced_grid + + def _first_representative_with(self, reduced_grid, n_members): + loads_df = reduced_grid.topology.loads_df + candidates = loads_df[ + loads_df["old_name"].apply( + lambda v: isinstance(v, list) and len(v) == n_members + ) + ] + assert not candidates.empty, ( + f"fixture grid has no aggregated load representative with " + f"exactly {n_members} old_name member(s); adjust the fixture " + f"or reduction_factor." + ) + return candidates.index[0] + + def test_by_name_write_back_aggregation_mode_false(self, test_edisgo_obj): + # aggregation_mode=False: no merging, so restore is a plain by-name + # write-back for every flexibility type. + full_grid = copy.deepcopy(test_edisgo_obj) + reduced_grid, _, _ = full_grid.spatial_complexity_reduction( + copy_edisgo=True, + mode="kmeansdijkstra", + cluster_area="feeder", + reduction_factor=0.1, + aggregation_mode=False, + ) + ti = full_grid.timeseries.timeindex + load_name = reduced_grid.topology.loads_df.index[0] + storage_name = reduced_grid.topology.storage_units_df.index[0] + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[load_name]) + + reduced_grid.timeseries._loads_active_power.loc[ti, load_name] = [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + reduced_grid.timeseries._storage_units_active_power.loc[ti, storage_name] = [ + 5.0, + 6.0, + 7.0, + 8.0, + ] + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, + reduced_grid=reduced_grid, + flexible_loads=[load_name], + flexible_storage_units=[storage_name], + ) + + assert result.timeseries.loads_active_power.loc[ti, load_name].tolist() == [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + assert result.timeseries.storage_units_active_power.loc[ + ti, storage_name + ].tolist() == [5.0, 6.0, 7.0, 8.0] + + def test_accepts_numpy_array_flexible_component_lists(self, test_edisgo_obj): + # Regression test: task_optimize derives flexible_loads as + # edisgo.dsm.p_min.columns.values (a numpy array), unlike the other + # three flexible_* lists which are built with .tolist(). "x or []" + # raises ValueError ("truth value of an array... is ambiguous") for + # any such array with more than one element - a real crash hit on + # the first end-to-end pipeline run using aggregation_mode=False. + full_grid = copy.deepcopy(test_edisgo_obj) + reduced_grid, _, _ = full_grid.spatial_complexity_reduction( + copy_edisgo=True, + mode="kmeansdijkstra", + cluster_area="feeder", + reduction_factor=0.1, + aggregation_mode=False, + ) + ti = full_grid.timeseries.timeindex + load_name = reduced_grid.topology.loads_df.index[0] + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[load_name]) + reduced_grid.timeseries._loads_active_power.loc[ti, load_name] = [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, + reduced_grid=reduced_grid, + flexible_loads=np.array([load_name]), + ) + + assert result.timeseries.loads_active_power.loc[ti, load_name].tolist() == [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + + def test_disaggregation_multi_member_sums_to_representative(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=2) + members = reduced_grid.topology.loads_df.at[rep, "old_name"] + + p_max = pd.DataFrame(0.0, index=ti, columns=members) + for i, member in enumerate(members): + p_max[member] = [0.1 * (i + 1), 0.0, 0.2 * (i + 1), 0.05 * (i + 1)] + full_grid.dsm.p_max = p_max + + rep_power = pd.Series([1.0, 2.0, 0.0, 3.0], index=ti) + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = rep_power.values + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + total = result.timeseries.loads_active_power.loc[ti, members].sum(axis=1) + assert np.allclose(total.values, rep_power.values) + # Member with zero envelope at t0/t2/t3 gets none of the dispatch; + # both members zero at t1 falls back to an equal split. + assert result.timeseries.loads_active_power.at[ + ti[1], members[0] + ] == pytest.approx(rep_power.iloc[1] / 2) + + def test_disaggregation_singleton_renamed_representative(self, full_and_reduced): + # Regression test: under aggregation_mode=True, spatial_complexity_ + # reduction renames every group's representative, including + # singleton groups (a bus with exactly one flexible load of a given + # type/sector) - so the representative's name can differ from its + # one old_name member's name. A by-name write-back using the + # representative's name would silently miss the real target column + # on full_grid (which only has the original, un-renamed name) and + # create a phantom column instead - this must not happen. + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=1) + member = reduced_grid.topology.loads_df.at[rep, "old_name"][0] + assert rep != member, "fixture assumption: representative was renamed" + + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[member]) + rep_power = pd.Series([7.0, 8.0, 9.0, 10.0], index=ti) + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = rep_power.values + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + assert result.timeseries.loads_active_power.loc[ti, member].tolist() == ( + rep_power.tolist() + ) + assert rep not in result.timeseries.loads_active_power.columns + + def test_raises_clear_error_on_time_index_mismatch(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=1) + member = reduced_grid.topology.loads_df.at[rep, "old_name"][0] + + # dsm.p_max missing the last time step of full_grid's active index. + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti[:-1], columns=[member]) + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + + with pytest.raises(ValueError, match="does not cover"): + spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + def test_storage_units_never_aggregated_always_by_name(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + storage_name = full_grid.topology.storage_units_df.index[0] + assert storage_name in reduced_grid.topology.storage_units_df.index + assert "old_name" not in reduced_grid.topology.storage_units_df.columns + + reduced_grid.timeseries._storage_units_active_power.loc[ti, storage_name] = [ + 1.0, + 1.0, + 1.0, + 1.0, + ] + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, + reduced_grid=reduced_grid, + flexible_storage_units=[storage_name], + ) + assert result.timeseries.storage_units_active_power.loc[ + ti, storage_name + ].tolist() == [1.0, 1.0, 1.0, 1.0] + + def test_reactive_power_recomputed_after_restore(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=1) + member = reduced_grid.topology.loads_df.at[rep, "old_name"][0] + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[member]) + + reactive_before = full_grid.timeseries.loads_reactive_power.loc[ + ti, member + ].copy() + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = [ + 50.0, + 50.0, + 50.0, + 50.0, + ] + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + reactive_after = result.timeseries.loads_reactive_power.loc[ti, member] + assert not reactive_after.equals(reactive_before) From 69c9f3073e4a75efb7d3797c6f84a099fb7c115f Mon Sep 17 00:00:00 2001 From: "Moritz.Schloesser" Date: Thu, 16 Jul 2026 14:31:29 +0000 Subject: [PATCH 52/66] Remove local-only notes from tracking CONTEXT.md and various docs_notes/ writeups were meant to stay local but got included in the spatial complexity reduction merge. Untrack them (kept on disk) and gitignore them going forward. --- .gitignore | 7 + CONTEXT.md | 92 ----- ...n_mode_flexibility_bands_not_aggregated.md | 83 ----- ...mplexity_reduction_pipeline_integration.md | 140 -------- ...ue_temporal_reduction_flexibility_bands.md | 81 ----- .../spatial_reduction_grilling_session.md | 332 ------------------ 6 files changed, 7 insertions(+), 728 deletions(-) delete mode 100644 CONTEXT.md delete mode 100644 docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md delete mode 100644 docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md delete mode 100644 docs_notes/issue_temporal_reduction_flexibility_bands.md delete mode 100644 docs_notes/spatial_reduction_grilling_session.md diff --git a/.gitignore b/.gitignore index 9c649eb27..837d63484 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,10 @@ eDisGo.egg-info/ .vscode/settings.json *OEP_TOKEN.* + +# local-only notes, not for sharing on this branch +/CONTEXT.md +/docs_notes/spatial_reduction_grilling_session.md +/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md +/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md +/docs_notes/issue_temporal_reduction_flexibility_bands.md diff --git a/CONTEXT.md b/CONTEXT.md deleted file mode 100644 index 7e923e900..000000000 --- a/CONTEXT.md +++ /dev/null @@ -1,92 +0,0 @@ -# Context / Glossary - -Domain vocabulary for the eDisGo run pipeline. Glossary only — no implementation -details, no decisions (those live in `docs/adr/`). - -## Complexity reduction - -- **Spatial complexity reduction** — merging nearby buses into a smaller set of - representative buses (clustering *along the grid*, keeping it radial) to shrink the - grid for faster power flow / optimization. Implemented by - `EDisGo.spatial_complexity_reduction`, which builds a *busmap* + *linemap* and mutates - the Topology. Used only to accelerate the optimization; reinforcement runs on the - **full grid**. -- **Temporal complexity reduction** — keeping only grid-critical time steps/intervals - instead of the full year. See the `select_timesteps` task. -- **Busmap** — DataFrame mapping each original bus to its clustered *new_bus* (with new - coordinates). Index = original bus names. -- **Linemap** — DataFrame mapping original line names to *new_line_name* after lines are - recalculated/merged. -- **Reduced grid** — the spatially-reduced EDisGo object the OPF runs on. -- **Full grid** — the original, unreduced EDisGo object; reinforcement always runs here. -- **Map-back / restore** — writing the OPF flexibility dispatch from the reduced grid - onto the full grid. Only the components the OPF *rewrites* are mapped back — flexible - charging points, heat pumps, DSM loads, and storage. Inflexible loads/generators are - **skipped** (the OPF does not change their series; the full grid already holds them - correctly). Implemented by core function - `tools/spatial_complexity_reduction.py::apply_reduced_results_to_full_grid(full_grid, - reduced_grid, *, flexible_cps=None, flexible_hps=None, flexible_loads=None, - flexible_storage_units=None)` + thin wrapper `EDisGo.map_reduced_results_to_full_grid`. - Provenance for disaggregation comes solely from `old_name` on the reduced grid's - `loads_df`/`generators_df` — no busmap/linemap stash needed on `ctx`. - - `aggregation_mode=False`: components keep their names → write back **by component - name**. - - `aggregation_mode=True`: loads/generators may be merged into a representative - (originals recorded in `old_name`; storage is never aggregated). The representative's - optimized series is **disaggregated** onto its `old_name` members. -- **Disaggregation rule** — split a merged representative's optimized series onto its - original members **per time step**, weighted by each member's own *pre-OPF flexibility - envelope* (a known input, never the optimized result): `upper_power(t)` band for - charging points, heat-demand/thermal envelope for heat pumps, `p_max(t)` band for DSM. - Per-step weighting means a charging point only receives power at steps where it has a - connected vehicle (`upper_power(t) > 0`). Sums back to the representative exactly at - every step; equal split as the zero-envelope fallback. -- **Reactive power on restore** — `spatial_restore` writes active power only, then calls - `EDisGo.set_time_series_reactive_power_control()` itself (plain fixed-cosphi default), - mirroring exactly how `pm_optimize`'s own results-writer - (`io/powermodels_io.py::from_powermodels`) handles it: write P, then blanket-recompute Q. - No bespoke reactive-power logic — proportional-split and recompute are mathematically - identical under fixed-cosphi since all `old_name` members of one representative share - the same `power_factor`. - -## Pipeline tasks (spatial reduction) - -- **spatial_reduce** — task run *before* `optimize`: deepcopy the full grid, stash it, - and spatially reduce the working object so `optimize` runs on the reduced grid. -- **spatial_restore** — task run *after* `optimize`: write the optimized dispatch time - series back onto the stashed full grid and make it active again. **Optional** — a run - may legitimately stop after the reduced-grid optimization when only the derived time - series matter. -- **Full-grid stash** — the deepcopied full grid kept in memory on `ctx` between - `spatial_reduce` and `spatial_restore`. Held in-memory (matching legacy eGo, which kept - both full and reduced grids resident during optimize). Persisting it to a disk artifact - to lower peak memory is a possible later optimization, not the initial design. - -## Config surface (spatial reduction) - -- Top-level YAML block `spatial_reduction:` (mirrors `timeseries_selection:`), holding - `mode`, `cluster_area`, `reduction_factor`, `reduction_factor_not_focused`, - `aggregation_mode`, and aggregation sub-modes. Read inside the `spatial_reduce` task via - `ctx.raw_config.get("spatial_reduction", {})`. -- eGo injects it exactly like `timeseries_selection`: a global `spatial_reduction` default - plus a `spatial_reduction_per_grid` dict keyed by `str(mv_grid_id)`, whole-block - replacement (not a field-level merge). If neither is set the key is omitted entirely and - the eDisGo preset's own default applies. - -## Ordering (spatial reduction) - -Pipeline order: `select_timesteps` → `spatial_reduce` → `optimize` → `spatial_restore` -→ `reinforce`. - -- Spatial reduction touches only **topology**; temporal reduction touches only the - **time index** — orthogonal operations that commute as *mechanisms*. Spatial reduction - works on any time index, including full-year (clustering depends on coordinates/graph - distance, not the time series; the `reduction_factor_not_focused` worst-case power flow - runs on an internal deepcopy and does not disturb the working series). -- But the **pipeline** pins `select_timesteps` before `spatial_reduce`: the full-grid - stash inherits whatever time index exists at deepcopy time, and reinforce runs on that - stash. Reducing timesteps first means the stash (and therefore reinforce) carries the - reduced index — full **topology**, reduced **time index**. Reversing the two would - force separately re-reducing the stash's index. -- `spatial_restore` does **no time-index surgery** — it only writes flexible-component - dispatch back onto the stashed full grid. diff --git a/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md b/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md deleted file mode 100644 index 37be52abe..000000000 --- a/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md +++ /dev/null @@ -1,83 +0,0 @@ -# `spatial_complexity_reduction(aggregation_mode=True)` doesn't aggregate `electromobility.flexibility_bands`, breaking `optimize` on merged charging points - -**Type:** bug -**Found:** 2026-07-15, running the real `uc6_spatial_reduction.yaml` pipeline -on grid 32377 with `spatial_reduction.aggregation_mode: true`. -**Affects:** `spatial_complexity_reduction`/`apply_busmap` -(`edisgo/tools/spatial_complexity_reduction.py`), specifically the load -aggregation step; consumed by `_build_electromobility` -(`edisgo/io/powermodels_io.py`). - -## Problem - -When `aggregation_mode=True`, `apply_busmap` merges loads at the same bus -(`aggregate_loads_df`, `spatial_complexity_reduction.py:1587-1599`) and -correctly aggregates their **time series** via `aggregate_timeseries` -(`spatial_complexity_reduction.py:1718`, called for `loads_active_power` / -`loads_reactive_power`). This works fine for DSM loads and heat pumps, whose -OPF constraints are read directly from `loads_df`/`dsm.p_max`/ -`heat_pump.heat_demand_df` keyed by whatever name is currently in `loads_df`. - -Charging points are different: the OPF's own constraint builder, -`_build_electromobility` (`edisgo/io/powermodels_io.py:1212`), does **not** -read its upper-power bound from `loads_df` — it reads -`electromobility.flexibility_bands["upper_power"][cp_name]`, a separate -DataFrame keyed by charging-point name. `spatial_complexity_reduction` never -touches `flexibility_bands` during aggregation, so it still only has entries -for the *original*, pre-merge charging-point names. - -When a bus has 2+ charging points and gets merged into one representative -load (e.g. `Load_Bus_mvgd_32377_F2_B2_charging_point_hpc`), that -representative name has: -- a correctly-aggregated `loads_active_power` entry (summed from the - originals), but -- **no** entry at all in `flexibility_bands["upper_power"]`. - -`task_optimize` derives `flexible_cps` from `loads_df` (filtering -`type == "charging_point"`), so it passes the representative's name into -`pm_optimize` → `to_powermodels` → `_build_electromobility`, which then does -`flex_bands_df["upper_power"][emob_df.index[cp_i]]` and raises `KeyError` -for the representative name. - -## Reproduction - -Run `uc6_spatial_reduction.yaml` (or any preset with the spatial bracket) on -a grid with 2+ charging points sharing a bus, with -`spatial_reduction: {enabled: true, aggregation_mode: true, -load_aggregation_mode: bus}` and `optimize: {flexible: [..., -charging_points, ...]}`. Crashes inside `optimize`, before `spatial_restore` -ever runs: - -``` -KeyError: 'Load_Bus_mvgd_32377_F2_B2_charging_point_hpc' - .../edisgo/io/powermodels_io.py:1212, in _build_electromobility - * flex_bands_df["upper_power"][emob_df.index[cp_i]].iloc[0] -``` - -## Scope note - -This is independent of the spatial-reduction pipeline-integration work -(`spatial_reduce`/`spatial_restore`/`apply_reduced_results_to_full_grid`) — -it's a gap in `spatial_complexity_reduction` itself (the reduction side), -already reachable via `EDisGo.spatial_complexity_reduction` + -`EDisGo.pm_optimize` directly, with no pipeline involved. It only manifests -under `aggregation_mode=True` with charging points present; `aggregation_ -mode=False` is unaffected since every component keeps its original name -there, and `flexibility_bands` stays valid. - -Also worth checking as part of the same fix: whether heat pumps have an -analogous issue if the OPF ever reads a heat-pump-specific band keyed -differently from `loads_df` (current understanding, from the spatial- -reduction design session, is that heat pumps use `heat_demand_df`/`cop_df`/ -`loads_df.p_set` directly, which DO get aggregated correctly — but worth -double-checking with a merged multi-heat-pump bus once this issue is picked -up, in case some other heat-pump-specific structure has the same gap). - -## Suggested fix - -Extend `apply_busmap`'s load-aggregation step to also aggregate -`electromobility.flexibility_bands` (`upper_power`, `lower_energy`, -`upper_energy`) onto the same representative name, using the same -`old_name`-based grouping/summing already used for `aggregate_timeseries` -— summing `upper_power` per merged group is the natural analogue of summing -`p_set`/active power for the representative. diff --git a/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md b/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md deleted file mode 100644 index be650dadd..000000000 --- a/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md +++ /dev/null @@ -1,140 +0,0 @@ -# Integrate spatial complexity reduction into the run pipeline - -## Goal - -Add **spatial complexity reduction** to the eDisGo run pipeline as a -counterpart to the existing temporal complexity reduction (timestep -selection, `select_timesteps`). Spatial reduction merges nearby buses into -representative buses to shrink the grid so the **optimization (OPF)** runs -faster. **Reinforcement must run on the FULL grid** (full topology) — but on -the *reduced* time index (temporal reduction still applies). - -## What's done - -- **Two bracketing pipeline tasks**, `spatial_reduce` (before `optimize`) - and `spatial_restore` (after, optional): `spatial_reduce` deepcopies and - stashes the full grid on the run context, then spatially reduces the - working object so `optimize` runs on a smaller grid; `spatial_restore` - writes the optimized flexible-component dispatch back onto the stashed - full grid and makes it active again for `reinforce`. Both are no-ops when - `spatial_reduction.enabled` is false, so a pipeline carrying the bracket - behaves identically to one without it when disabled. -- **New core function** `apply_reduced_results_to_full_grid(full_grid, - reduced_grid, *, flexible_cps, flexible_hps, flexible_loads, - flexible_storage_units)` in `edisgo/tools/spatial_complexity_reduction.py`, - plus a thin `EDisGo.map_reduced_results_to_full_grid` wrapper — mirroring - how `spatial_complexity_reduction`/`EDisGo.spatial_complexity_reduction` - already pair up for the reduction half. -- **Map-back only touches components the OPF rewrites**: flexible charging - points, heat pumps, DSM loads, and storage units. Inflexible - loads/generators are skipped since the OPF never changes their series. - With `aggregation_mode=False` (see "What's still open" below), every - component keeps its own name, so restore is a plain by-name write-back. -- **Reactive power on restore**: `spatial_restore` writes active power only, - then calls `EDisGo.set_time_series_reactive_power_control()` itself — - mirroring exactly how the OPF's own results-writer - (`io/powermodels_io.py::from_powermodels`) handles it (write P, then - blanket-recompute Q). No new reactive-power logic anywhere. -- **Validator integration**: extended the existing `requires`/`provides` - capability system rather than adding new validator machinery. - `spatial_reduce` provides `reduced_grid`; `optimize` additionally provides - `optimized_dispatch`; `spatial_restore` requires both — so a misordered - pipeline (e.g. `spatial_restore` before `optimize`) fails static - validation instead of crashing mid-run. -- **Config surface**: top-level `spatial_reduction:` YAML block (`enabled`, - `mode`, `cluster_area`, `reduction_factor`, `reduction_factor_not_focused`, - `aggregation_mode`), mirroring `timeseries_selection:` exactly, read via - `ctx.raw_config.get("spatial_reduction", {})`. -- **New preset** `edisgo/run/presets/uc6_spatial_reduction.yaml` wiring the - bracket into a real pipeline (`select_timesteps → reactive_power → - spatial_reduce → optimize → spatial_restore → reinforce → save`), disabled - by default via `spatial_reduction.enabled: false`. -- **eGo integration**: `EDisGoNetworks._build_run_edisgo_config` injects - `spatial_reduction`/`spatial_reduction_per_grid` exactly like - `timeseries_selection`/`timeseries_selection_per_grid` (global default + - per-grid override keyed by MV grid id, whole-block replacement). New - `scenario_setting_uc6_example.json`, plus unit tests for the injection - logic. -- **Tests**: `tests/tools/test_spatial_complexity_reduction.py` - (`TestApplyReducedResultsToFullGrid`) covers by-name write-back, the - numpy-array-input regression, the time-index-mismatch guard, and the - reactive-power recompute, with stub OPF results (no real Julia/Gurobi - dependency in CI). -- **Verified end-to-end** against a real grid (32377) through both eDisGo - directly and through eGo: `spatial_reduce → optimize (Gurobi) → - spatial_restore → reinforce → save` all completed successfully, and a - dedicated notebook (`analyse_spatial_reduction.ipynb`) confirms the - reduced grid's OPF output exactly matches the full grid's post-restore - value for every flexible component (0 mismatches across 388 components on - the test run). -- **Two bugs found and fixed during implementation** (unrelated to the - design, surfaced by actually running the code): a `flexible_* or []` - crash on numpy-array input (`task_optimize` derives `flexible_loads` as an - array, not a list); and `electromobility.flexibility_bands` not being - trimmed to the active time index after `build_flexibility_bands`, causing - a `KeyError` deep inside the OPF's charging-point constraint builder. - -## What's still open - -### `aggregation_mode=True` is not usable with charging points present - -`spatial_complexity_reduction` (`aggregation_mode=True`) merges loads at the -same bus and correctly aggregates their **time series** -(`loads_active_power`/`loads_reactive_power`). This works for DSM loads and -heat pumps, whose OPF constraints are read directly from -`loads_df`/`dsm.p_max`/`heat_pump.heat_demand_df`. - -Charging points are different: the OPF's constraint builder -(`_build_electromobility` in `edisgo/io/powermodels_io.py`) reads its -upper-power bound from `electromobility.flexibility_bands["upper_power"]`, a -separate DataFrame keyed by charging-point name — and -`spatial_complexity_reduction` never aggregates `flexibility_bands` during -load merging. So a merged representative has a correctly-summed -`loads_active_power` entry but **no** entry in `flexibility_bands`, and -`optimize` crashes with a `KeyError` for the representative's name before -`spatial_restore` ever runs. - -This is a gap in the reduction side (`spatial_complexity_reduction`/ -`apply_busmap`), not in `spatial_restore`/`apply_reduced_results_to_full_grid` -— reachable via `EDisGo.spatial_complexity_reduction` + `EDisGo.pm_optimize` -directly, no pipeline involved. It only manifests under -`aggregation_mode=True` with charging points present at a bus with 2+ of -them; `aggregation_mode=False` is unaffected. Full writeup with the exact -traceback and a suggested fix (aggregate `flexibility_bands` in -`apply_busmap`'s load-merging step, the same way `loads_active_power` -already is) is in -`docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md`. - -**Until this is fixed, `aggregation_mode` should stay `False`** — that is -the default in the new preset and the only mode covered by the disaggregation -tests and the end-to-end verification above. - -### Related, deferred design gap (not blocking, tracked separately) - -`build_flexibility_bands` builds bands over whatever date range the -underlying SimBEV data spans and only resamples to the target *frequency*, -not the target *date range* — the fix applied here trims them after the -fact (`reduce_timeseries_data_to_given_timeindex`). Building them -pre-scoped to the selected window in the first place would be cleaner and -more efficient; see -`docs_notes/issue_temporal_reduction_flexibility_bands.md`. - -### Not yet done - -- Reactive-power write-back has no dedicated integration test against a - real OPF run (covered by unit test with stub dispatch only). -- No pipeline/task-level integration test for `spatial_reduce`/ - `spatial_restore` (current test scope is core-function-only, per an - explicit scoping decision — see `docs_notes/spatial_reduction_grilling_session.md`). -- Two ADR candidates flagged during design, not yet written: "spatial - reduction as two bracketing pipeline tasks, restore optional" (breaks the - established "logic lives inside `pm_optimize`" convention, trading - consistency for the ability to stop early after the reduced-grid OPF), and - "disaggregation by pre-OPF flexibility envelope" (a modeling choice with - real alternatives). - -## References - -- Full design session record: `docs_notes/spatial_reduction_grilling_session.md` -- `aggregation_mode=True` + charging points gap: `docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md` -- Flexibility-bands time-index gap (fixed): `docs_notes/issue_temporal_reduction_flexibility_bands.md` diff --git a/docs_notes/issue_temporal_reduction_flexibility_bands.md b/docs_notes/issue_temporal_reduction_flexibility_bands.md deleted file mode 100644 index 5aac8caca..000000000 --- a/docs_notes/issue_temporal_reduction_flexibility_bands.md +++ /dev/null @@ -1,81 +0,0 @@ -# Flexibility bands should be built scoped to the selected time index, not built-then-trimmed - -**Type:** design gap / bug -**Found:** 2026-07-15, while testing spatial complexity reduction against a -manual-mode `select_timesteps` run (`uc6_spatial_reduction.yaml`). -**Affects:** `task_build_flexibility_bands` (`edisgo/run/tasks/flex.py`), -`Electromobility.get_flexibility_bands` (`edisgo/network/electromobility.py`), -and by extension any other per-component band/envelope built after -`select_timesteps` (heat pump `heat_demand_df`/`cop_df`, DSM `p_max`/`p_min`). - -## Problem - -`select_timesteps` (manual mode, `position: pre_import`) fixes -`edisgo.timeseries.timeindex` to an arbitrary, possibly short window (e.g. 24 -hours starting `2035-01-15`) early in the pipeline, before electromobility -data is even imported. - -`build_flexibility_bands` runs later and calls -`Electromobility.get_flexibility_bands(edisgo, use_case=...)`. Per that -method's own docstring, `resample=True` only "resamples the bands to the same -**frequency** as time series data in the `TimeSeries` object" — it does -*not* clip the bands to the same *date range*. The bands are built from the -raw SimBEV charging-process data and keep whatever date range that data -spans, matching the target row spacing (e.g. hourly) but not the target -window. - -Nothing downstream re-trims `electromobility.flexibility_bands` to the -selected 24-hour window afterward. The mismatch went unnoticed until a piece -of code tried to actually index `flexibility_bands` using -`edisgo.timeseries.timeindex` and hit a `KeyError`-shaped failure (missing -time steps) — in this case, the new -`apply_reduced_results_to_full_grid`/`spatial_restore` disaggregation logic, -which reads `flexibility_bands["upper_power"]` as a per-charging-point, -per-time-step weighting envelope. - -This is a **pre-existing gap**, not something introduced by spatial -reduction — it already exists in `uc5_select_timesteps.yaml` (the preset -`uc6_spatial_reduction.yaml`/`uc5_spatial_reduction.yaml` were both derived -from). It simply had no consumer that indexed `flexibility_bands` by the -active time index before now. - -## Immediate fix applied (unblocks spatial-reduction testing) - -`task_build_flexibility_bands` now calls -`reduce_timeseries_data_to_given_timeindex(edisgo, edisgo.timeseries.timeindex, -electromobility=True, timeseries=False, heat_pump=False, dsm=False, -overlying_grid=False)` right after `get_flexibility_bands`, trimming -`flexibility_bands` down to the active index. See -`edisgo/run/tasks/flex.py::task_build_flexibility_bands`. - -This is a workaround (build full, then trim), not the better design below. - -## Suggested proper fix (not yet implemented — this issue) - -Build flexibility bands (and, likely, heat-pump/DSM bands) scoped to the -already-selected time index from the start, rather than building over the -full/native data range and trimming afterward: - -- `Electromobility.get_flexibility_bands` (or its caller) should accept/use - the target time index *before* running the difference-array band - construction, so the SimBEV charging-process data outside that window is - never even considered. -- Audit whether `HeatPump`/`DSM` band construction (wherever their - time-varying bounds are first populated — likely in the `import_heat_pumps` - / `import_dsm` tasks or their underlying `edisgo/io/*` importers) has the - same built-on-full-range-then-maybe-trimmed pattern, since - `reduce_timeseries_data_to_given_timeindex` already has `heat_pump=True`/ - `dsm=True` flags suggesting this was anticipated but may not be - consistently invoked at the right point in every pipeline path. -- Consider whether this should be a single, explicit "finalize time index" - pipeline hook that every band-producing task can rely on having already - run, rather than each task needing to remember to trim itself. - -## Reproduction - -Run `uc6_spatial_reduction.yaml` (or `uc5_select_timesteps.yaml`) with -`timeseries_selection: {mode: manual, start: "2035-01-15 00:00", periods: -24, freq: h}` and `overlying_grid.enabled: true`, then inspect -`edisgo.electromobility.flexibility_bands["upper_power"].index` after -`build_flexibility_bands` runs — before the fix above, its date range does -not match `edisgo.timeseries.timeindex`. diff --git a/docs_notes/spatial_reduction_grilling_session.md b/docs_notes/spatial_reduction_grilling_session.md deleted file mode 100644 index b90ddb190..000000000 --- a/docs_notes/spatial_reduction_grilling_session.md +++ /dev/null @@ -1,332 +0,0 @@ -# Spatial complexity reduction — pipeline integration design (grilling session) - -**Status:** DESIGN CLOSED. All questions resolved 2026-07-14. Next: implement, and decide -the two ADR candidates below. -**Date:** 2026-07-09 (started), closed 2026-07-14. -**Repo/branch:** `/storage/MS/ego/eDisGo`, branch `edisgo_run_edisgo`. -**Companion files:** `CONTEXT.md` (glossary, at repo root), this file. - ---- - -## Goal - -Integrate eDisGo's **spatial complexity reduction** into the run pipeline, as a -counterpart to the temporal complexity reduction (timestep selection) already added. - -Spatial reduction merges nearby buses into representative buses to shrink the grid so -the **optimization (OPF)** runs faster. **Reinforcement must run on the FULL grid** -(full topology) — but on the *reduced* time index (temporal reduction still applies). - ---- - -## How spatial reduction works (established from code) - -- `EDisGo.spatial_complexity_reduction()` (edisgo.py:3439) wraps - `tools/spatial_complexity_reduction.py::spatial_complexity_reduction()` (line 1830). -- It builds a **busmap** (original bus → clustered `new_bus`) + **linemap**, then mutates - the Topology in place (or on a copy if `copy_edisgo=True`). Returns `(edisgo, busmap_df, - linemap_df)`. -- Clustering uses coordinates / grid-graph distance — **independent of the time index**. - Works on a full-year grid too. `apply_pseudo_coordinates=True` (default) fills missing - coords. -- **Storage is never aggregated** (only bus-relabeled) — keeps its name/identity even - with `aggregation_mode=True` (spatial_complexity_reduction.py:1756–1760). -- **Loads/generators ARE merged** when `aggregation_mode=True` (lines 1699–1754), grouped - by bus(+type+sector). Originals recorded in an **`old_name`** column on the reduced - component rows; their series summed via `aggregate_timeseries`. -- `reduction_factor_not_focused` uses `find_buses_of_interest` which runs a worst-case - power flow — but on an **internal deepcopy** (line 93), so it does NOT disturb the - working object's time series. - -### Legacy eGo reference (the pattern we are re-implementing cleanly) -`eGo/ego/tools/edisgo_integration.py::_run_edisgo_task_optimisation` (~line 1632): -- `edisgo_copy = deepcopy(edisgo_grid)` (full grid stays in `edisgo_grid`) -- temporal-reduce copy → spatial-reduce copy → `pm_optimize` on copy -- write dispatch back onto the full grid **by component name** (lines 1763–1795): - loads/generators/storage active+reactive power, sliced by `time_steps` -- `edisgo_grid.timeseries.timeindex = timeindex` (union of optimized intervals) -- reinforce runs on the full grid. Both grids were resident in memory simultaneously. - ---- - -## Decisions made (confirmed with user) - -1. **Two bracketing tasks (Option A), NOT inside `pm_optimize`.** - - `spatial_reduce` (before `optimize`) and `spatial_restore` (after `optimize`). - - Rationale: reduce/restore are topology operations, not OPF concerns; must be - visible/toggleable in YAML; enables stopping after the reduced-grid OPF when only - the derived time series matter. - - This deliberately breaks the "all logic inside pm_optimize" principle we used for - the multi-interval split — justified by the stop-early capability. **ADR candidate.** - -2. **`spatial_restore` is OPTIONAL** — a run may stop after optimizing on the reduced - grid. (But when present it must follow optimize; see validator note.) - -3. **Full-grid stash kept IN MEMORY on `ctx`** (matches legacy, which held both grids - resident). Disk-artifact persistence to cut peak memory is a deferred optimization, - not v1. (Runner does support disk reload via `save` + `stage_artifacts` + `load_from`, - but we are not using it here.) - -4. **Aggregation support:** implement `aggregation_mode=False` FIRST, then design so - `aggregation_mode=True` follows. - -5. **Map-back only touches components the OPF rewrites:** flexible charging points, heat - pumps, DSM loads, and storage. **Inflexible loads/generators are SKIPPED** — the OPF - doesn't change them; the full grid already holds their correct series. - - `aggregation_mode=False`: write back **by component name**. - - `aggregation_mode=True`: disaggregate the representative's series onto its `old_name` - members. - -6. **Disaggregation rule (aggregation_mode=True):** split the representative's optimized - series onto original members **per time step**, weighted by each member's own **pre-OPF - flexibility envelope** (a known input, never the optimized result): - - charging points → `electromobility.flexibility_bands["upper_power"][cp_name]` (a CP - with no connected vehicle has `upper_power(t)=0`, so it receives no charge that step - — physically correct); - - heat pumps → `weight(t) = min(heat_demand_df[hp_name][t] / cop_df[hp_name][t], - loads_df.p_set[hp_name])`. **Refined during implementation (2026-07-14):** the - original "heat-demand/thermal envelope" phrasing was underspecified. Investigated - `edisgo/io/powermodels_io.py::_build_heatpump` (~lines 1259-1282): the OPF's actual - per-unit electrical cap is the CONSTANT rated power `loads_df.p_set`, not a - time-varying series — the genuinely time-varying pre-OPF quantity is - `heat_demand_df` (thermal, MW) divided by `cop_df` (electrical-equivalent demand). - Capping that ratio at each member's own `p_set` mirrors charging points exactly (a - CP's `upper_power(t)` is already a capped bound, not raw uncapped vehicle demand) and - ensures no member is ever assigned a share exceeding what it could physically draw; - - DSM → `dsm.p_max[load_name][t]` band (`edisgo/network/dsm.py:63`). - - All three sources share the same shape: rows = timestamps matching - `TimeSeries.timeindex`, columns = component names matching `Topology.loads_df.index`. - - Sums back to the representative exactly at each step; equal split as zero-envelope - fallback. (User explicitly preferred a time-series-based split over a static scalar, - because these envelopes are known pre-OPF and reflect actual flexibility-relevant - events, e.g. a connected vehicle or nonzero heat demand.) - -7. **Ordering:** `select_timesteps` → `spatial_reduce` → `optimize` → `spatial_restore` - → `reinforce`. - - The two reduction *mechanisms* commute (orthogonal: topology vs time index), BUT the - pipeline pins `select_timesteps` before `spatial_reduce` so the stashed full grid - (hence reinforce) inherits the **reduced** time index. Result: reinforce on full - **topology** × reduced **time index**. - - `spatial_restore` does **no time-index surgery** — only writes flexible dispatch back. - -8. **Tasks stay thin; computation lives in eDisGo core** (same principle as the - timestep-selection refactor). See open question for how this applies to restore. - ---- - -## Where we paused — OPEN QUESTION (resume here) - -Applying "tasks are thin wrappers" to the restore half. Established: -- **Reduction half is already correct:** `spatial_complexity_reduction()` is already a - self-contained core function + EDisGo method. `spatial_reduce` task just needs to - deepcopy+stash the full grid and call it. Nothing to extract. -- **Restore half is the gap:** there is **NO** existing core function that maps reduced - OPF results back onto a full grid (the legacy logic lived inline in eGo's private - method; `_restore_pristine_inputs` in powermodels_opf.py:225 is unrelated — it's the - multi-interval snapshot/restore). - -**Proposal put to the user (awaiting confirmation):** -- (a) Create a NEW core function, e.g. - `tools/spatial_complexity_reduction.py::apply_reduced_results_to_full_grid(full_grid, - reduced_grid, *, flexible_cps, flexible_hps, flexible_loads, flexible_storage_units)` - + a thin `EDisGo` method wrapper (mirroring `spatial_complexity_reduction`). The task - `spatial_restore` just reads the stashed full grid + flexible sets from `ctx` and calls - it. Used **outside** the pipeline, a caller passes `full_grid`, `reduced_grid`, and the - flexible sets directly (no `ctx`). This gives symmetry: both halves = core fn + method - wrapper + thin task; both usable standalone; disaggregation rule lives/tested in core. -- (b) **`old_name`** carried on the reduced grid's `loads_df`/`generators_df` is - sufficient provenance for disaggregation — the reduced grid self-describes its origins, - so **no busmap needs stashing**. Full grid needed as the write target (holds individual - members + their pre-OPF weighting envelopes); reduced grid supplies optimized series + - `old_name`. Both grids are required args. - -**User's last message (the prompt to answer):** agrees restore logic should NOT live in -the task and should become its own eDisGo function; flexible-component names stored in -`ctx` and passed to the function, or passed differently when used outside the pipeline; -reduction is already an independent eDisGo function. - -→ So (a) and (b) are essentially aligned with the user's view; next step is to CONFIRM the -signature details (both grids as args; `old_name` sufficient, no busmap) and then move on. - -**RESOLVED (2026-07-14):** -- Core function signature: - `apply_reduced_results_to_full_grid(full_grid, reduced_grid, *, flexible_cps=None, - flexible_hps=None, flexible_loads=None, flexible_storage_units=None)` — four separate - kwargs, one per flexible-component type, each defaulting to `None`/skip. -- `EDisGo` method wrapper name: `map_reduced_results_to_full_grid` (full symmetry with the - core function name, no abbreviation). -- `old_name` on the reduced grid's `loads_df`/`generators_df` is CONFIRMED sufficient - provenance for disaggregation. No busmap/linemap stash on `ctx`. - ---- - -## Remaining questions still to grill (not yet discussed) - -- ~~Validator ordering~~ — **RESOLVED (2026-07-14).** Investigated - `edisgo/run/validator.py` (`validate()`, lines 47-139) + `edisgo/run/registry.py` - (`TaskMeta`, `register_task()`): ordering today is capability-based (`requires`/ - `provides` sets accumulated linearly across the pipeline, `validator.py:96,118-130`), - NOT a dependency graph and NOT named task-to-task precedence. The one existing hardcoded - exception is `reactive_power` must be last among `ts_altering` tasks - (`validator.py:110-116`). `select_timesteps`'s optional dual-position behavior - (`timeseries.py:328-403`) is NOT validator-enforced — it's a runtime-only check inside - the task, so it was not usable as a precedent. - - **Decision:** extend the existing capability system rather than add a new validator - concept or fall back to runtime-only checking (matches the pipeline's existing - mechanism everywhere else): - - `spatial_reduce` declares `provides={"reduced_grid"}`. - - `optimize` declares `provides={"optimized_dispatch", ...}` (in addition to its - existing provides). - - `spatial_restore` declares `requires={"reduced_grid", "optimized_dispatch"}`. - - This closes the gap where presence-only capability accumulation would otherwise let - `spatial_restore` validate successfully even if placed before `optimize` (both - `reduced_grid`-derived requirements would already be "satisfied" from - `spatial_reduce` alone) — requiring `optimized_dispatch` too means `spatial_restore` - cannot pass validation until `optimize` has actually appeared earlier in the - pipeline. - - **Correction (2026-07-14, during implementation):** `register_task`'s `requires`/ - `provides` (`edisgo/run/registry.py:52-58`) are fixed at decoration time (module - load), NOT evaluated per-run — so "optimize requires `reduced_grid` only when spatial - reduction is configured for THIS run" is not expressible and was dropped. - `optimize`'s `requires` stays exactly `{"timeseries", "flex"}`, unchanged — it does - not need to know spatial reduction exists. Ordering is fully enforced from - `spatial_restore`'s side alone; adding `reduced_grid` to `optimize`'s `requires` - unconditionally would have broken every existing preset that runs `optimize` without - `spatial_reduce` (uc2, uc4, uc5_select_timesteps). -- ~~YAML config surface~~ — **RESOLVED (2026-07-14).** Top-level `spatial_reduction:` - block, mirroring `timeseries_selection:`. Read via - `ctx.raw_config.get("spatial_reduction", {})` inside the `spatial_reduce` task, same - pattern as `select_timesteps` (`timeseries.py:415`). Holds `mode`, `cluster_area`, - `reduction_factor`, `reduction_factor_not_focused`, `aggregation_mode`, aggregation - sub-modes. -- ~~eGo injection~~ — **RESOLVED (2026-07-14).** Verified `timeseries_selection`'s actual - injection in `EDisGoNetworks._build_run_edisgo_config()`, - `eGo/ego/tools/edisgo_integration.py:675-685`: global `timeseries_selection` default + - `timeseries_selection_per_grid` dict keyed by `str(mv_grid_id)` - (`edisgo_integration.py:681-684`), **whole-block replacement** (not field-level merge), - key omitted from `cfg` entirely if both are unset (line 684: `if ts_selection is not - None`), letting the eDisGo preset's own default apply. Granularity is truly per - individual MV grid (`mv_grid_id`, looped in `run_all`, `edisgo_integration.py:597-626`). - - **Decision:** `spatial_reduction` replicates this exactly — global `spatial_reduction` - default + `spatial_reduction_per_grid` dict keyed by `str(mv_grid_id)`, whole-block - replacement, omitted if unset (same as `timeseries_selection`, not the simpler - `overlying_grid` hardcoded-fallback pattern). -- ~~Reactive power~~ — **RESOLVED (2026-07-14).** Investigated existing convention: - `pm_optimize`'s results-writer (`edisgo/io/powermodels_io.py::from_powermodels`, - lines 283-352) writes ONLY active power for flex components (heat pumps, CPs, DSM, - storage) into `_generators_active_power`/`_loads_active_power`/ - `_storage_units_active_power`; reactive power is untouched there. Immediately after - (line 354-355), it calls the plain `edisgo_object.set_time_series_reactive_power_control()` - — same generic fixed-cosphi default (`network/timeseries.py::fixed_cosphi`, - `flex_opt/q_control.py`) used everywhere else in eDisGo, applied blanket over the whole - object, not scoped to flex components. Confirmed the existing `reactive_power` pipeline - task (`edisgo/run/tasks/timeseries.py:584-629`) is just a thin wrapper around the exact - same call — no special-casing for OPF-derived components anywhere in the codebase today. - - Considered alternative: split reactive power proportionally to each `old_name` - member's share of the representative's active power (mirroring the active-power - disaggregation rule) instead of recomputing. **Verified mathematically equivalent** - under fixed-cosphi: all `old_name` members of one representative share the same - `type` → same `power_factor`, so `Q = P · tan(φ)` per member gives an identical result - whether derived by proportional split or by recomputing from each member's - disaggregated P directly. - - **Decision:** `spatial_restore` writes active power for flexible components onto the - full grid, then calls `set_time_series_reactive_power_control()` itself — mirrors - `pm_optimize`'s own convention exactly (write P, then blanket-recompute Q). Reuses the - existing method with no new reactive-power math anywhere, and makes `spatial_restore` - correct standalone even in pipelines with no downstream `reactive_power` task. -- ~~Testing strategy~~ — **RESOLVED (2026-07-14).** Scope for this first pass: core - function only (`apply_reduced_results_to_full_grid`), NOT pipeline/task-level - integration tests (deferred). Cover both aggregation modes: - - `aggregation_mode=False`: by-name write-back correctness. - - `aggregation_mode=True`: disaggregation math — per-step envelope-weighted split, - exact-sum-back-to-representative check, and the equal-split zero-envelope fallback. - - Stub/fake OPF results as fixtures; no real `pm_optimize` call, no real pipeline run - through `ctx`/validator. -- ~~uc5 preset~~ — **RESOLVED (2026-07-14).** New standalone preset - `edisgo/run/presets/uc5_spatial_reduction.yaml` (full copy of - `uc5_select_timesteps.yaml` + the spatial bracket, NOT an `extends` overlay — tasks - don't exist in code yet so this is documentation/example, and a standalone file matches - `uc5_select_timesteps.yaml`'s own self-contained style). Disable switch: explicit - `spatial_reduction.enabled` flag (mirrors `overlying_grid.enabled`, NOT - `timeseries_selection`'s absent-block-is-the-toggle style) — lets params stay in the - YAML while toggling on/off with one flag. Bracket placement: `spatial_reduce` right - before `optimize`, `spatial_restore` right after; `reactive_power` stays where it already - is (pre-OPF full-series cosphi on the reduced index), unaffected by the spatial bracket. - Final order: `select_timesteps(post_grid) → reactive_power → spatial_reduce → optimize → - spatial_restore → reinforce`. - ---- - -## Implementation (2026-07-14) - -Implemented and tested end-to-end (real venv, python3.10, `pip install -e ".[dev]"`, -real ding0 test grid `tests/data/ding0_test_network_1`): - -- `apply_reduced_results_to_full_grid` + - `EDisGo.map_reduced_results_to_full_grid` — - `edisgo/tools/spatial_complexity_reduction.py`, `edisgo/edisgo.py`. -- `spatial_reduce` / `spatial_restore` tasks — new file `edisgo/run/tasks/spatial.py`, - registered in `edisgo/run/tasks/__init__.py`. -- `RunContext.full_grid_stash` — new field, `edisgo/run/context.py`. -- `task_optimize` writes `flexible_cps`/`flexible_hps`/`flexible_loads`/ - `flexible_storage_units` to `ctx.flags`; `@register_task("optimize", ...)` gained - `provides={"optimized_dispatch"}` — `edisgo/run/tasks/analysis.py`. -- New preset `edisgo/run/presets/uc5_spatial_reduction.yaml` (already covered above). -- New tests `tests/tools/test_spatial_complexity_reduction.py::TestApplyReducedResultsToFullGrid` - (6 tests: by-name write-back, multi-member disaggregation sum check, singleton-rename - regression, time-index-mismatch error, storage-unit by-name path, reactive-power - recompute). Full existing suite (`tests/tools/`, `tests/run/`, `tests/opf/`) reverified - green: 99 passed, 1 unrelated skip. - -**Two real bugs found by a dedicated code-review agent (dispatched because no import- -capable env existed initially) and fixed before landing:** - -1. **Singleton-rename data corruption (serious).** Original code took a `_write_by_name` - fast path whenever a flexible-component set had NO multi-member merged group, - assuming an unmerged representative's name always equals its member's name. FALSE - under `aggregation_mode=True`: `spatial_complexity_reduction` renames **every** - group's representative, including singleton groups (a bus with exactly one flexible - load of a type/sector) — confirmed empirically on the real test grid (11 such - singletons exist in `ding0_test_network_1` alone). `_write_by_name` using the - representative's (renamed) name against `full_grid` (which only has the original, - un-renamed name) silently created a phantom column via pandas' `.loc[]` auto-vivify - behavior, leaving the real target column stale — a silent, hard-to-detect data - corruption, not a crash. **Fix:** removed the `_write_by_name` fast path for CPs/HPs/ - DSM loads entirely; always route through `_disaggregate`, which was proven correct for - every case (matching name, mismatched singleton, multi-member) by direct test. - `_write_by_name` now only serves storage units, which are genuinely never renamed. -2. **`flexible_* or []` crashes on numpy-array input (real, hit on first live - end-to-end run).** `task_optimize` derives `flexible_loads` as - `edisgo.dsm.p_min.columns.values` (`analysis.py:357`) — a numpy array — unlike the - other three `flexible_*` lists, which use `.tolist()`. `array or []` raises - `ValueError: The truth value of an array with more than one element is ambiguous...` - for any such array with 2+ elements. This surfaced immediately on the very first real - pipeline run (`run_example_06.py`, `uc6_spatial_reduction.yaml`, `aggregation_mode: - false`) — the OPF/Gurobi solve completed successfully, `spatial_restore` crashed on - the very next line. **Fix:** replaced `flexible_x = flexible_x or []` with - `flexible_x = list(flexible_x) if flexible_x is not None else []` for all four - parameters in `apply_reduced_results_to_full_grid` — `is None` is the correct - emptiness check for an optional list-like argument that may be a list, tuple, or numpy - array (the codebase's own docstrings elsewhere already document these params as - accepting `numpy.ndarray or None`). Added regression test - `test_accepts_numpy_array_flexible_component_lists`. -3. **Unguarded `KeyError` on time-index mismatch (robustness).** If a flexibility-band/ - DSM/heat-pump attribute on `full_grid` doesn't cover `full_grid.timeseries.timeindex` - (e.g. the full-grid stash was taken before time-index selection ran — a real risk for - any pipeline not following the `select_timesteps` → `spatial_reduce` convention, since - nothing in the registry enforces that order), the disaggregation envelope lookup would - raise a bare `KeyError` deep inside a `.loc` call. **Fix:** added - `_require_full_timeindex`, a defensive check before each envelope lookup that raises a - clear `ValueError` naming the mismatch and the likely cause. Decided against also - adding a validator `requires={"timeseries"}` declaration — the pipeline is already - constructed so a time index is guaranteed set before `spatial_reduce` runs (decision 7, - above), so the defensive check alone is sufficient without touching registry metadata. - -## ADR candidates (offer at end of session) - -1. "Spatial reduction as two bracketing pipeline tasks, restore optional" — hard to - reverse, surprising (breaks the pm_optimize-owns-its-logic principle), real trade-off - (consistency vs stop-early). → write when design settles. -2. Possibly: "Disaggregation by pre-OPF flexibility envelope per time step" — a modeling - choice with alternatives (static scalar, proportional-to-original-series). Borderline; - decide at end. From beedffddcba57cd3483b28c6cbb98e7e4ff4da7b Mon Sep 17 00:00:00 2001 From: "Moritz.Schloesser" Date: Thu, 16 Jul 2026 14:50:40 +0000 Subject: [PATCH 53/66] Ignore additional local-only scratch files Notebooks, a scratch runner script, and results/ outputs that live locally but aren't meant to be tracked on this branch. --- .gitignore | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.gitignore b/.gitignore index 837d63484..ed8ff1563 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,17 @@ eDisGo.egg-info/ /docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md /docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md /docs_notes/issue_temporal_reduction_flexibility_bands.md +analyse_spatial_reduction_executed_example.ipynb +analyse_spatial_reduction.ipynb +analyse_uc5_results copy.ipynb +analyse_uc5_results.ipynb +handson_edisgo_starter.ipynb +run_example_04.py +docs_notes/ego_timeseries_selection_plan.md +docs_notes/pr_description_spatial_complexity_reduction.md +docs_notes/timeseries_selection_remaining_tasks.md +results/uc4_example/main.zip +results/uc5_select_timesteps/32377_t_168_residual_load.csv +results/uc5_select_timesteps/main.zip +results/uc5_spatial_reduction/main.zip +results/uc6_spatial_reduction/main.zip From 9e409972fee450f42e0e4bed8f25f7fd5fc6b6df Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Mon, 20 Jul 2026 15:58:45 +0200 Subject: [PATCH 54/66] fix: trim apply_charging_strategy output to the active timeindex (#710) apply_charging_strategy (all three modes: dumb/reduced/residual) wrote a full SimBEV-simulation-length series into loads_active_power/ loads_reactive_power unconditionally, via add_component_time_series's raw concat - a shorter or gapped edisgo.timeseries.timeindex (e.g. after select_timesteps) never trimmed the excess, leaving stale rows in the private _loads_active_power that any direct reader (not just the timeindex-scoped public getter) would see. Trim to edisgo.timeseries.timeindex when SimBEV's stepsize already matches eDisGo's own frequency, the common case. When it doesn't, this function internally resamples edisgo_obj.timeseries to reconcile frequencies and back - that round-trip currently fabricates a contiguous index, silently reopening any gap select_timesteps (auto mode) left in place, so the trim is skipped there rather than risk operating on the wrong window. Documented as a known limitation pending a fix to TimeSeries.resample() itself (tracked separately, not part of this change). Part of the #703 checklist (docs_notes/issue_temporal_reduction_flexibility_bands.md). --- edisgo/edisgo.py | 8 ++++ edisgo/flex_opt/charging_strategies.py | 60 +++++++++++++++++++++++- tests/flex_opt/test_charging_strategy.py | 59 ++++++++++++++++++++++- 3 files changed, 123 insertions(+), 4 deletions(-) diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index e3ca90422..433e26091 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -2190,6 +2190,14 @@ def apply_charging_strategy( match the SimBEV data frequency and after determining the charging demand time series resampled back to the original frequency. + The written charging point time series are trimmed to + :attr:`~.network.timeseries.TimeSeries.timeindex` when no such frequency + mismatch occurs. When it does occur, the resample round-trip above + currently fabricates a contiguous timeindex, which can reopen a gap + left by a prior manual/auto time step selection - the trim is skipped + in that case rather than risk operating on the wrong window. See + :func:`~.flex_opt.charging_strategies.charging_strategy` for details. + """ charging_strategy( self, strategy=strategy, charging_park_ids=charging_park_ids, **kwargs diff --git a/edisgo/flex_opt/charging_strategies.py b/edisgo/flex_opt/charging_strategies.py index dd260e6a9..b9829412a 100644 --- a/edisgo/flex_opt/charging_strategies.py +++ b/edisgo/flex_opt/charging_strategies.py @@ -89,7 +89,29 @@ def charging_strategy( :attr:`~.edisgo.EDisGo.apply_charging_strategy` for more information. Default: 0.1. + Notes + ----- + The written ``loads_active_power``/``loads_reactive_power`` are trimmed to + ``edisgo_obj.timeseries.timeindex`` when its frequency already matches the + SimBEV charging-process data's ``stepsize`` (the common case). When it + doesn't, this function internally resamples ``edisgo_obj.timeseries`` to + SimBEV's frequency and back (see the frequency-mismatch warning below); + that round-trip currently fabricates a contiguous timeindex, which can + reopen a gap left by ``select_timesteps`` (auto mode). The trim is + skipped in that case rather than risk operating on the wrong window - + tracked as a known limitation in + ``docs_notes/issue_temporal_reduction_flexibility_bands.md``. + """ + # Capture the target time index before any internal frequency resampling + # (below) can mutate it. `TimeSeries.resample` fabricates a contiguous + # index spanning first-to-last timestamp, which would silently reopen any + # gap `select_timesteps` (auto mode) deliberately left in the timeindex - + # trimming against this entry-time snapshot instead ensures only the + # steps actually selected by the caller are written. Only used when no + # internal resample round-trip happens (see Notes above). + target_timeindex = edisgo_obj.timeseries.timeindex + # get integrated charging parks integrated_parks = edisgo_obj.electromobility.integrated_charging_parks_df @@ -364,14 +386,48 @@ def charging_strategy( if resample: edisgo_obj.timeseries.resample(freq=edisgo_timedelta) + # `TimeSeries.resample` fabricates a contiguous index spanning + # first-to-last timestamp, which would reopen any gap + # `select_timesteps` (auto mode) left in `target_timeindex`. The trim + # below only removes *extra trailing* rows past `target_timeindex`'s + # own span - it does not (and, given the above, safely cannot) + # reintroduce a gap `resample` already closed. Fixing that root cause + # in `TimeSeries.resample` itself is tracked separately (see + # docs_notes/issue_temporal_reduction_flexibility_bands.md); until + # then, a `select_timesteps`-produced gap combined with a + # SimBEV/edisgo frequency mismatch is a known limitation here. + else: + # Trim the newly written columns down to the target time index. The + # writes above (all three strategies) span the full SimBEV + # simulation length rather than the active timeindex. + # `TimeSeries.loads_active_power` itself already scopes reads to + # `self.timeindex`, but the private `_loads_active_power` can still + # carry the untrimmed rows (visible to anything reading the private + # attribute directly, e.g. `reduce_timeseries_data_to_given_timeindex`) + # - rebuild just the touched columns via drop+add so only the extra + # rows for `edisgo_ids_to_update` are removed, leaving other + # components untouched. + trimmed_active_power = edisgo_obj.timeseries._loads_active_power.loc[ + :, edisgo_ids_to_update + ].reindex(target_timeindex) + edisgo_obj.timeseries.drop_component_time_series( + "loads_active_power", edisgo_ids_to_update + ) + edisgo_obj.timeseries.add_component_time_series( + "loads_active_power", trimmed_active_power + ) - # set reactive power time series to 0 Mvar + # set reactive power time series to 0 Mvar. Use `target_timeindex` only + # when it still matches `edisgo_obj.timeseries.timeindex` (i.e. no + # internal resample round-trip happened above) - see the comment on the + # active-power trim above for why a resampled, gap-closed timeindex isn't + # safely reconcilable with `target_timeindex` here yet. # fmt: off edisgo_obj.timeseries.add_component_time_series( "loads_reactive_power", pd.DataFrame( data=0.0, - index=edisgo_obj.timeseries.timeindex, + index=target_timeindex if not resample else edisgo_obj.timeseries.timeindex, columns=edisgo_ids_to_update, ), ) diff --git a/tests/flex_opt/test_charging_strategy.py b/tests/flex_opt/test_charging_strategy.py index fe533e4e4..96dc85b4e 100644 --- a/tests/flex_opt/test_charging_strategy.py +++ b/tests/flex_opt/test_charging_strategy.py @@ -97,6 +97,63 @@ def test_charging_strategy(self, caplog): charging_strategy(self.edisgo_obj, strategy="dumb") assert ts._loads_active_power.index.freqstr == "15T" + @pytest.mark.parametrize("strategy", ["dumb", "reduced", "residual"]) + def test_charging_strategy_trims_to_short_timeindex(self, strategy): + """ + Regression test for eDisGo#703: charging_strategy used to write the + full SimBEV-simulation-length series into loads_active_power/ + loads_reactive_power regardless of a shorter active timeindex. When + the edisgo/SimBEV frequencies already match (no internal resample + round-trip), the written series must be trimmed to exactly + edisgo.timeseries.timeindex - no extra rows, no missing rows. + """ + edisgo = EDisGo(ding0_grid=self.ding0_path) + # 15-min frequency matches the SimBEV fixture's stepsize (see + # metadata_simbev_run.json), so no internal resample round-trip is + # triggered - one day instead of the fixture's full simulated week. + short_timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") + edisgo.set_timeindex(short_timeindex) + edisgo.import_electromobility( + data_source="directory", + charging_processes_dir=self.simbev_path, + potential_charging_points_dir=self.tracbev_path, + ) + + charging_strategy(edisgo, strategy=strategy) + + pd.testing.assert_index_equal( + edisgo.timeseries._loads_active_power.index, short_timeindex + ) + pd.testing.assert_index_equal( + edisgo.timeseries._loads_reactive_power.index, short_timeindex + ) + assert not edisgo.timeseries.loads_active_power.isna().any().any() + + def test_charging_strategy_trims_to_gapped_timeindex(self): + """ + Regression test for eDisGo#703: a gapped timeindex (as produced by + select_timesteps in auto mode) must survive charging_strategy + unchanged when no internal frequency resample round-trip is + triggered - the written series must match the gapped index exactly, + not a contiguous range spanning it. + """ + edisgo = EDisGo(ding0_grid=self.ding0_path) + gapped_timeindex = pd.date_range("1/1/2011", periods=24, freq="15min").union( + pd.date_range("1/6/2011 18:00", periods=24, freq="15min") + ) + edisgo.set_timeindex(gapped_timeindex) + edisgo.import_electromobility( + data_source="directory", + charging_processes_dir=self.simbev_path, + potential_charging_points_dir=self.tracbev_path, + ) + + charging_strategy(edisgo, strategy="dumb") + + pd.testing.assert_index_equal( + edisgo.timeseries._loads_active_power.index, gapped_timeindex + ) + def test_charging_strategy_with_subset_of_parks(self): """ Charging strategies can be applied to different subsets of charging parks @@ -128,7 +185,6 @@ def test_charging_strategy_with_subset_of_parks(self): # store baseline time series for both parks loads_before = ts._loads_active_power.copy() - ts_a_before = loads_before[edisgo_id_a].copy() ts_b_before = loads_before[edisgo_id_b].copy() # 1) apply a strategy only to park A @@ -152,7 +208,6 @@ def test_charging_strategy_with_subset_of_parks(self): loads_after_second = ts._loads_active_power ts_a_after_second = loads_after_second[edisgo_id_a].copy() - ts_b_after_second = loads_after_second[edisgo_id_b].copy() # park A must not be changed by the second call that targets only park B pd.testing.assert_series_equal( From 22845fbe863519de4d2cd276eb21d68c666bb9d4 Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Mon, 20 Jul 2026 16:16:57 +0200 Subject: [PATCH 55/66] fix: scope apply_heat_pump_operating_strategy output to the active timeindex (#711) operating_strategy computed loads_active_power from all rows of heat_demand_df/cop_df, with no scoping to edisgo.timeseries.timeindex. import_heat_pumps trims both to the timeindex active at import time, but nothing re-trims them if the timeindex changes afterward (e.g. a later select_timesteps step) - any such staleness would silently propagate into loads_active_power via add_component_time_series's raw concat. Scope both operands to edisgo.timeseries.timeindex via .loc[] before dividing. Raises KeyError if heat_demand_df/cop_df are missing data for a time step in the active timeindex, rather than silently writing rows outside it - this is data staleness the caller should fix (re-import or re-set the heat pump time series), not something to paper over. Part of the #703 checklist (docs_notes/issue_temporal_reduction_flexibility_bands.md). --- edisgo/edisgo.py | 10 ++++++++ edisgo/flex_opt/heat_pump_operation.py | 23 +++++++++++++++-- tests/flex_opt/test_heat_pump_operation.py | 29 ++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 433e26091..bef01d254 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -2376,6 +2376,16 @@ def apply_heat_pump_operating_strategy( pumps for which COP information in :attr:`~.edisgo.EDisGo.heat_pump` is given are used. Default: None. + Notes + ----- + The written load time series are scoped to + :attr:`~.network.timeseries.TimeSeries.timeindex`, regardless of + whether :attr:`~.edisgo.EDisGo.heat_pump`'s COP/heat demand time + series currently span a wider or different range. Raises + ``KeyError`` if they are missing data for a time step in the active + timeindex - see :func:`~.flex_opt.heat_pump_operation.operating_strategy` + for details. + """ hp_operating_strategy(self, strategy=strategy, heat_pump_names=heat_pump_names) diff --git a/edisgo/flex_opt/heat_pump_operation.py b/edisgo/flex_opt/heat_pump_operation.py index e6a0d6fb9..1d31006c5 100644 --- a/edisgo/flex_opt/heat_pump_operation.py +++ b/edisgo/flex_opt/heat_pump_operation.py @@ -38,14 +38,33 @@ def operating_strategy( parameter in :attr:`~.edisgo.EDisGo.apply_heat_pump_operating_strategy` for more information. Default: None. + Notes + ----- + The written ``loads_active_power`` is scoped to + ``edisgo_obj.timeseries.timeindex``, regardless of whether + ``edisgo_obj.heat_pump.heat_demand_df``/``cop_df`` currently span a wider + or different range (e.g. because the timeindex changed after + :attr:`~.edisgo.EDisGo.import_heat_pumps` ran). Raises ``KeyError`` if + either is missing data for a time step in ``timeindex`` - this is + data-staleness the caller should fix (re-import or re-set the heat pump + time series for the active timeindex), not something to silently paper + over. + """ if heat_pump_names is None: heat_pump_names = edisgo_obj.heat_pump.cop_df.columns if strategy == "uncontrolled": + # Scope to the active timeindex explicitly rather than relying on + # heat_demand_df/cop_df already matching it - import_heat_pumps trims + # both to the timeindex active at import time, but nothing re-trims + # them if the timeindex changes afterward (e.g. a later + # select_timesteps step), which would otherwise silently write rows + # outside the current timeindex into loads_active_power. + timeindex = edisgo_obj.timeseries.timeindex ts = ( - edisgo_obj.heat_pump.heat_demand_df.loc[:, heat_pump_names] - / edisgo_obj.heat_pump.cop_df.loc[:, heat_pump_names] + edisgo_obj.heat_pump.heat_demand_df.loc[timeindex, heat_pump_names] + / edisgo_obj.heat_pump.cop_df.loc[timeindex, heat_pump_names] ) edisgo_obj.timeseries.add_component_time_series( "loads_active_power", diff --git a/tests/flex_opt/test_heat_pump_operation.py b/tests/flex_opt/test_heat_pump_operation.py index f5b3cad9c..c3fa866f1 100644 --- a/tests/flex_opt/test_heat_pump_operation.py +++ b/tests/flex_opt/test_heat_pump_operation.py @@ -70,3 +70,32 @@ def test_operating_strategy(self): msg = "Heat pump operating strategy dummy is not a valid option." with pytest.raises(ValueError, match=msg): operating_strategy(self.edisgo, strategy="dummy") + + def test_operating_strategy_trims_to_short_timeindex(self): + """ + Regression test for eDisGo#703: operating_strategy used to write + loads_active_power over the full span of heat_demand_df/cop_df + regardless of the active edisgo.timeseries.timeindex. When those are + wider or shifted relative to the active timeindex (e.g. because the + timeindex changed after import_heat_pumps ran), the written series + must be trimmed to exactly edisgo.timeseries.timeindex. + """ + timeindex = pd.date_range("1/1/2011 12:00", periods=2, freq="H") + wide_timeindex = pd.date_range("1/1/2011", periods=24, freq="H") + + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path, timeindex=timeindex) + edisgo.heat_pump.cop_df = pd.DataFrame( + data={"hp1": [5.0] * 24, "hp2": [7.0] * 24}, index=wide_timeindex + ) + edisgo.heat_pump.heat_demand_df = pd.DataFrame( + data={"hp1": [1.0] * 24, "hp2": [3.0] * 24}, index=wide_timeindex + ) + + operating_strategy(edisgo) + + pd.testing.assert_index_equal( + edisgo.timeseries._loads_active_power.index, timeindex + ) + pd.testing.assert_index_equal( + edisgo.timeseries._loads_reactive_power.index, timeindex + ) From 0556909c2beed988b56d20c882fe6b0b7b0a24cb Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Mon, 20 Jul 2026 16:49:40 +0200 Subject: [PATCH 56/66] fix: raise ValueError when set_time_series_manual data misses timesteps (#712) set_time_series_manual only warned when no time index was set, but never validated that a user-supplied DataFrame actually covers a pre-existing edisgo.timeseries.timeindex - a DataFrame missing required timestamps was silently accepted and concatenated via add_component_time_series's raw concat, leaving a partially-populated (NaN-gapped) series with no indication anything was wrong. Add a coverage check: raise ValueError naming the missing timestamps when a non-empty DataFrame doesn't cover the active timeindex. Two exemptions, both required for existing behavior: - Skip entirely when the timeindex is empty - this is the documented "set data first, timeindex later" workflow, and there is nothing meaningful to validate coverage against. - Skip DataFrames with zero columns - nothing is written, so there is nothing to check. Confirmed necessary: eGo's edisgo_integration.py has a real call site passing a zero-column DataFrame as a placeholder no-op. Part of the #703 checklist (docs_notes/issue_temporal_reduction_flexibility_bands.md). --- edisgo/edisgo.py | 48 +++++++++++++++++++++++++++++++++- tests/test_edisgo.py | 62 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index bef01d254..82445a208 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -79,6 +79,38 @@ logger = logging.getLogger(__name__) +def _check_timeindex_coverage(timeindex, name, df): + """ + Raises ``ValueError`` if `df` has columns but is missing data for a time + step in `timeindex`. + + Used by :attr:`~.edisgo.EDisGo.set_time_series_manual` to enforce that + user-provided time series actually cover the active timeindex, instead of + silently writing a partially- or non-overlapping series. A DataFrame with + no columns is exempt - nothing is being written, so there is nothing to + validate coverage for. + + Parameters + ---------- + timeindex : :pandas:`pandas.DatetimeIndex` + Time index to check coverage against. Assumed non-empty by the + caller. + name : str + Parameter name to reference in the raised error message. + df : :pandas:`pandas.DataFrame` or None + DataFrame to check. Skipped if ``None`` or has no columns. + + """ + if df is None or df.shape[1] == 0: + return + missing = timeindex.difference(df.index) + if len(missing) > 0: + raise ValueError( + f"'{name}' does not cover the current timeindex - missing time " + f"steps: {list(missing)}." + ) + + class EDisGo: """ Provides the top-level API for invocation of data import, power flow @@ -351,7 +383,11 @@ def set_time_series_manual( providing the input parameter 'timeindex' or using the function :attr:`~.edisgo.EDisGo.set_timeindex`. Also make sure that the time steps for which time series are provided include - the set time index. + the set time index - this is now enforced: a `ValueError` is raised if a + non-empty DataFrame is missing data for a time step in + :attr:`~.network.timeseries.TimeSeries.timeindex` when a time index is + already set. A DataFrame with no columns is exempt from this check (nothing + is being written, so there is nothing to validate coverage for). """ # check if time index is already set, otherwise raise warning @@ -362,6 +398,16 @@ def set_time_series_manual( "upon initialisation of the EDisGo object by providing the input " "parameter 'timeindex' or using the function EDisGo.set_timeindex()." ) + else: + for name, df in ( + ("generators_p", generators_p), + ("loads_p", loads_p), + ("storage_units_p", storage_units_p), + ("generators_q", generators_q), + ("loads_q", loads_q), + ("storage_units_q", storage_units_q), + ): + _check_timeindex_coverage(self.timeseries.timeindex, name, df) self.timeseries.set_active_power_manual( self, ts_generators=generators_p, diff --git a/tests/test_edisgo.py b/tests/test_edisgo.py index 9847cc7a1..bb6530e59 100755 --- a/tests/test_edisgo.py +++ b/tests/test_edisgo.py @@ -164,7 +164,60 @@ def test_set_time_series_manual(self, caplog): storage_units_ts, self.edisgo.timeseries.storage_units_reactive_power ) - def test_set_time_series_active_power_predefined_demandlib_auto_sets_timeindex(self): + def test_set_time_series_manual_raises_on_missing_timesteps(self): + """ + Regression test for eDisGo#703: set_time_series_manual used to + silently accept a DataFrame missing time steps required by the + active timeindex. It must now raise ValueError instead. + """ + timeindex = pd.date_range("1/1/2018", periods=3, freq="H") + self.edisgo.set_timeindex(timeindex) + + # only 2 of the 3 required time steps + incomplete_ts = pd.DataFrame( + data={"GeneratorFluctuating_15": [2.0, 5.0]}, + index=timeindex[:2], + ) + + with pytest.raises(ValueError, match="generators_p"): + self.edisgo.set_time_series_manual(generators_p=incomplete_ts) + + def test_set_time_series_manual_exempts_zero_column_dataframe(self): + """ + A DataFrame with no columns writes nothing, so it must be exempt + from the timeindex-coverage check even if its (empty) column + selection would otherwise be checked against a mismatched index. + Mirrors a real eGo call site that passes such a DataFrame as a + no-op placeholder. + """ + timeindex = pd.date_range("1/1/2018", periods=3, freq="H") + self.edisgo.set_timeindex(timeindex) + + empty_cols_ts = pd.DataFrame(index=pd.date_range("1/1/1970", periods=1)) + + # must not raise + self.edisgo.set_time_series_manual(generators_q=empty_cols_ts) + + def test_set_time_series_manual_allows_covering_superset(self): + """ + A DataFrame covering the active timeindex (even as a superset with + extra time steps outside it) must still be accepted. + """ + timeindex = pd.date_range("1/1/2018", periods=3, freq="H") + self.edisgo.set_timeindex(timeindex) + + wider_timeindex = pd.date_range("1/1/2018", periods=5, freq="H") + wider_ts = pd.DataFrame( + data={"GeneratorFluctuating_15": [2.0, 5.0, 6.0, 7.0, 8.0]}, + index=wider_timeindex, + ) + + # must not raise + self.edisgo.set_time_series_manual(generators_p=wider_ts) + + def test_set_time_series_active_power_predefined_demandlib_auto_sets_timeindex( + self, + ): edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) # Ensure timeindex is empty initially assert edisgo.timeseries.timeindex.empty @@ -219,7 +272,9 @@ def test_set_time_series_active_power_predefined(self, caplog): # check warning self.edisgo.set_time_series_active_power_predefined() - assert "No timeindex was set. TimeSeries.timeindex is automatically" in caplog.text + assert ( + "No timeindex was set. TimeSeries.timeindex is automatically" in caplog.text + ) # check if right functions are called timeindex = pd.date_range("1/1/2011 12:00", periods=2, freq="H") @@ -422,7 +477,8 @@ def test_generator_import(self): except Exception as e: if "Table does not exist" in str(e) or "HTTP 404" in str(e): pytest.skip( - "Database table not accessible (requires external database connection)" + "Database table not accessible (requires external database " + "connection)" ) else: raise From d68b40ee65aafb76b4b075637a92406687e42604 Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Mon, 20 Jul 2026 17:20:39 +0200 Subject: [PATCH 57/66] fix: scope predefined_* self-provided-DataFrame inputs to the timeindex (#713) The self-provided-DataFrame branches of TimeSeries's four predefined_* methods (predefined_fluctuating_generators_by_technology, predefined_dispatchable_generators_by_technology, predefined_conventional_loads_by_sector, predefined_charging_points_by_use_case) had no coverage check against edisgo.timeseries.timeindex - same gap as set_time_series_manual (#712), just at different call sites. Their 'oedb'/'demandlib' string-option siblings were already correctly scoped via _timeindex_helper_func. Reuse the check_timeindex_coverage helper from #712, relocated from edisgo.py to edisgo/tools/tools.py (both edisgo.py and edisgo/network/timeseries.py need it; a direct cross-import the other way would be circular). Apply it to all four self-provided-DataFrame branches, right after their existing type/emptiness validation. Part of the #703 checklist (docs_notes/issue_temporal_reduction_flexibility_bands.md). --- edisgo/edisgo.py | 35 +------------ edisgo/network/timeseries.py | 57 ++++++++++++++++++++- edisgo/tools/tools.py | 34 ++++++++++++ tests/network/test_timeseries.py | 88 +++++++++++++++++++++++++++++++- 4 files changed, 178 insertions(+), 36 deletions(-) diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 82445a208..6d1c0e557 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -69,6 +69,7 @@ spatial_complexity_reduction, ) from edisgo.tools.tools import ( + check_timeindex_coverage, determine_grid_integration_voltage_level, get_path_length_to_station, ) @@ -79,38 +80,6 @@ logger = logging.getLogger(__name__) -def _check_timeindex_coverage(timeindex, name, df): - """ - Raises ``ValueError`` if `df` has columns but is missing data for a time - step in `timeindex`. - - Used by :attr:`~.edisgo.EDisGo.set_time_series_manual` to enforce that - user-provided time series actually cover the active timeindex, instead of - silently writing a partially- or non-overlapping series. A DataFrame with - no columns is exempt - nothing is being written, so there is nothing to - validate coverage for. - - Parameters - ---------- - timeindex : :pandas:`pandas.DatetimeIndex` - Time index to check coverage against. Assumed non-empty by the - caller. - name : str - Parameter name to reference in the raised error message. - df : :pandas:`pandas.DataFrame` or None - DataFrame to check. Skipped if ``None`` or has no columns. - - """ - if df is None or df.shape[1] == 0: - return - missing = timeindex.difference(df.index) - if len(missing) > 0: - raise ValueError( - f"'{name}' does not cover the current timeindex - missing time " - f"steps: {list(missing)}." - ) - - class EDisGo: """ Provides the top-level API for invocation of data import, power flow @@ -407,7 +376,7 @@ def set_time_series_manual( ("loads_q", loads_q), ("storage_units_q", storage_units_q), ): - _check_timeindex_coverage(self.timeseries.timeindex, name, df) + check_timeindex_coverage(self.timeseries.timeindex, name, df) self.timeseries.set_active_power_manual( self, ts_generators=generators_p, diff --git a/edisgo/network/timeseries.py b/edisgo/network/timeseries.py index 0c4bc9659..eeceb2d9a 100644 --- a/edisgo/network/timeseries.py +++ b/edisgo/network/timeseries.py @@ -23,7 +23,11 @@ from edisgo.flex_opt import q_control from edisgo.io import timeseries_import -from edisgo.tools.tools import assign_voltage_level_to_component, resample +from edisgo.tools.tools import ( + assign_voltage_level_to_component, + check_timeindex_coverage, + resample, +) if TYPE_CHECKING: from edisgo import EDisGo @@ -1264,6 +1268,14 @@ def predefined_fluctuating_generators_by_technology( `ts_generators` is 'oedb' and new ding0 grids with geo-referenced LV grids are used. + Notes + ----- + When `ts_generators` is a self-provided DataFrame and a timeindex is + already set on `edisgo_object`, its index must cover that timeindex - + a `ValueError` is raised naming any missing time steps, rather than + silently writing a partially-covering series. Not checked for the + `'oedb'` option, which is already scoped to the timeindex. + """ # in case time series from oedb are used, retrieve oedb time series if isinstance(ts_generators, str) and ts_generators == "oedb": @@ -1279,6 +1291,13 @@ def predefined_fluctuating_generators_by_technology( raise ValueError( "'ts_generators' must either be a pandas DataFrame or 'oedb'." ) + else: + # self-provided DataFrame - the oedb path above is already scoped + # to edisgo_object's timeindex by feedin_oedb/feedin_oedb_legacy + if not edisgo_object.timeseries.timeindex.empty: + check_timeindex_coverage( + edisgo_object.timeseries.timeindex, "ts_generators", ts_generators + ) # set generator_names if None if generator_names is None: @@ -1361,9 +1380,20 @@ def predefined_dispatchable_generators_by_technology( 'other', all dispatchable generators in the network (i.e. all but solar and wind generators) are used. + Notes + ----- + If a timeindex is already set on `edisgo_object`, `ts_generators`' + index must cover it - a `ValueError` is raised naming any missing + time steps, rather than silently writing a partially-covering + series. + """ if not isinstance(ts_generators, pd.DataFrame): raise ValueError("'ts_generators' must be a pandas DataFrame.") + if not edisgo_object.timeseries.timeindex.empty: + check_timeindex_coverage( + edisgo_object.timeseries.timeindex, "ts_generators", ts_generators + ) # write to TimeSeriesRaw for col in ts_generators: @@ -1444,6 +1474,14 @@ def predefined_conventional_loads_by_sector( in :func:`edisgo.io.timeseries_import.load_time_series_demandlib` for more information. + Notes + ----- + When `ts_loads` is a self-provided DataFrame and a timeindex is + already set on `edisgo_object`, its index must cover that timeindex - + a `ValueError` is raised naming any missing time steps, rather than + silently writing a partially-covering series. Not checked for the + `'demandlib'` option, which is already scoped to the timeindex. + """ # in case time series from demandlib are used, retrieve demandlib time series if isinstance(ts_loads, str) and ts_loads == "demandlib": @@ -1457,6 +1495,12 @@ def predefined_conventional_loads_by_sector( elif ts_loads.empty: logger.warning("The profile you entered is empty. Method is skipped.") return + elif not edisgo_object.timeseries.timeindex.empty: + # self-provided DataFrame - the demandlib path above is already + # scoped to edisgo_object's timeindex by load_time_series_demandlib + check_timeindex_coverage( + edisgo_object.timeseries.timeindex, "ts_loads", ts_loads + ) # write to TimeSeriesRaw for col in ts_loads: @@ -1520,12 +1564,23 @@ def predefined_charging_points_by_use_case( If None, all charging points of use cases for which use-case-specific time series are provided are used. + Notes + ----- + If a timeindex is already set on `edisgo_object`, `ts_loads`' index + must cover that timeindex - a `ValueError` is raised naming any + missing time steps, rather than silently writing a + partially-covering series. + """ if not isinstance(ts_loads, pd.DataFrame): raise ValueError("'ts_loads' must be a pandas DataFrame.") elif ts_loads.empty: logger.warning("The profile you entered is empty. Method is skipped.") return + elif not edisgo_object.timeseries.timeindex.empty: + check_timeindex_coverage( + edisgo_object.timeseries.timeindex, "ts_loads", ts_loads + ) # write to TimeSeriesRaw for col in ts_loads: diff --git a/edisgo/tools/tools.py b/edisgo/tools/tools.py index ead0219e5..8a208cd56 100644 --- a/edisgo/tools/tools.py +++ b/edisgo/tools/tools.py @@ -87,6 +87,40 @@ def align_series_to_timeindex(ts, timeindex, extra_step=False): return ts.reindex(target) +def check_timeindex_coverage(timeindex, name, df): + """ + Raises ``ValueError`` if `df` has columns but is missing data for a time + step in `timeindex`. + + Used by :attr:`~.edisgo.EDisGo.set_time_series_manual` and the + self-provided-DataFrame options of + :class:`~.network.timeseries.TimeSeries`'s ``predefined_*`` methods to + enforce that user-provided time series actually cover the active + timeindex, instead of silently writing a partially- or non-overlapping + series. A DataFrame with no columns is exempt - nothing is being + written, so there is nothing to validate coverage for. + + Parameters + ---------- + timeindex : :pandas:`pandas.DatetimeIndex` + Time index to check coverage against. Assumed non-empty by the + caller. + name : str + Parameter name to reference in the raised error message. + df : :pandas:`pandas.DataFrame` or None + DataFrame to check. Skipped if ``None`` or has no columns. + + """ + if df is None or df.shape[1] == 0: + return + missing = timeindex.difference(df.index) + if len(missing) > 0: + raise ValueError( + f"'{name}' does not cover the current timeindex - missing time " + f"steps: {list(missing)}." + ) + + def select_worstcase_snapshots(edisgo_obj): """ Select two worst-case snapshots from time series diff --git a/tests/network/test_timeseries.py b/tests/network/test_timeseries.py index 15b9c1071..ee0274cc9 100644 --- a/tests/network/test_timeseries.py +++ b/tests/network/test_timeseries.py @@ -1403,6 +1403,27 @@ def test_predefined_fluctuating_generators_by_technology(self): ) # fmt: on + def test_predefined_fluctuating_generators_by_technology_missing_timesteps( + self, + ): + """ + Regression test for eDisGo#703: the self-provided-DataFrame path + used to silently accept a DataFrame missing time steps required by + the active timeindex. + """ + timeindex = pd.date_range("1/1/2011 12:00", periods=2, freq="H") + self.edisgo.timeseries.timeindex = timeindex + + incomplete_ts = pd.DataFrame( + data={"wind": [1], "solar": [3]}, + index=timeindex[:1], + ) + + with pytest.raises(ValueError, match="ts_generators"): + self.edisgo.timeseries.predefined_fluctuating_generators_by_technology( + self.edisgo, incomplete_ts + ) + def test_predefined_fluctuating_generators_by_technology_oedb(self): edisgo_object = EDisGo( ding0_grid=pytest.ding0_test_network_3_path, legacy_ding0_grids=False @@ -1548,6 +1569,27 @@ def test_predefined_dispatchable_generators_by_technology(self): ) # fmt: on + def test_predefined_dispatchable_generators_by_technology_missing_timesteps( + self, + ): + """ + Regression test for eDisGo#703: the self-provided-DataFrame path + used to silently accept a DataFrame missing time steps required by + the active timeindex. + """ + timeindex = pd.date_range("1/1/2011 12:00", periods=2, freq="H") + self.edisgo.timeseries.timeindex = timeindex + + incomplete_ts = pd.DataFrame( + data={"other": [5]}, + index=timeindex[:1], + ) + + with pytest.raises(ValueError, match="ts_generators"): + self.edisgo.timeseries.predefined_dispatchable_generators_by_technology( + self.edisgo, incomplete_ts + ) + def test_predefined_conventional_loads_by_sector(self, caplog): index = pd.date_range("1/1/2018", periods=3, freq="H") self.edisgo.timeseries.timeindex = index @@ -1812,6 +1854,27 @@ def test_predefined_conventional_loads_by_sector(self, caplog): original_annual_consumption ) + def test_predefined_conventional_loads_by_sector_raises_on_missing_timesteps( + self, + ): + """ + Regression test for eDisGo#703: the self-provided-DataFrame path + used to silently accept a DataFrame missing time steps required by + the active timeindex. + """ + timeindex = pd.date_range("1/1/2018", periods=3, freq="H") + self.edisgo.timeseries.timeindex = timeindex + + incomplete_ts = pd.DataFrame( + data={"residential": [1, 2]}, + index=timeindex[:2], + ) + + with pytest.raises(ValueError, match="ts_loads"): + self.edisgo.timeseries.predefined_conventional_loads_by_sector( + self.edisgo, incomplete_ts + ) + def test_predefined_charging_points_by_use_case(self, caplog): index = pd.date_range("1/1/2018", periods=3, freq="H") self.edisgo.timeseries.timeindex = index @@ -1919,6 +1982,27 @@ def test_predefined_charging_points_by_use_case(self, caplog): == (3, 5) # fmt: on + def test_predefined_charging_points_by_use_case_raises_on_missing_timesteps( + self, + ): + """ + Regression test for eDisGo#703: the self-provided-DataFrame path + used to silently accept a DataFrame missing time steps required by + the active timeindex. + """ + timeindex = pd.date_range("1/1/2018", periods=3, freq="H") + self.edisgo.timeseries.timeindex = timeindex + + incomplete_ts = pd.DataFrame( + data={"home": [1, 2]}, + index=timeindex[:2], + ) + + with pytest.raises(ValueError, match="ts_loads"): + self.edisgo.timeseries.predefined_charging_points_by_use_case( + self.edisgo, incomplete_ts + ) + def test_fixed_cosphi(self): # set active power time series for fixed cosphi timeindex = pd.date_range("1/1/1970", periods=3, freq="H") @@ -2339,8 +2423,8 @@ def test_integrity_check(self, caplog): setattr(self.edisgo.timeseries, attr, ts_tmp_duplicated) self.edisgo.timeseries.check_integrity() assert ( - f"{attr} has duplicated columns: {ts_tmp.iloc[:, 0:2].columns.values}" - in caplog.text + f"{attr} has duplicated columns: " + f"{ts_tmp.iloc[:, 0:2].columns.values}" in caplog.text ) caplog.clear() setattr(self.edisgo.timeseries, attr, ts_tmp) From dc1249baa87e7d19de6bac6626b602d6efc79b27 Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Tue, 21 Jul 2026 09:51:33 +0200 Subject: [PATCH 58/66] fix: scope get_flexibility_bands to the active timeindex (#714) get_flexibility_bands built EV charging flexibility bands spanning SimBEV's own native calendar and simulated date range, independent of edisgo.timeseries.timeindex. SimBEV's calendar is commonly a fixed reference year (2011 in the test fixtures); a scenario's own timeindex is commonly a different year (e.g. 2035) and/or a much shorter window - indexing the returned bands by such a timeindex raised KeyError, the literal crash originally reported in #703. A workaround already existed at one call site (task_build_flexibility_bands, which called reduce_timeseries_data_to_given_timeindex right after get_flexibility_bands), but this was a local patch, not a fix to the function's own contract - any other caller was still exposed. get_flexibility_bands now year-aligns (reusing align_series_to_timeindex) and trims its returned/stored bands to edisgo_obj.timeseries.timeindex whenever it's non-empty, regardless of the resample parameter, since this is a correctness fix rather than an optional resampling convenience. Band construction itself is untouched (still spans SimBEV's full native range) - a charging event straddling a later window's boundary must still count toward the band inside that window. Empty timeindex remains a no-op. Remove the now-redundant explicit reduce_timeseries_data_to_given_timeindex call in task_build_flexibility_bands, since the fix makes it a no-op there. --- edisgo/network/electromobility.py | 43 ++++++++++++++++---- edisgo/run/tasks/flex.py | 34 +++++----------- tests/network/test_electromobility.py | 56 ++++++++++++++++++++++++++- tests/run/test_tasks.py | 36 +++++++++++++++++ 4 files changed, 136 insertions(+), 33 deletions(-) diff --git a/edisgo/network/electromobility.py b/edisgo/network/electromobility.py index 8237febff..7d72f6341 100644 --- a/edisgo/network/electromobility.py +++ b/edisgo/network/electromobility.py @@ -23,6 +23,7 @@ from sklearn import preprocessing from edisgo.network.components import PotentialChargingParks +from edisgo.tools.tools import align_series_to_timeindex if "READTHEDOCS" not in os.environ: import geopandas as gpd @@ -396,6 +397,21 @@ def get_flexibility_bands( for more information. To avoid this behaviour, set `tol` to 0.0. Default: 1e-6. + Notes + ----- + The bands are always built spanning SimBEV's own native calendar and + simulated date range (independent of ``edisgo_obj.timeseries.timeindex`` + - a charging process straddling a later window's boundary must still + count toward the band inside that window). If + ``edisgo_obj.timeseries.timeindex`` is non-empty, the returned/stored + bands are then year-aligned (SimBEV's calendar is commonly a fixed + reference year, independent of the scenario year) and trimmed to + exactly that timeindex - this is done regardless of `resample`, since + it is a correctness fix (avoiding a ``KeyError`` when a consumer later + indexes the bands by ``edisgo_obj.timeseries.timeindex``), not an + optional resampling convenience. When the timeindex is empty, the + bands are returned untouched, spanning SimBEV's own range/calendar. + Returns -------- dict(str, :pandas:`pandas.DataFrame`) @@ -582,15 +598,26 @@ def get_flexibility_bands( # sanity check self.check_integrity() - # check time index + + # Scope the bands to edisgo_obj's own timeindex, so this method is + # correct regardless of caller (not just the run pipeline, which + # previously had to patch this up itself via + # reduce_timeseries_data_to_given_timeindex right after calling this + # method). The bands built above always span SimBEV's own native + # calendar (its start_date, typically a fixed reference year like + # 2011) and simulated range - independent of edisgo_timeindex, which + # is why this can't just be a `.loc[edisgo_timeindex]` here: a year + # mismatch alone would raise KeyError, and a shorter/different-range + # edisgo_timeindex would too. align_series_to_timeindex year-shifts + # and reindexes (filling any still-missing steps with NaN rather than + # raising) before the final trim below. if len(edisgo_timeindex) > 0: - missing_indices = [_ for _ in edisgo_timeindex if _ not in flex_band_index] - if len(missing_indices) > 0: - logger.warning( - "There are time steps in timeindex of TimeSeries object that " - "are not in the index of the flexibility bands. This may lead " - "to problems." - ) + for key, df in self.flexibility_bands.items(): + if not df.empty: + self.flexibility_bands[key] = align_series_to_timeindex( + df, edisgo_timeindex + ).loc[edisgo_timeindex] + return self.flexibility_bands def fix_flexibility_bands_rounding_errors(self, tol=1e-6): diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py index 9c3edab80..541b4abe4 100644 --- a/edisgo/run/tasks/flex.py +++ b/edisgo/run/tasks/flex.py @@ -210,19 +210,16 @@ def task_build_flexibility_bands(edisgo, ctx, *, use_case=None): :func:`task_import_electromobility` can do inline. Running it as a separate step lets it execute *after* the analysis time index is fixed (e.g. after ``oedb_ts`` / timestep selection), so - :meth:`Electromobility.get_flexibility_bands` resamples the bands to the - edisgo time-series frequency instead of leaving them at the raw SimBEV - resolution. This mirrors how the heat-pump time series are set outside - ``import_heat_pumps``, and is more efficient than building bands over a - non-final index. - - ``get_flexibility_bands`` only resamples to the active *frequency* - the - bands still span whatever date range the underlying SimBEV charging- - process data covers, which is not necessarily the same range as - ``edisgo.timeseries.timeindex`` (e.g. a manually-selected window). This - task additionally trims the bands down to that exact index, so - ``electromobility.flexibility_bands`` always matches - ``edisgo.timeseries.timeindex`` after this step runs. + :meth:`Electromobility.get_flexibility_bands` resamples/scopes the bands + to the edisgo time-series frequency and timeindex instead of leaving + them at the raw SimBEV resolution and range. This mirrors how the + heat-pump time series are set outside ``import_heat_pumps``, and is more + efficient than building bands over a non-final index. + + ``get_flexibility_bands`` itself year-aligns and trims the bands down to + ``edisgo.timeseries.timeindex`` (see its docstring) whenever that + timeindex is non-empty, so ``electromobility.flexibility_bands`` always + matches it after this step runs - no separate trim call needed here. Parameters ---------- @@ -240,20 +237,9 @@ def task_build_flexibility_bands(edisgo, ctx, *, use_case=None): edisgo.EDisGo The modified EDisGo instance. """ - from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex - if use_case is None: use_case = ["home", "work", "public", "hpc"] edisgo.electromobility.get_flexibility_bands(edisgo, use_case=use_case) - reduce_timeseries_data_to_given_timeindex( - edisgo, - edisgo.timeseries.timeindex, - timeseries=False, - electromobility=True, - heat_pump=False, - dsm=False, - overlying_grid=False, - ) return edisgo diff --git a/tests/network/test_electromobility.py b/tests/network/test_electromobility.py index 2afeeb660..dfc5450ab 100644 --- a/tests/network/test_electromobility.py +++ b/tests/network/test_electromobility.py @@ -8,7 +8,7 @@ import pandas as pd import pytest -from pandas.testing import assert_frame_equal +from pandas.testing import assert_frame_equal, assert_index_equal from edisgo.edisgo import EDisGo from edisgo.io import electromobility_import @@ -185,6 +185,60 @@ def test_get_flexibility_bands(self): ].index assert (flex_bands_index[1] - flex_bands_index[0]) == pd.Timedelta("1H") + def test_get_flexibility_bands_scopes_to_mismatched_timeindex(self): + """ + Regression test for eDisGo#703: get_flexibility_bands used to build + bands spanning SimBEV's own native calendar/range only, with no + alignment to edisgo.timeseries.timeindex - indexing the bands by a + timeindex in a different year (SimBEV's start_date here is 2011) + and/or a shorter window than SimBEV's simulated range raised + KeyError. The bands must now be year-aligned and trimmed to exactly + that timeindex. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + assert edisgo_obj.electromobility.simbev_config_df.start_date.values[ + 0 + ] == np.datetime64("2011-01-01") + short_timeindex = pd.date_range("2035-01-15", periods=24, freq="h") + edisgo_obj.set_timeindex(short_timeindex) + + bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, ["work", "public"] + ) + + for key in ("upper_power", "lower_energy", "upper_energy"): + assert_index_equal(bands[key].index, short_timeindex) + # must not raise KeyError + edisgo_obj.electromobility.flexibility_bands[key].loc[short_timeindex] + + def test_get_flexibility_bands_empty_timeindex_is_a_no_op(self): + """ + With no timeindex set at all, get_flexibility_bands must return the + bands untouched, spanning SimBEV's own native calendar/range - there + is nothing to align/trim against yet. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + assert edisgo_obj.timeseries.timeindex.empty + + bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, ["work", "public"] + ) + + assert len(bands["upper_power"].index) == 7 * 96 # 7 days, 15-min steps + assert bands["upper_power"].index[0] == pd.Timestamp("2011-01-01") + def test_fix_flexibility_bands_rounding_errors(self, caplog): # set up test data # set charging efficiency to 1 to make things easier diff --git a/tests/run/test_tasks.py b/tests/run/test_tasks.py index 1f1db2237..46ba7a9cc 100644 --- a/tests/run/test_tasks.py +++ b/tests/run/test_tasks.py @@ -17,9 +17,11 @@ import edisgo.run as edisgo_run from edisgo.edisgo import EDisGo +from edisgo.io import electromobility_import from edisgo.run.config import load_config from edisgo.run.context import RunContext from edisgo.run.tasks.analysis import task_optimize +from edisgo.run.tasks.flex import task_build_flexibility_bands from edisgo.run.tasks.io import task_import_overlying_grid_data from edisgo.run.tasks.timeseries import ( task_manual_ts, @@ -58,6 +60,40 @@ def test_manual_ts_applies_active_power(self, edisgo_obj): assert ctx.flags["timeseries_set"] is True +class TestBuildFlexibilityBands: + def test_build_flexibility_bands_scopes_to_timeindex(self): + """ + Regression test for eDisGo#703: task_build_flexibility_bands used to + need an explicit reduce_timeseries_data_to_given_timeindex call after + get_flexibility_bands to trim/year-align the bands to the active + timeindex; get_flexibility_bands now does this itself, so the task + (which no longer makes that call) must still produce bands matching + edisgo.timeseries.timeindex exactly - including across the year + mismatch between SimBEV's own calendar (2011 in this fixture) and + the scenario timeindex (2035 here). + """ + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo, + pytest.simbev_example_scenario_path, + pytest.tracbev_example_scenario_path, + ) + electromobility_import.distribute_charging_demand(edisgo) + electromobility_import.integrate_charging_parks(edisgo) + + short_timeindex = pd.date_range("2035-01-15", periods=24, freq="h") + edisgo.set_timeindex(short_timeindex) + + ctx = RunContext() + result = task_build_flexibility_bands(edisgo, ctx) + + for key in ("upper_power", "lower_energy", "upper_energy"): + pd.testing.assert_index_equal( + result.electromobility.flexibility_bands[key].index, + short_timeindex, + ) + + class TestImportOverlyingGridData: def _ctx(self, og_cfg, overlying_grid_data=None): return RunContext( From 94292ab0c39c40b1cdec46efb5cc702a517d84ed Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Tue, 21 Jul 2026 10:47:24 +0200 Subject: [PATCH 59/66] fix: add DSM.resample(), wire it into EDisGo.resample_timeseries (#715) DSM had no resample() method, unlike every sibling container (TimeSeries, HeatPump, Electromobility, OverlyingGrid). EDisGo.resample_timeseries() - the public API for changing an entire EDisGo object's time resolution - never touched dsm, silently leaving p_min/p_max/e_min/e_max at their original frequency while everything else changed, with no warning. Add DSM.resample(method, freq), mirroring HeatPump.resample_timeseries's signature/pattern and iterating the existing DSM._attributes list. Wire it into EDisGo.resample_timeseries and update its docstring's affected-attributes list, which previously omitted DSM entirely. Also remove a manual DSM-resample workaround in tests/network/test_overlying_grid.py's setup_flexibility_data() that predated this fix (it hand-rolled exactly what DSM.resample() now does correctly). With the real fix in place, that workaround double-resampled already-correct data and broke test_distribute_overlying_grid_timeseries with a shape mismatch - removing it resolves that. Part of the #703 checklist (docs_notes/issue_temporal_reduction_flexibility_bands.md). --- edisgo/edisgo.py | 13 +++++++++++-- edisgo/network/dsm.py | 28 ++++++++++++++++++++++++++++ tests/network/test_dsm.py | 27 +++++++++++++++++++++++++++ tests/network/test_overlying_grid.py | 23 +++-------------------- tests/test_edisgo.py | 10 ++++++++++ 5 files changed, 79 insertions(+), 22 deletions(-) diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 6d1c0e557..03f4caea7 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -3811,8 +3811,8 @@ def resample_timeseries( """ Resamples time series data in :class:`~.network.timeseries.TimeSeries`, :class:`~.network.heat.HeatPump`, - :class:`~.network.electromobility.Electromobility` and - :class:`~.network.overlying_grid.OverlyingGrid`. + :class:`~.network.electromobility.Electromobility`, + :class:`~.network.dsm.DSM` and :class:`~.network.overlying_grid.OverlyingGrid`. Both up- and down-sampling methods are possible. @@ -3836,6 +3836,14 @@ def resample_timeseries( * :attr:`~.network.heat.HeatPump.heat_demand_df` + * :attr:`~.network.dsm.DSM.p_min` + + * :attr:`~.network.dsm.DSM.p_max` + + * :attr:`~.network.dsm.DSM.e_min` + + * :attr:`~.network.dsm.DSM.e_max` + * All data in :class:`~.network.overlying_grid.OverlyingGrid` Parameters @@ -3868,6 +3876,7 @@ def resample_timeseries( self.timeseries.resample(method=method, freq=freq) self.electromobility.resample(freq=freq) self.heat_pump.resample_timeseries(method=method, freq=freq) + self.dsm.resample(method=method, freq=freq) self.overlying_grid.resample(method=method, freq=freq) diff --git a/edisgo/network/dsm.py b/edisgo/network/dsm.py index a19258752..88065c7eb 100644 --- a/edisgo/network/dsm.py +++ b/edisgo/network/dsm.py @@ -19,6 +19,8 @@ import numpy as np import pandas as pd +from edisgo.tools import tools + logger = logging.getLogger(__name__) @@ -152,6 +154,32 @@ def _attributes(self): "e_max", ] + def resample(self, method: str = "ffill", freq: str | pd.Timedelta = "15min"): + """ + Resamples DSM potential time series to a desired resolution. + + Both up- and down-sampling methods are possible. + + Parameters + ---------- + method : str, optional + See :attr:`~.EDisGo.resample_timeseries` for more information. + + freq : str, optional + See :attr:`~.EDisGo.resample_timeseries` for more information. + + """ + for attr in self._attributes: + attr_index = getattr(self, attr).index + if len(attr_index) < 2: + logger.debug( + f"{attr} cannot be resampled as it contains less than two " + f"time steps." + ) + else: + freq_orig = attr_index[1] - attr_index[0] + tools.resample(self, freq_orig, method, freq, attr_to_resample=[attr]) + def reduce_memory( self, attr_to_reduce=None, diff --git a/tests/network/test_dsm.py b/tests/network/test_dsm.py index 3e2ea7201..087de2071 100644 --- a/tests/network/test_dsm.py +++ b/tests/network/test_dsm.py @@ -66,6 +66,33 @@ def test_reduce_memory(self): self.dsm.e_max = pd.DataFrame() self.dsm.reduce_memory() + def test_resample(self): + """ + Regression test for eDisGo#703: DSM had no resample() method, so + EDisGo.resample_timeseries silently left DSM data at its original + frequency while every sibling container (TimeSeries, Electromobility, + HeatPump, OverlyingGrid) was resampled. + """ + # test up-sampling with default parameters + self.dsm.resample() + assert len(self.dsm.p_max) == 8 + assert (self.dsm.p_max.iloc[0:4, 0] == 5).all() + assert (self.dsm.p_max.iloc[4:8, 1] == 8).all() + assert len(self.dsm.p_min) == 8 + assert len(self.dsm.e_max) == 8 + assert len(self.dsm.e_min) == 8 + + # test down-sampling + self.dsm.resample(freq="1H") + assert len(self.dsm.p_max) == 2 + assert len(self.dsm.p_min) == 2 + assert len(self.dsm.e_max) == 2 + assert len(self.dsm.e_min) == 2 + + # test with empty dataframes - must not raise + self.dsm.e_max = pd.DataFrame() + self.dsm.resample() + def test_to_csv(self): # test with default values save_dir = os.path.join(os.getcwd(), "dsm_csv") diff --git a/tests/network/test_overlying_grid.py b/tests/network/test_overlying_grid.py index 83ff3564c..da77ef202 100644 --- a/tests/network/test_overlying_grid.py +++ b/tests/network/test_overlying_grid.py @@ -273,28 +273,11 @@ def setup_flexibility_data(self): df, ) - # Resample timeseries and reindex to hourly timedelta + # Resample timeseries and reindex to hourly timedelta. DSM (p_min/p_max) + # is resampled by this call too (eDisGo#703) - no separate manual + # DSM resample needed anymore. self.edisgo.resample_timeseries(freq="1min") - for attr in ["p_min", "p_max"]: - new_dates = pd.DatetimeIndex( - [getattr(self.edisgo.dsm, attr).index[-1] + pd.Timedelta("1h")] - ) - setattr( - self.edisgo.dsm, - attr, - getattr(self.edisgo.dsm, attr) - .reindex( - getattr(self.edisgo.dsm, attr) - .index.union(new_dates) - .unique() - .sort_values() - ) - .ffill() - .resample("1min") - .ffill() - .iloc[:-1], - ) self.timesteps = pd.date_range(start="01/01/2018", periods=240, freq="h") attributes = self.edisgo.timeseries._attributes for attr in attributes: diff --git a/tests/test_edisgo.py b/tests/test_edisgo.py index bb6530e59..dd9849d63 100755 --- a/tests/test_edisgo.py +++ b/tests/test_edisgo.py @@ -2055,9 +2055,19 @@ def test_resample_timeseries(self): }, index=pd.date_range("1/1/2011 12:00", periods=2, freq="H"), ) + # regression test for eDisGo#703: DSM data used to be silently left + # at its original frequency by resample_timeseries + self.edisgo.dsm.p_max = pd.DataFrame( + data={ + "load_1": [5.0, 6.0], + "load_2": [7.0, 8.0], + }, + index=pd.date_range("1/1/2011 12:00", periods=2, freq="H"), + ) self.edisgo.resample_timeseries(freq="30min") assert len(self.edisgo.timeseries.loads_active_power) == 8 assert len(self.edisgo.heat_pump.cop_df) == 4 + assert len(self.edisgo.dsm.p_max) == 4 class TestEDisGoFunc: From bc24afaff5f3cfd9fc66ca63256a4c77a5259e74 Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Tue, 21 Jul 2026 11:53:36 +0200 Subject: [PATCH 60/66] fix: expose reduce_timeseries_data_to_given_timeindex on EDisGo (#716) reduce_timeseries_data_to_given_timeindex (edisgo/tools/tools.py) is a well-implemented, correct function that trims TimeSeries, Electromobility.flexibility_bands (with year-align), HeatPump, DSM, and OverlyingGrid attributes consistently to a target timeindex - but it was only ever importable from edisgo.tools.tools, never exposed on the EDisGo class. Its only prior callers were internal run-pipeline tasks; anyone using EDisGo directly had no discoverable way to find or use it. Add EDisGo.reduce_timeseries_data_to_given_timeindex(...), a thin wrapper delegating to the existing free function with the same signature and defaults, mirroring how EDisGo.resample_timeseries already delegates to underlying implementations. No changes to the tested logic in tools.py; the three existing internal call sites can keep importing the free function directly. Part of the #703 checklist (docs_notes/issue_temporal_reduction_flexibility_bands.md). This completes the #703 checklist (7/7). --- edisgo/edisgo.py | 69 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_edisgo.py | 25 ++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 03f4caea7..23bdf1fc0 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -72,6 +72,7 @@ check_timeindex_coverage, determine_grid_integration_voltage_level, get_path_length_to_station, + reduce_timeseries_data_to_given_timeindex, ) if "READTHEDOCS" not in os.environ: @@ -3879,6 +3880,74 @@ def resample_timeseries( self.dsm.resample(method=method, freq=freq) self.overlying_grid.resample(method=method, freq=freq) + def reduce_timeseries_data_to_given_timeindex( + self, + timeindex, + freq="1H", + timeseries=True, + electromobility=True, + save_ev_soc_initial=True, + heat_pump=True, + dsm=True, + overlying_grid=True, + ): + """ + Reduces timeseries data in this EDisGo object to given time index. + + Thin wrapper around + :func:`edisgo.tools.tools.reduce_timeseries_data_to_given_timeindex`, + exposed here for discoverability - the underlying implementation is + otherwise only importable directly from ``edisgo.tools.tools``, which + made it easy to miss for anyone using :class:`~.EDisGo` outside the + ``run`` pipeline (its only prior callers). + + Parameters + ----------- + timeindex : :pandas:`pandas.DatetimeIndex` + Time index to set. + freq : str or :pandas:`pandas.Timedelta`, optional + Frequency of time series data. This is only needed if it cannot be + inferred from the given `timeindex` and if electromobility data + and/or overlying grid data is reduced, as the initial SoC is + tried to be set using the time step before the first time step in + the given `timeindex`. Offset aliases can be found here: + https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases. + Default: '1H'. + timeseries : bool + Indicates whether timeseries in + :class:`~.network.timeseries.TimeSeries` are reduced to given + time index. Default: True. + electromobility : bool + Indicates whether timeseries in + :class:`~.network.electromobility.Electromobility` are reduced to + given time index. Default: True. + save_ev_soc_initial : bool + Indicates whether to save initial EV SOC from timestep before + first timestep of given time index. Default: True. + heat_pump : bool + Indicates whether timeseries in :class:`~.network.heat.HeatPump` + are reduced to given time index. Default: True. + dsm : bool + Indicates whether timeseries in :class:`~.network.dsm.DSM` are + reduced to given time index. Default: True. + overlying_grid : bool + Indicates whether timeseries in + :class:`~.network.overlying_grid.OverlyingGrid` are reduced to + given time index. Default: True. + + """ + reduce_timeseries_data_to_given_timeindex( + self, + timeindex, + freq=freq, + timeseries=timeseries, + electromobility=electromobility, + save_ev_soc_initial=save_ev_soc_initial, + heat_pump=heat_pump, + dsm=dsm, + overlying_grid=overlying_grid, + ) + def import_edisgo_from_pickle(filename, path=""): """ diff --git a/tests/test_edisgo.py b/tests/test_edisgo.py index dd9849d63..9184a44a7 100755 --- a/tests/test_edisgo.py +++ b/tests/test_edisgo.py @@ -18,6 +18,7 @@ from edisgo.edisgo import import_edisgo_from_files from edisgo.flex_opt.reinforce_grid import enhanced_reinforce_grid from edisgo.network.results import Results +from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex class TestEDisGo: @@ -2069,6 +2070,30 @@ def test_resample_timeseries(self): assert len(self.edisgo.heat_pump.cop_df) == 4 assert len(self.edisgo.dsm.p_max) == 4 + def test_reduce_timeseries_data_to_given_timeindex(self): + """ + EDisGo.reduce_timeseries_data_to_given_timeindex is a thin wrapper + around edisgo.tools.tools.reduce_timeseries_data_to_given_timeindex, + added for discoverability (eDisGo#703 checklist item 7). Must produce + the same result as calling the free function directly. + """ + self.setup_worst_case_time_series() + target_timeindex = self.edisgo.timeseries.timeindex[:2] + + edisgo_via_method = deepcopy(self.edisgo) + edisgo_via_method.reduce_timeseries_data_to_given_timeindex(target_timeindex) + + edisgo_via_function = deepcopy(self.edisgo) + reduce_timeseries_data_to_given_timeindex(edisgo_via_function, target_timeindex) + + assert_frame_equal( + edisgo_via_method.timeseries.loads_active_power, + edisgo_via_function.timeseries.loads_active_power, + ) + pd.testing.assert_index_equal( + edisgo_via_method.timeseries.timeindex, target_timeindex + ) + class TestEDisGoFunc: def test_import_edisgo_from_files(self): From 800cc9e45a6c2a5b916cd4026d2902c350d5d7dc Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Wed, 22 Jul 2026 14:23:37 +0200 Subject: [PATCH 61/66] fix: align flexibility_bands reads to the timeindex in the OPF builder (#719) _build_electromobility and _build_component_timeseries's electromobility branch read edisgo_obj.electromobility.flexibility_bands positionally (.iloc[0] for static bounds, .values.tolist() for the full per-timestep column) instead of aligning to edisgo_obj.timeseries.timeindex first. This was only safe because the standard run pipeline always finalizes the timeindex before building bands and running optimize - an implicit ordering convention, not something either function or the validator enforced. Breaking that ordering would silently feed wrong/misaligned or length-mismatched OPF input to PowerModels/Julia rather than raising a clear error. Both functions now explicitly select flexibility_bands[key].loc[timeindex] (or .loc[timeindex, flexible_cps]) before any positional access, mirroring the alignment pattern already established in Electromobility.get_flexibility_bands (#703). A genuine mismatch now raises KeyError immediately instead of silently producing wrong data. No separate length assertion against pm["time_series"]["num_steps"] was needed: to_powermodels always builds psa_net via edisgo_object.to_pypsa() with no explicit timesteps override, so psa_net.snapshots is always exactly edisgo_obj.timeseries.timeindex here - the .loc[timeindex] alignment already guarantees the length matches. Fixes #718. --- edisgo/io/powermodels_io.py | 29 +++++++--- tests/io/test_powermodels_io.py | 97 +++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 7 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 223d96cf3..fbf6457e1 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1181,7 +1181,16 @@ def _build_electromobility(edisgo_obj, psa_net, pm, s_base, flexible_cps): Updated array containing all charging points that allow for flexible charging. """ - flex_bands_df = edisgo_obj.electromobility.flexibility_bands + # Align to the active timeindex explicitly rather than relying on + # flexibility_bands already being in the same order/length - correct + # regardless of what ran before this function (e.g. a second + # select_timesteps step, or direct EDisGo API use), not just when the + # standard run pipeline's task ordering happens to keep them aligned. + timeindex = edisgo_obj.timeseries.timeindex + flex_bands_df = { + key: df.loc[timeindex] + for key, df in edisgo_obj.electromobility.flexibility_bands.items() + } if (flex_bands_df["lower_energy"] > flex_bands_df["upper_energy"]).any().any(): logger.warning( "Upper energy level is smaller than lower energy level for " @@ -1920,21 +1929,27 @@ def _build_component_timeseries( } elif kind == "electromobility": if len(flexible_cps) > 0: + # Align to the active timeindex explicitly (see + # _build_electromobility for the same fix and its rationale) - + # also guards against a length mismatch with + # pm["time_series"]["num_steps"] (set from len(psa_net.snapshots) + # independently of flexibility_bands' own length). + timeindex = edisgo_obj.timeseries.timeindex p_set = ( - edisgo_obj.electromobility.flexibility_bands["upper_power"][ - flexible_cps + edisgo_obj.electromobility.flexibility_bands["upper_power"].loc[ + timeindex, flexible_cps ] / s_base ).round(20) e_min = ( - edisgo_obj.electromobility.flexibility_bands["lower_energy"][ - flexible_cps + edisgo_obj.electromobility.flexibility_bands["lower_energy"].loc[ + timeindex, flexible_cps ] / s_base ).round(20) e_max = ( - edisgo_obj.electromobility.flexibility_bands["upper_energy"][ - flexible_cps + edisgo_obj.electromobility.flexibility_bands["upper_energy"].loc[ + timeindex, flexible_cps ] / s_base ).round(20) diff --git a/tests/io/test_powermodels_io.py b/tests/io/test_powermodels_io.py index b3bfab036..556e7bdd0 100644 --- a/tests/io/test_powermodels_io.py +++ b/tests/io/test_powermodels_io.py @@ -304,6 +304,103 @@ def test_to_powermodels(self): ) ) + def test_to_powermodels_flexibility_bands_wider_than_timeindex(self): + """ + Regression test for eDisGo#718: _build_electromobility and + _build_component_timeseries used to read flexibility_bands + positionally (.iloc[0], .values.tolist() on the whole column) + instead of aligning to edisgo.timeseries.timeindex first. When + flexibility_bands spans more rows than the active timeindex (e.g. + stale data from before a later select_timesteps step), this used to + silently take the wrong/misaligned static p_max/e_min/e_max and + write a longer time series than pm["time_series"]["num_steps"] - + both must now be exactly scoped to the active timeindex. + """ + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) + edisgo.set_time_series_worst_case_analysis() + timeindex = edisgo.timeseries.timeindex + + edisgo.add_component( + comp_type="load", + type="charging_point", + ts_active_power=pd.Series(index=timeindex, data=[0.5] * 4), + ts_reactive_power="default", + bus=edisgo.topology.buses_df.index[32], + p_set=3, + ) + + # flexibility_bands spans 8 steps, twice the active 4-step timeindex, + # with distinctive values so misalignment is obvious + wide_index = pd.date_range(timeindex[0], periods=8, freq=timeindex.freq) + edisgo.electromobility.flexibility_bands = { + "lower_energy": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [0.0] * 8}, index=wide_index + ), + "upper_energy": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [9.0, 1, 2, 3, 4, 5, 6, 7]}, + index=wide_index, + ), + "upper_power": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [9.0, 1, 2, 3, 4, 5, 6, 7]}, + index=wide_index, + ), + } + + pm, _ = powermodels_io.to_powermodels( + edisgo, flexible_cps=["Charging_Point_LVGrid_6_1"] + ) + + num_steps = pm["time_series"]["num_steps"] + assert num_steps == len(timeindex) + for key in ("p_max", "e_min", "e_max"): + assert len(pm["time_series"]["electromobility"]["1"][key]) == num_steps + assert pm["time_series"]["electromobility"]["1"]["p_max"] == [ + 9.0, + 1.0, + 2.0, + 3.0, + ] + assert pm["electromobility"]["1"]["p_max"] == pytest.approx(9.0) + + def test_to_powermodels_flexibility_bands_wrong_calendar_raises(self): + """ + Regression test for eDisGo#718: when flexibility_bands doesn't cover + edisgo.timeseries.timeindex at all (genuine staleness, not just a + wider/narrower matching-calendar range), to_powermodels must raise a + clear KeyError rather than silently building wrong OPF input from + mismatched rows. + """ + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) + edisgo.set_time_series_worst_case_analysis() + timeindex = edisgo.timeseries.timeindex + + edisgo.add_component( + comp_type="load", + type="charging_point", + ts_active_power=pd.Series(index=timeindex, data=[0.5] * 4), + ts_reactive_power="default", + bus=edisgo.topology.buses_df.index[32], + p_set=3, + ) + + wrong_index = pd.date_range("2035-01-01", periods=4, freq=timeindex.freq) + edisgo.electromobility.flexibility_bands = { + "lower_energy": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [0.0] * 4}, index=wrong_index + ), + "upper_energy": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [1.0] * 4}, index=wrong_index + ), + "upper_power": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [1.0] * 4}, index=wrong_index + ), + } + + with pytest.raises(KeyError): + powermodels_io.to_powermodels( + edisgo, flexible_cps=["Charging_Point_LVGrid_6_1"] + ) + def test__get_pf(self): self.edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) self.edisgo.set_time_series_worst_case_analysis() From e4e64d88de5617d4ce8f8746ee61e787783ab490 Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Wed, 22 Jul 2026 16:18:07 +0200 Subject: [PATCH 62/66] fix: preserve gaps in the timeindex when resampling (#721) The shared tools.resample() function called pandas' own .resample() directly on each attribute's data, which always produces a contiguous bucket sequence spanning the data's own min-to-max timestamp - any gap in a legitimately gapped timeindex (e.g. as produced by select_timesteps in auto mode, which deliberately keeps two disjoint intervals separate) got silently filled with resample artifacts (forward-filled/averaged copies of adjacent real values) rather than preserved. TimeSeries.resample compounded this: after calling the shared function, it also rebuilt self._timeindex via one pd.date_range(first, last, freq) span, discarding the gap from the index itself, not just the data. Add split_into_contiguous_runs, splitting a DataFrame into its maximal contiguous runs (a run boundary is any gap larger than the original frequency). The shared resample() function now resamples each run independently and concatenates the results, so a gap is never bridged. TimeSeries.resample's own _timeindex rebuild now reconstructs the new index per contiguous run and unions them back together, fixing the compounding issue on top. Affects every caller of the shared function: TimeSeries.resample, HeatPump.resample_timeseries, DSM.resample, OverlyingGrid.resample. Electromobility.resample is unaffected - it already warns on non-continuous input rather than silently fabricating data. Found during the #703 PR series (time-series writers not uniformly respecting a pre-existing timeindex), tracked separately since it's a distinct pre-existing bug in shared infrastructure, not part of #703's own scope. --- edisgo/network/dsm.py | 5 ++ edisgo/network/heat.py | 5 ++ edisgo/network/overlying_grid.py | 6 ++ edisgo/network/timeseries.py | 42 ++++++--- edisgo/tools/tools.py | 125 +++++++++++++++++---------- tests/network/test_dsm.py | 16 ++++ tests/network/test_heat.py | 17 ++++ tests/network/test_overlying_grid.py | 18 ++++ tests/network/test_timeseries.py | 24 +++++ 9 files changed, 198 insertions(+), 60 deletions(-) diff --git a/edisgo/network/dsm.py b/edisgo/network/dsm.py index 88065c7eb..2b9c97838 100644 --- a/edisgo/network/dsm.py +++ b/edisgo/network/dsm.py @@ -168,6 +168,11 @@ def resample(self, method: str = "ffill", freq: str | pd.Timedelta = "15min"): freq : str, optional See :attr:`~.EDisGo.resample_timeseries` for more information. + Notes + ----- + A gapped index is resampled per contiguous run - see + :func:`edisgo.tools.tools.split_into_contiguous_runs`. + """ for attr in self._attributes: attr_index = getattr(self, attr).index diff --git a/edisgo/network/heat.py b/edisgo/network/heat.py index 414a1b5b3..624a2ac48 100644 --- a/edisgo/network/heat.py +++ b/edisgo/network/heat.py @@ -602,6 +602,11 @@ def resample_timeseries( freq : str, optional See :attr:`~.EDisGo.resample_timeseries` for more information. + Notes + ----- + A gapped index is resampled per contiguous run - see + :func:`edisgo.tools.tools.split_into_contiguous_runs`. + """ for attr in self._timeseries_attributes: attr_index = getattr(self, attr).index diff --git a/edisgo/network/overlying_grid.py b/edisgo/network/overlying_grid.py index bc49530ee..1eb5fd744 100644 --- a/edisgo/network/overlying_grid.py +++ b/edisgo/network/overlying_grid.py @@ -106,6 +106,7 @@ def __init__(self, **kwargs): self.renewables_potential = kwargs.get( "renewables_potential", pd.Series(dtype="float64") ) + @property def _attributes(self): return [ @@ -270,6 +271,11 @@ def resample(self, method: str = "ffill", freq: str | pd.Timedelta = "15min"): freq : str, optional See :attr:`~.EDisGo.resample_timeseries` for more information. + Notes + ----- + A gapped index is resampled per contiguous run - see + :func:`edisgo.tools.tools.split_into_contiguous_runs`. + """ # get frequency of time series data timeindex = [] diff --git a/edisgo/network/timeseries.py b/edisgo/network/timeseries.py index eeceb2d9a..f07ebfd22 100644 --- a/edisgo/network/timeseries.py +++ b/edisgo/network/timeseries.py @@ -27,6 +27,7 @@ assign_voltage_level_to_component, check_timeindex_coverage, resample, + split_into_contiguous_runs, ) if TYPE_CHECKING: @@ -2325,20 +2326,33 @@ def resample(self, method: str = "ffill", freq: str | pd.Timedelta = "15min"): resample(self, freq_orig, method, freq) - # create new index - if pd.Timedelta(freq) < freq_orig: # up-sampling - index = pd.date_range( - self.timeindex[0], - self.timeindex[-1] + freq_orig, - freq=freq, - inclusive="left", - ) - else: # down-sampling - index = pd.date_range( - self.timeindex[0], - self.timeindex[-1], - freq=freq, - ) + # Rebuild the new index per contiguous run of the original timeindex + # (mirroring how `resample()` above already resamples the data + # per-run) and union them back together, so a gap in the original + # timeindex (e.g. from `select_timesteps` in auto mode) is preserved + # here too, rather than bridged by one date_range(first, last, freq) + # span. + freq_td = pd.Timedelta(freq) + new_indices = [] + for run in split_into_contiguous_runs( + pd.DataFrame(index=self.timeindex), freq_orig + ): + if freq_td < freq_orig: # up-sampling + new_indices.append( + pd.date_range( + run.index[0], + run.index[-1] + freq_orig, + freq=freq, + inclusive="left", + ) + ) + else: # down-sampling + new_indices.append( + pd.date_range(run.index[0], run.index[-1], freq=freq) + ) + index = new_indices[0] + for other in new_indices[1:]: + index = index.union(other) # set new timeindex self._timeindex = index diff --git a/edisgo/tools/tools.py b/edisgo/tools/tools.py index 8a208cd56..6d3ed1768 100644 --- a/edisgo/tools/tools.py +++ b/edisgo/tools/tools.py @@ -1367,6 +1367,51 @@ def reduce_timeseries_data_to_given_timeindex( ) +def split_into_contiguous_runs(df, freq_orig): + """ + Splits a DataFrame with a (possibly gapped) `DatetimeIndex` into its + maximal contiguous runs. + + A run boundary is any gap between consecutive index entries strictly + larger than `freq_orig`. Used by :func:`resample` so that resampling a + gapped timeindex (e.g. as produced by ``select_timesteps`` in auto mode, + which deliberately keeps two disjoint intervals separate) resamples each + contiguous block independently, rather than silently bridging the gap + with resample artifacts (pandas' own `.resample()` always buckets + contiguously across whatever span the data's index covers, filling any + gap with forward-filled/averaged data rather than leaving it empty). + + Parameters + ---------- + df : :pandas:`pandas.DataFrame` + DataFrame with a :pandas:`pandas.DatetimeIndex`. + Assumed non-empty and sorted. + freq_orig : :pandas:`pandas.Timedelta` + Frequency of the original time series data. Any gap larger than this + is treated as a run boundary. + + Returns + ------- + list(:pandas:`pandas.DataFrame`) + The contiguous runs, in order. A continuous `df` returns a + single-element list containing `df` itself unchanged. + + """ + if len(df.index) < 2: + return [df] + gaps = df.index.to_series().diff().iloc[1:] + run_boundaries = np.flatnonzero((gaps > freq_orig).to_numpy()) + 1 + if len(run_boundaries) == 0: + return [df] + return [ + df.iloc[start:end] + for start, end in zip( + [0, *run_boundaries.tolist()], + [*run_boundaries.tolist(), len(df.index)], + ) + ] + + def resample( object, freq_orig, @@ -1396,58 +1441,46 @@ def resample( List of attributes to resample. Per default, all attributes specified in respective object's `_attributes` are resampled. + Notes + ----- + A gapped index (e.g. as produced by ``select_timesteps`` in auto mode) is + resampled per contiguous run (see :func:`split_into_contiguous_runs`), so + a gap is preserved rather than silently bridged with resample artifacts. + """ if attr_to_resample is None: attr_to_resample = object._attributes + freq_orig = pd.Timedelta(freq_orig) + freq = pd.Timedelta(freq) if not isinstance(freq, pd.Timedelta) else freq + up_sampling = freq < freq_orig + + if method not in ("interpolate", "ffill", "bfill"): + raise NotImplementedError(f"Resampling method {method} is not implemented.") - # add time step at the end of the time series in case of up-sampling so that - # last time interval in the original time series is still included - df_dict = {} for attr in attr_to_resample: - if not getattr(object, attr).empty: - df_dict[attr] = getattr(object, attr) - if pd.Timedelta(freq) < freq_orig: # up-sampling - new_dates = pd.DatetimeIndex([df_dict[attr].index[-1] + freq_orig]) - else: # down-sampling - new_dates = pd.DatetimeIndex([df_dict[attr].index[-1]]) - df_dict[attr] = ( - df_dict[attr] - .reindex(df_dict[attr].index.union(new_dates).unique().sort_values()) - .ffill() - ) + df = getattr(object, attr) + if df.empty: + continue + + resampled_runs = [] + for run in split_into_contiguous_runs(df, freq_orig): + # add time step at the end of the run in case of up-sampling so + # that the last time interval in the run is still included + if up_sampling: + new_dates = pd.DatetimeIndex([run.index[-1] + freq_orig]) + else: + new_dates = pd.DatetimeIndex([run.index[-1]]) + run = run.reindex(run.index.union(new_dates).unique().sort_values()).ffill() - # resample time series - if pd.Timedelta(freq) < freq_orig: # up-sampling - if method == "interpolate": - for attr in df_dict.keys(): - setattr( - object, - attr, - df_dict[attr].resample(freq, closed="left").interpolate().iloc[:-1], - ) - elif method == "ffill": - for attr in df_dict.keys(): - setattr( - object, - attr, - df_dict[attr].resample(freq, closed="left").ffill().iloc[:-1], - ) - elif method == "bfill": - for attr in df_dict.keys(): - setattr( - object, - attr, - df_dict[attr].resample(freq, closed="left").bfill().iloc[:-1], - ) - else: - raise NotImplementedError(f"Resampling method {method} is not implemented.") - else: # down-sampling - for attr in df_dict.keys(): - setattr( - object, - attr, - df_dict[attr].resample(freq).mean(), - ) + if up_sampling: + resampled = getattr(run.resample(freq, closed="left"), method)().iloc[ + :-1 + ] + else: + resampled = run.resample(freq).mean() + resampled_runs.append(resampled) + + setattr(object, attr, pd.concat(resampled_runs)) def reduce_memory_usage(df: pd.DataFrame, show_reduction: bool = False) -> pd.DataFrame: diff --git a/tests/network/test_dsm.py b/tests/network/test_dsm.py index 087de2071..b9981b26c 100644 --- a/tests/network/test_dsm.py +++ b/tests/network/test_dsm.py @@ -93,6 +93,22 @@ def test_resample(self): self.dsm.e_max = pd.DataFrame() self.dsm.resample() + def test_resample_preserves_gapped_index(self): + """ + Regression test: resampling a gapped index must not bridge the gap + with resample artifacts. + """ + gapped_index = pd.date_range("2035-01-08", periods=24, freq="h").union( + pd.date_range("2035-06-10", periods=24, freq="h") + ) + dsm = DSM() + dsm.p_max = pd.DataFrame({"load_1": [5.0] * 48}, index=gapped_index) + + dsm.resample(freq="15min") + gap = dsm.p_max.index.to_series().diff().max() + assert gap > pd.Timedelta("15min") + assert len(dsm.p_max) == 192 # 2 runs * 24h * 4 (15min steps/h) + def test_to_csv(self): # test with default values save_dir = os.path.join(os.getcwd(), "dsm_csv") diff --git a/tests/network/test_heat.py b/tests/network/test_heat.py index 6ed384bef..aabebdfbd 100644 --- a/tests/network/test_heat.py +++ b/tests/network/test_heat.py @@ -363,6 +363,23 @@ def test_resample_timeseries(self): assert len(heatpump.heat_demand_df) == 2 assert len(heatpump.cop_df) == 2 + def test_resample_timeseries_preserves_gapped_index(self): + """ + Regression test: resampling a gapped index must not bridge the gap + with resample artifacts (pandas' own .resample() would otherwise + fabricate contiguous data across it). + """ + gapped_index = pd.date_range("2035-01-08", periods=24, freq="h").union( + pd.date_range("2035-06-10", periods=24, freq="h") + ) + heatpump = HeatPump() + heatpump.cop_df = pd.DataFrame({"hp1": [5.0] * 48}, index=gapped_index) + + heatpump.resample_timeseries(freq="15min") + gap = heatpump.cop_df.index.to_series().diff().max() + assert gap > pd.Timedelta("15min") + assert len(heatpump.cop_df) == 192 # 2 runs * 24h * 4 (15min steps/h) + def test_check_integrity(self, caplog): # check for empty HeatPump class heatpump = HeatPump() diff --git a/tests/network/test_overlying_grid.py b/tests/network/test_overlying_grid.py index da77ef202..fc2c32d2c 100644 --- a/tests/network/test_overlying_grid.py +++ b/tests/network/test_overlying_grid.py @@ -156,6 +156,24 @@ def test_resample(self, caplog): "Data cannot be resampled as it only contains one time step." in caplog.text ) + def test_resample_preserves_gapped_index(self): + """ + Regression test: resampling a gapped index must not bridge the gap + with resample artifacts. + """ + gapped_index = pd.date_range("2035-01-08", periods=24, freq="h").union( + pd.date_range("2035-06-10", periods=24, freq="h") + ) + overlying_grid = OverlyingGrid() + overlying_grid.feedin_district_heating = pd.DataFrame( + {"dh1": [1.4] * 48}, index=gapped_index + ) + + overlying_grid.resample(freq="15min") + gap = overlying_grid.feedin_district_heating.index.to_series().diff().max() + assert gap > pd.Timedelta("15min") + assert len(overlying_grid.feedin_district_heating) == 192 + class TestOverlyingGridFunc: @classmethod diff --git a/tests/network/test_timeseries.py b/tests/network/test_timeseries.py index ee0274cc9..274ede7c4 100644 --- a/tests/network/test_timeseries.py +++ b/tests/network/test_timeseries.py @@ -2581,6 +2581,30 @@ def test_resample(self): atol=1e-5, ) + def test_resample_preserves_gapped_timeindex(self): + """ + Regression test: resampling a gapped timeindex (as produced by + select_timesteps in auto mode, which deliberately keeps two disjoint + intervals separate) must not bridge the gap with resample artifacts - + the gap must survive an up-sample/down-sample round-trip, and the + timeindex must be restored exactly. + """ + gapped_index = pd.date_range("2035-01-08", periods=24, freq="h").union( + pd.date_range("2035-06-10", periods=24, freq="h") + ) + self.edisgo.set_timeindex(gapped_index) + gen = self.edisgo.topology.generators_df.index[0] + self.edisgo.set_time_series_manual( + generators_p=pd.DataFrame({gen: [0.1] * 48}, index=gapped_index) + ) + + self.edisgo.timeseries.resample(freq="15min") + gap = self.edisgo.timeseries.timeindex.to_series().diff().max() + assert gap > pd.Timedelta("15min") + + self.edisgo.timeseries.resample(freq="1h") + assert_index_equal(self.edisgo.timeseries.timeindex, gapped_index) + def test_scale_timeseries(self): self.edisgo.set_time_series_worst_case_analysis() edisgo_scaled = copy.deepcopy(self.edisgo) From 6f8d00e3e4901fcf0b73d4562057212ae5b5bbce Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Fri, 24 Jul 2026 10:44:58 +0200 Subject: [PATCH 63/66] fix: replace residual_load tiling with active-timeindex proration in residual charging strategy (#726) Fixes #724. residual previously tiled (cyclically repeated) residual_load to cover charging events beyond the active eDisGo timeindex, ranking timesteps against a fabricated, non-periodic-in-reality signal. Now scopes to the active timeindex: fully out-of-window events are dropped, boundary-straddling events are prorated by in-window parking-time fraction, and both the residual_dumb sub-bucket and the argpartition ranking correctly exclude out-of-window steps even when a single event's window spans a gap in a non-contiguous active timeindex. --- edisgo/flex_opt/charging_strategies.py | 155 ++++++++++++--- tests/flex_opt/test_charging_strategy.py | 242 +++++++++++++++++++++++ 2 files changed, 371 insertions(+), 26 deletions(-) diff --git a/edisgo/flex_opt/charging_strategies.py b/edisgo/flex_opt/charging_strategies.py index b9829412a..706400288 100644 --- a/edisgo/flex_opt/charging_strategies.py +++ b/edisgo/flex_opt/charging_strategies.py @@ -286,35 +286,118 @@ def charging_strategy( eta_cp=eta_cp, ) - # get residual load - init_residual_load = edisgo_obj.timeseries.residual_load - len_residual_load = int(charging_processes_df.park_end_timesteps.max()) - if len(init_residual_load) >= len_residual_load: - init_residual_load = init_residual_load.loc[timeindex] + if not resample: + # The active timeindex can extend past the last charging event + # (e.g. a trailing gapped run with no events in it at all) - the + # step-space array built below must cover at least as far as + # target_timeindex itself, or the crop-after-build step later + # would reindex into positions that were never built, producing + # NaN rather than a legitimate zero. + target_span_steps = int( + (target_timeindex[-1] - target_timeindex[0]) + / pd.Timedelta(f"{edisgo_obj.electromobility.stepsize}min") + ) + len_residual_load = max(len_residual_load, target_span_steps) + + # Map each SimBEV step position (0 .. len_residual_load, the same + # positional space as park_start_timesteps/park_end_timesteps) to + # whether it is present in the active timeindex (`target_timeindex`). + # Real residual_load only exists for `target_timeindex`. Rather than + # tiling (cyclically repeating) it to cover steps beyond the active + # timeindex - which would rank timesteps against a fabricated, + # non-periodic-in-reality signal (see ADR 0001) - steps outside the + # active timeindex are simply marked as having no usable data. + # `resample=True` means `target_timeindex` predates an internal + # frequency round-trip and is no longer in the same step space as + # `park_start_timesteps` - the crop-after-build step already skips + # trimming in that case (see the module docstring), so this reduction + # is skipped here too and every step is treated as in-window, + # preserving today's (tiling) behavior only for that known + # limitation. + if resample: + step_in_window = np.ones(len_residual_load + 1, dtype=bool) else: - while len(init_residual_load) < len_residual_load: - len_rl = len(init_residual_load) - len_append = min(len_rl, len_residual_load - len_rl) - - s_append = init_residual_load.iloc[:len_append] - - init_residual_load = pd.concat( - [ - init_residual_load, - s_append, - ], - ignore_index=True, - ) + step_in_window = np.isin( + pd.date_range( + target_timeindex[0], + periods=len_residual_load + 1, + freq=f"{edisgo_obj.electromobility.stepsize}min", + ), + target_timeindex, + ) + in_window_steps_cumsum = np.concatenate(([0], np.cumsum(step_in_window))) + + if not resample: + # Events are reduced to the active timeindex before being + # scheduled: fully in-window events are untouched, fully + # out-of-window events are dropped, and boundary-straddling + # events have their charging demand prorated by how much of + # their parking time is actually observable. This mirrors + # `harmonize_charging_processes_df`'s own derivation of + # `minimum_charging_time` from demand and nominal power. + parking_time = ( + charging_processes_df.park_end_timesteps + - charging_processes_df.park_start_timesteps + + 1 + ) + overlap_steps = ( + in_window_steps_cumsum[ + charging_processes_df.park_end_timesteps.to_numpy() + 1 + ] + - in_window_steps_cumsum[ + charging_processes_df.park_start_timesteps.to_numpy() + ] + ) + + # drop events with zero overlap - nothing to schedule, no + # residual_load data exists for them at all + in_window = overlap_steps > 0 + charging_processes_df = charging_processes_df.loc[in_window] + in_window_fraction = ( + (overlap_steps[in_window]) / (parking_time.to_numpy()[in_window]) + ) + + scaled_demand_kWh = ( + charging_processes_df.harmonized_chargingdemand * in_window_fraction + ) + scaled_minimum_charging_time = ( + scaled_demand_kWh + / charging_processes_df.nominal_charging_capacity_kW + * 60 + / edisgo_obj.electromobility.stepsize + ) + scaled_minimum_charging_time = np.ceil(scaled_minimum_charging_time).astype( + np.uint16 + ) + + # defensive clamp: proration preserves + # minimum_charging_time <= parking_time, so this should only ever + # bind on pre-existing anomalous input (an event whose full, + # unscaled demand already didn't fit its own parking time) + scaled_minimum_charging_time = np.minimum( + scaled_minimum_charging_time, overlap_steps[in_window] + ) - init_residual_load = init_residual_load.to_numpy() + charging_processes_df = charging_processes_df.assign( + minimum_charging_time=scaled_minimum_charging_time, + flex_time=charging_processes_df.park_time_timesteps + - scaled_minimum_charging_time, + ) + + # get residual load; steps outside the active timeindex carry no + # real data (see above) and are set to NaN so they can never be + # selected as charging candidates below + init_residual_load = edisgo_obj.timeseries.residual_load timeindex_residual = pd.date_range( edisgo_obj.timeseries.timeindex[0], - periods=len(init_residual_load), + periods=len_residual_load + 1, freq=f"{edisgo_obj.electromobility.stepsize}min", ) + init_residual_load = init_residual_load.reindex(timeindex_residual).to_numpy() + init_residual_load[~step_in_window] = np.nan dummy_ts = pd.DataFrame( data=0.0, columns=[_.id for _ in charging_parks], index=timeindex_residual @@ -335,7 +418,15 @@ def charging_strategy( RELEVANT_CHARGING_STRATEGIES_COLUMNS["residual_dumb"] ].itertuples(): try: - dummy_ts.loc[:, cp_id].iloc[start : start + stop] += cap + # Write only to in-window positions of the deterministic + # charging interval [start, start+stop) - if the active + # timeindex has a gap inside this interval, every in-window + # sub-slice still gets the event's full, unscaled power (see + # ADR 0002); out-of-window positions are simply not written. + in_window_idx = ( + np.flatnonzero(step_in_window[start : start + stop]) + start + ) + dummy_ts.loc[:, cp_id].iloc[in_window_idx] += cap except Exception: maximum_ts = len(dummy_ts) @@ -351,11 +442,23 @@ def charging_strategy( for _, start, end, k, cp_id, cap in flex_charging_processes_df[ RELEVANT_CHARGING_STRATEGIES_COLUMNS["residual"] ].itertuples(): - flex_band = residual_load[start : end + 1] - - # get k time steps with the lowest residual load in the parking - # time - idx = np.argpartition(flex_band, k)[:k] + start + # Restrict ranking candidates to timesteps that are both within + # the parking window and present in the active timeindex - + # `residual_load` is NaN outside the active timeindex (no real + # data exists there, see above), so those positions must never + # be selected, even if the parking window itself spans a gap. + candidates = np.flatnonzero(step_in_window[start : end + 1]) + start + + if k >= len(candidates): + # k charging demand may (after proration/clamping) exactly + # saturate the available in-window candidates - nothing left + # to rank, every candidate is used. + idx = candidates + else: + flex_band = residual_load[candidates] + # get k time steps with the lowest residual load in the + # parking time, among the valid (in-window) candidates only + idx = candidates[np.argpartition(flex_band, k)[:k]] try: dummy_ts[cp_id].iloc[idx] += cap diff --git a/tests/flex_opt/test_charging_strategy.py b/tests/flex_opt/test_charging_strategy.py index 96dc85b4e..13d369293 100644 --- a/tests/flex_opt/test_charging_strategy.py +++ b/tests/flex_opt/test_charging_strategy.py @@ -154,6 +154,248 @@ def test_charging_strategy_trims_to_gapped_timeindex(self): edisgo.timeseries._loads_active_power.index, gapped_timeindex ) + def _setup_edisgo_with_single_synthetic_event( + self, timeindex, park_start_timesteps, park_end_timesteps, chargingdemand_kWh + ): + """ + Helper for the ADR 0001 regression tests below: imports the real + SimBEV/TracBEV fixture (so charging park integration/topology wiring + is realistic), then overwrites charging_processes_df with a single, + fully controlled synthetic event on one of the fixture's own + integrated charging parks, reusing that park's own use_case/capacity + so `harmonize_charging_processes_df` sees realistic values. + """ + edisgo = EDisGo(ding0_grid=self.ding0_path) + edisgo.set_timeindex(timeindex) + edisgo.import_electromobility( + data_source="directory", + charging_processes_dir=self.simbev_path, + potential_charging_points_dir=self.tracbev_path, + ) + integrated = edisgo.electromobility.integrated_charging_parks_df + park_id = integrated.index[0] + template = edisgo.electromobility.charging_processes_df[ + edisgo.electromobility.charging_processes_df.charging_park_id == park_id + ].iloc[0] + + park_time_timesteps = park_end_timesteps - park_start_timesteps + 1 + edisgo.electromobility.charging_processes_df = pd.DataFrame( + { + "ags": [template.ags], + "car_id": [0], + "destination": [template.destination], + # avoid public/hpc (always dumb-charged even under + # "residual") so these events exercise the flex-ranking path + "use_case": ["work"], + "nominal_charging_capacity_kW": [template.nominal_charging_capacity_kW], + "grid_charging_capacity_kW": [template.grid_charging_capacity_kW], + "chargingdemand_kWh": [chargingdemand_kWh], + "park_time_timesteps": [park_time_timesteps], + "park_start_timesteps": [park_start_timesteps], + "park_end_timesteps": [park_end_timesteps], + "charging_park_id": [park_id], + "charging_point_id": [template.charging_point_id], + } + ) + return edisgo, park_id + + def test_residual_drops_fully_out_of_window_events(self): + """ + Regression test for ADR 0001: a charging event whose parking window + has zero overlap with the active timeindex must contribute nothing - + not be tiled/fabricated against repeated residual_load data. + """ + timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") # 1 day + # park window entirely on day 3, well beyond the 1-day active window + edisgo, park_id = self._setup_edisgo_with_single_synthetic_event( + timeindex, + park_start_timesteps=300, + park_end_timesteps=320, + chargingdemand_kWh=10.0, + ) + + charging_strategy(edisgo, strategy="residual") + + edisgo_id = edisgo.electromobility.integrated_charging_parks_df.at[ + park_id, "edisgo_id" + ] + written = edisgo.timeseries.loads_active_power[edisgo_id] + assert (written == 0).all() + + def test_residual_prorates_boundary_straddling_event(self): + """ + Regression test for ADR 0001: a charging event whose parking window + straddles the active timeindex boundary must have its charging + demand prorated by the in-window fraction of its parking time, not + fully charged (which would require tiled/fabricated residual_load + data beyond the active timeindex) and not dropped. + """ + timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") # steps 0-95 + # park window [90, 149] (60 steps), only steps 90-95 (6 steps) are + # in-window -> in_window_fraction = 6/60 = 0.1 + edisgo, park_id = self._setup_edisgo_with_single_synthetic_event( + timeindex, + park_start_timesteps=90, + park_end_timesteps=149, + chargingdemand_kWh=12.0, + ) + edisgo_id = edisgo.electromobility.integrated_charging_parks_df.at[ + park_id, "edisgo_id" + ] + + charging_strategy(edisgo, strategy="residual") + + written = edisgo.timeseries.loads_active_power[edisgo_id] + + # only in-window steps (90-95) may carry any charging + assert (written.iloc[:90] == 0).all() + assert written.iloc[90:].sum() > 0 + + # compare against the same event fully inside a timeindex covering + # its whole parking window - the straddling case must deliver + # strictly less energy than the fully-observable case, since only + # 1/10th of its parking time is actually in-window here + full_timeindex = pd.date_range("1/1/2011", periods=150, freq="15min") + edisgo_full, park_id_full = self._setup_edisgo_with_single_synthetic_event( + full_timeindex, + park_start_timesteps=90, + park_end_timesteps=149, + chargingdemand_kWh=12.0, + ) + charging_strategy(edisgo_full, strategy="residual") + edisgo_id_full = edisgo_full.electromobility.integrated_charging_parks_df.at[ + park_id_full, "edisgo_id" + ] + full_energy = edisgo_full.timeseries.loads_active_power[edisgo_id_full].sum() + + straddling_energy = written.sum() + assert 0 < straddling_energy < full_energy + + def test_residual_fully_inside_event_unaffected(self): + """ + Regression test for ADR 0001: an event whose parking window is + entirely inside the active timeindex must be scheduled exactly the + same regardless of how much longer the active timeindex extends + beyond the event's own window - proration must not affect + fully-observable events at all. + """ + edisgo_short, park_id_short = self._setup_edisgo_with_single_synthetic_event( + pd.date_range("1/1/2011", periods=60, freq="15min"), + park_start_timesteps=10, + park_end_timesteps=50, + chargingdemand_kWh=8.0, + ) + edisgo_long, park_id_long = self._setup_edisgo_with_single_synthetic_event( + pd.date_range("1/1/2011", periods=200, freq="15min"), + park_start_timesteps=10, + park_end_timesteps=50, + chargingdemand_kWh=8.0, + ) + + charging_strategy(edisgo_short, strategy="residual") + charging_strategy(edisgo_long, strategy="residual") + + edisgo_id_short = edisgo_short.electromobility.integrated_charging_parks_df.at[ + park_id_short, "edisgo_id" + ] + edisgo_id_long = edisgo_long.electromobility.integrated_charging_parks_df.at[ + park_id_long, "edisgo_id" + ] + energy_short = edisgo_short.timeseries.loads_active_power[edisgo_id_short].sum() + energy_long = edisgo_long.timeseries.loads_active_power[edisgo_id_long].sum() + + assert energy_short > 0 + assert energy_short == pytest.approx(energy_long) + + def test_residual_no_tiling_across_gapped_timeindex(self): + """ + Regression test for ADR 0001: with a gapped active timeindex, an + event that overlaps both disjoint runs must only ever be scheduled + into steps actually present in the active timeindex - never into the + gap, and never against fabricated/tiled residual_load data. + """ + # two disjoint 15-min runs: steps 0-23 and steps 100-123 (gap of 76 + # steps in between, well beyond any real residual_load coverage) + run_1 = pd.date_range("1/1/2011", periods=24, freq="15min") + run_2 = pd.date_range("1/1/2011", periods=24, freq="15min") + pd.Timedelta( + minutes=15 * 100 + ) + gapped_timeindex = run_1.union(run_2) + + edisgo, park_id = self._setup_edisgo_with_single_synthetic_event( + gapped_timeindex, + park_start_timesteps=10, + park_end_timesteps=110, + chargingdemand_kWh=6.0, + ) + + charging_strategy(edisgo, strategy="residual") + + edisgo_id = edisgo.electromobility.integrated_charging_parks_df.at[ + park_id, "edisgo_id" + ] + written = edisgo.timeseries.loads_active_power[edisgo_id] + + # written series must exactly match the gapped index - nothing + # fabricated to bridge the gap + pd.testing.assert_index_equal(written.index, gapped_timeindex) + assert written.sum() > 0 + + def test_residual_dumb_subbucket_respects_internal_gap(self): + """ + Regression test for ADR 0001: a "dumb-charged" event within the + residual strategy (use_case in {public, hpc} or flex_time == 0) + whose deterministic charging interval spans a gap in the active + timeindex must only ever write to in-window positions - never a + blind contiguous slice bridging the gap. + """ + run_1 = pd.date_range("1/1/2011", periods=10, freq="15min") + run_2 = pd.date_range("1/1/2011", periods=10, freq="15min") + pd.Timedelta( + minutes=15 * 20 + ) + gapped_timeindex = run_1.union(run_2) + + edisgo = EDisGo(ding0_grid=self.ding0_path) + edisgo.set_timeindex(gapped_timeindex) + edisgo.import_electromobility( + data_source="directory", + charging_processes_dir=self.simbev_path, + potential_charging_points_dir=self.tracbev_path, + ) + integrated = edisgo.electromobility.integrated_charging_parks_df + park_id = integrated.index[0] + template = edisgo.electromobility.charging_processes_df[ + edisgo.electromobility.charging_processes_df.charging_park_id == park_id + ].iloc[0] + + # public use_case -> always dumb-charged even under "residual"; + # park window [5, 24] straddles the gap (steps 10-19) + edisgo.electromobility.charging_processes_df = pd.DataFrame( + { + "ags": [template.ags], + "car_id": [0], + "destination": [template.destination], + "use_case": ["public"], + "nominal_charging_capacity_kW": [template.nominal_charging_capacity_kW], + "grid_charging_capacity_kW": [template.grid_charging_capacity_kW], + "chargingdemand_kWh": [2.0], + "park_time_timesteps": [20], + "park_start_timesteps": [5], + "park_end_timesteps": [24], + "charging_park_id": [park_id], + "charging_point_id": [template.charging_point_id], + } + ) + + charging_strategy(edisgo, strategy="residual") + + edisgo_id = integrated.at[park_id, "edisgo_id"] + written = edisgo.timeseries.loads_active_power[edisgo_id] + + pd.testing.assert_index_equal(written.index, gapped_timeindex) + # no NaNs, no crash from writing into a position that doesn't exist + assert not written.isna().any() + def test_charging_strategy_with_subset_of_parks(self): """ Charging strategies can be applied to different subsets of charging parks From c48af1bf862476ec508c41b3b090bd8a3c7d80a1 Mon Sep 17 00:00:00 2001 From: MoritzSchloesser Date: Mon, 27 Jul 2026 13:18:48 +0200 Subject: [PATCH 64/66] fix: reduce dumb/reduced/flexibility_bands to the active timeindex before building (#727) Part of #703 follow-up (ADR 0002). dumb/reduced previously wrote each charging event's deterministic interval unconditionally into a full-SimBEV-length series, relying solely on a later crop step to trim to the active timeindex. Now clips the placement slice itself to whatever overlaps the active timeindex, at unchanged power - correctly handling a gap inside a single event's interval by writing each in-window sub-slice independently. get_flexibility_bands had the same underlying issue: its output values were already correct after the existing final .loc[timeindex] clip, but construction itself built over EVERY charging process and SimBEV's entire simulated range regardless of how much shorter the active timeindex was - real "build full, then crop" waste, just hidden behind correct output. Now filters to events overlapping the active timeindex first and sizes the construction array to only what those events plus the active window require, while avoiding a pre-existing edge case (an event's true end landing exactly on the array's last row gets silently excluded) that the tighter sizing would otherwise trigger far more often. See docs/adr/0002-charging-time-series-reduced-to-active-timeindex-before-building.md. --- edisgo/flex_opt/charging_strategies.py | 43 +++- edisgo/network/electromobility.py | 98 ++++++++- tests/flex_opt/test_charging_strategy.py | 141 +++++++++++++ tests/network/test_electromobility.py | 247 +++++++++++++++++++++++ 4 files changed, 523 insertions(+), 6 deletions(-) diff --git a/edisgo/flex_opt/charging_strategies.py b/edisgo/flex_opt/charging_strategies.py index 706400288..6abbc3c35 100644 --- a/edisgo/flex_opt/charging_strategies.py +++ b/edisgo/flex_opt/charging_strategies.py @@ -192,6 +192,27 @@ def charging_strategy( edisgo_obj.timeseries.resample(freq=simbev_timedelta) + # Map each SimBEV step position (0 .. len_ts - 1, the same positional + # space as park_start_timesteps/the placement slices below) to whether it + # is present in the active timeindex (`target_timeindex`). `dumb` and + # `reduced` place each event's demand deterministically at + # [start, start+stop) - rather than building the full-SimBEV-length + # series unconditionally and cropping the *output* down to the active + # timeindex afterwards (as before), the placement itself is now clipped + # to whatever of that interval is actually in-window, so an event's + # reported energy is a direct consequence of which positions get + # written, not a separate proration calculation (see ADR 0002). + # `resample=True` means `target_timeindex` predates an internal + # frequency round-trip and is no longer in the same step space as + # `park_start_timesteps` - the crop-after-build step already skips + # trimming in that case (see the module docstring), so this reduction is + # skipped here too and every step is treated as in-window, preserving + # today's (build-full) behavior only for that known limitation. + if resample: + step_in_window = np.ones(len_ts, dtype=bool) + else: + step_in_window = np.isin(timeindex, target_timeindex) + if strategy == "dumb": # "dumb" charging # Collect each charging park's series and add them to the time series in a @@ -214,7 +235,15 @@ def charging_strategy( for _, start, stop, cap in charging_processes_df[ RELEVANT_CHARGING_STRATEGIES_COLUMNS["dumb"] ].itertuples(): - dummy_ts[start : start + stop] += cap + # Write only to in-window positions of the deterministic + # charging interval [start, start+stop) - if the active + # timeindex has a gap inside this interval, every in-window + # sub-slice still gets the event's full, unscaled power (see + # ADR 0002); out-of-window positions are simply not written. + in_window_idx = ( + np.flatnonzero(step_in_window[start : start + stop]) + start + ) + dummy_ts[in_window_idx] += cap cp_ts[cp.edisgo_id] = dummy_ts @@ -253,12 +282,20 @@ def charging_strategy( ) in charging_processes_df[ RELEVANT_CHARGING_STRATEGIES_COLUMNS["reduced"] ].itertuples(): + # See the "dumb" branch above for why the placement slice + # itself (not a separate energy calculation) is clipped to + # in-window positions. if use_case == "public" or use_case == "hpc": # if the charging process takes place in a "public" setting # the charging is "dumb" - dummy_ts[start : start + stop_dumb] += cap_dumb + start_, stop_, cap = start, stop_dumb, cap_dumb else: - dummy_ts[start : start + stop_reduced] += cap_reduced + start_, stop_, cap = start, stop_reduced, cap_reduced + + in_window_idx = ( + np.flatnonzero(step_in_window[start_ : start_ + stop_]) + start_ + ) + dummy_ts[in_window_idx] += cap cp_ts[cp.edisgo_id] = dummy_ts diff --git a/edisgo/network/electromobility.py b/edisgo/network/electromobility.py index 7d72f6341..55e4d1a59 100644 --- a/edisgo/network/electromobility.py +++ b/edisgo/network/electromobility.py @@ -489,6 +489,91 @@ def get_flexibility_bands( start=start_date, periods=t_max + 1, freq=f"{stepsize}min" ) + # Reduce to only the charging processes actually needed for + # edisgo_obj.timeseries.timeindex, instead of always building over + # every process regardless of how much shorter the active timeindex + # is (see ADR 0002). SimBEV's calendar (start_date) is typically a + # fixed reference year independent of the scenario year, so the + # active timeindex is inverse year-shifted onto that calendar to + # find which SimBEV steps are actually relevant - this mirrors the + # forward year-shift `align_series_to_timeindex` already applies to + # the built bands further down. Only events overlapping that window + # are kept (mirrors dumb/reduced/residual: zero overlap -> zero + # contribution, not a special case). + # + # The array itself is NOT truncated to the window's end, only + # (potentially) to the earliest relevant event's start: clamping a + # retained event's true end down to an artificial array boundary + # would push it onto the `end == n_steps - 1` edge case below (a + # pre-existing, unrelated exclusion for events ending exactly on the + # array's last row) for every boundary-straddling event, silently + # dropping their contribution entirely rather than only trimming it. + # The array is instead sized to comfortably cover every retained + # event's true end - the pre-existing final `.loc[edisgo_timeindex]` + # step below still discards whatever trailing rows aren't needed. + edisgo_timeindex = edisgo_obj.timeseries.timeindex + if len(edisgo_timeindex) > 0: + year_diff = flex_band_index[0].year - edisgo_timeindex[0].year + simbev_calendar_timeindex = edisgo_timeindex + pd.DateOffset( + years=year_diff + ) + window_start_step = int( + (simbev_calendar_timeindex.min() - flex_band_index[0]) + / pd.Timedelta(f"{stepsize}min") + ) + window_end_step = int( + (simbev_calendar_timeindex.max() - flex_band_index[0]) + / pd.Timedelta(f"{stepsize}min") + ) + + overlaps_window = ( + self.charging_processes_df.park_end_timesteps >= window_start_step + ) & (self.charging_processes_df.park_start_timesteps <= window_end_step) + relevant_processes = self.charging_processes_df.loc[overlaps_window] + + if relevant_processes.empty: + # No event overlaps the active window at all (e.g. the + # window falls entirely outside SimBEV's simulated range) - + # size the array to the window itself so the final + # `.loc[edisgo_timeindex]` step still produces the expected + # shape (filled with NaN via `align_series_to_timeindex`, + # exactly as the pre-existing full-span build already would + # have for this case), without ever touching t_max, which + # only bounds where real event data can be, not the window. + array_start_step = window_start_step + array_end_step = window_end_step + else: + array_start_step = min( + window_start_step, + int(relevant_processes.park_start_timesteps.min()), + ) + # Cover every retained event's true end (never clamped down + # below it) as well as the active window's own end. +1 extra + # step of padding mirrors the pre-existing `end_date + 1 day` + # padding on the full-span build (see above) - it exists so + # a retained event's true end never lands exactly on the + # array's last row, which would otherwise trip the + # pre-existing `end == n_steps - 1` exclusion below for + # every such event instead of just the rare full-span case + # it originally guarded against. + array_end_step = ( + max( + window_end_step, + int(relevant_processes.park_end_timesteps.max()), + ) + + 1 + ) + + flex_band_index = pd.date_range( + start=flex_band_index[0] + + pd.Timedelta(f"{array_start_step * stepsize}min"), + periods=max(array_end_step - array_start_step + 1, 0), + freq=f"{stepsize}min", + ) + else: + relevant_processes = self.charging_processes_df + array_start_step = 0 + # set up bands n_steps = len(flex_band_index) tmp_idx = range(n_steps) @@ -502,14 +587,21 @@ def get_flexibility_bands( # map every charging process to the column (charging point) it belongs to; # processes of charging points outside `cps` map to -1 and are dropped park_to_cp = self.integrated_charging_parks_df["edisgo_id"] - proc = self.charging_processes_df + proc = relevant_processes col = cps.index.get_indexer(proc["charging_park_id"].map(park_to_cp)) - end_all = proc["park_end_timesteps"].to_numpy() + # shift into the (possibly truncated) array's own step space - never + # clamped, since the array was sized to cover every retained event's + # true end above + start_all = proc["park_start_timesteps"].to_numpy() - array_start_step + end_all = proc["park_end_timesteps"].to_numpy() - array_start_step # the last time step can lead to problems --> skip those processes keep = (col >= 0) & (end_all != n_steps - 1) col = col[keep] - sub = proc.loc[keep] + sub = proc.loc[keep].assign( + park_start_timesteps=start_all[keep], + park_end_timesteps=end_all[keep], + ) start = sub["park_start_timesteps"].to_numpy().astype(int) end = sub["park_end_timesteps"].to_numpy().astype(int) power = sub["nominal_charging_capacity_kW"].to_numpy(dtype=float) diff --git a/tests/flex_opt/test_charging_strategy.py b/tests/flex_opt/test_charging_strategy.py index 13d369293..b94c7af3f 100644 --- a/tests/flex_opt/test_charging_strategy.py +++ b/tests/flex_opt/test_charging_strategy.py @@ -396,6 +396,147 @@ def test_residual_dumb_subbucket_respects_internal_gap(self): # no NaNs, no crash from writing into a position that doesn't exist assert not written.isna().any() + @pytest.mark.parametrize("strategy", ["dumb", "reduced"]) + def test_dumb_reduced_clip_placement_to_active_timeindex(self, strategy): + """ + Regression test for ADR 0002: dumb/reduced must clip their + deterministic charging placement slice to the active timeindex + instead of building the full-SimBEV-length series and relying on a + later crop - a charging interval that extends beyond the active + timeindex must only ever be written for its in-window positions, + at unchanged (unscaled) power. + """ + # active timeindex ends at step 92 (93 steps: 0-92) + timeindex = pd.date_range("1/1/2011", periods=93, freq="15min") + edisgo, park_id = self._setup_edisgo_with_single_synthetic_event( + timeindex, + park_start_timesteps=90, + park_end_timesteps=149, + chargingdemand_kWh=12.0, + ) + edisgo_id = edisgo.electromobility.integrated_charging_parks_df.at[ + park_id, "edisgo_id" + ] + + charging_strategy(edisgo, strategy=strategy) + + written = edisgo.timeseries.loads_active_power[edisgo_id] + nonzero = written[written != 0] + + # written series must match the active timeindex exactly - no extra + # rows for steps beyond it (nothing fabricated/built past the window) + pd.testing.assert_index_equal(written.index, timeindex) + # only in-window steps (<= step 92, i.e. before 1/1/2011 23:15) may + # ever carry a nonzero value + assert (nonzero.index <= timeindex[-1]).all() + assert len(nonzero) > 0 + + # compare against the same event given a timeindex long enough to + # cover its whole charging interval - the clipped case must report + # strictly less energy, since some of its charging steps fall + # outside the shorter active timeindex + long_timeindex = pd.date_range("1/1/2011", periods=150, freq="15min") + edisgo_long, park_id_long = self._setup_edisgo_with_single_synthetic_event( + long_timeindex, + park_start_timesteps=90, + park_end_timesteps=149, + chargingdemand_kWh=12.0, + ) + edisgo_id_long = edisgo_long.electromobility.integrated_charging_parks_df.at[ + park_id_long, "edisgo_id" + ] + charging_strategy(edisgo_long, strategy=strategy) + full_energy = edisgo_long.timeseries.loads_active_power[edisgo_id_long].sum() + + assert 0 < written.sum() < full_energy + + # power at each in-window step must be unchanged (no proration of + # the rate itself) - every nonzero value equals the same per-step + # power the long-timeindex (unclipped) run reports + long_nonzero_values = edisgo_long.timeseries.loads_active_power[edisgo_id_long] + long_nonzero_values = long_nonzero_values[long_nonzero_values != 0] + assert nonzero.iloc[0] == pytest.approx(long_nonzero_values.iloc[0]) + + @pytest.mark.parametrize("strategy", ["dumb", "reduced"]) + def test_dumb_reduced_fully_out_of_window_event_contributes_nothing(self, strategy): + """ + Regression test for ADR 0002: an event whose deterministic charging + interval has zero overlap with the active timeindex must contribute + nothing. + """ + timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") # 1 day + edisgo, park_id = self._setup_edisgo_with_single_synthetic_event( + timeindex, + park_start_timesteps=300, + park_end_timesteps=320, + chargingdemand_kWh=10.0, + ) + edisgo_id = edisgo.electromobility.integrated_charging_parks_df.at[ + park_id, "edisgo_id" + ] + + charging_strategy(edisgo, strategy=strategy) + + written = edisgo.timeseries.loads_active_power[edisgo_id] + assert (written == 0).all() + + @pytest.mark.parametrize("strategy", ["dumb", "reduced"]) + def test_dumb_reduced_respect_internal_gap(self, strategy): + """ + Regression test for ADR 0002: if an event's deterministic charging + interval itself spans a gap in a non-contiguous active timeindex, + every in-window sub-slice of that interval must be written + independently, at full unscaled power - never a blind contiguous + write bridging the gap. + """ + run_1 = pd.date_range("1/1/2011", periods=10, freq="15min") + run_2 = pd.date_range("1/1/2011", periods=10, freq="15min") + pd.Timedelta( + minutes=15 * 20 + ) + gapped_timeindex = run_1.union(run_2) + + edisgo = EDisGo(ding0_grid=self.ding0_path) + edisgo.set_timeindex(gapped_timeindex) + edisgo.import_electromobility( + data_source="directory", + charging_processes_dir=self.simbev_path, + potential_charging_points_dir=self.tracbev_path, + ) + integrated = edisgo.electromobility.integrated_charging_parks_df + park_id = integrated.index[0] + template = edisgo.electromobility.charging_processes_df[ + edisgo.electromobility.charging_processes_df.charging_park_id == park_id + ].iloc[0] + + # work use_case, small enough demand that minimum_charging_time (or + # reduced_charging_time) spans steps 5-24, straddling the gap + # (steps 10-19, absent from the active timeindex) + edisgo.electromobility.charging_processes_df = pd.DataFrame( + { + "ags": [template.ags], + "car_id": [0], + "destination": [template.destination], + "use_case": ["work"], + "nominal_charging_capacity_kW": [template.nominal_charging_capacity_kW], + "grid_charging_capacity_kW": [template.grid_charging_capacity_kW], + "chargingdemand_kWh": [2.0], + "park_time_timesteps": [20], + "park_start_timesteps": [5], + "park_end_timesteps": [24], + "charging_park_id": [park_id], + "charging_point_id": [template.charging_point_id], + } + ) + + charging_strategy(edisgo, strategy=strategy) + + edisgo_id = integrated.at[park_id, "edisgo_id"] + written = edisgo.timeseries.loads_active_power[edisgo_id] + + pd.testing.assert_index_equal(written.index, gapped_timeindex) + # no NaNs, no crash from writing into a position that doesn't exist + assert not written.isna().any() + def test_charging_strategy_with_subset_of_parks(self): """ Charging strategies can be applied to different subsets of charging parks diff --git a/tests/network/test_electromobility.py b/tests/network/test_electromobility.py index dfc5450ab..db78e79f5 100644 --- a/tests/network/test_electromobility.py +++ b/tests/network/test_electromobility.py @@ -217,6 +217,253 @@ def test_get_flexibility_bands_scopes_to_mismatched_timeindex(self): # must not raise KeyError edisgo_obj.electromobility.flexibility_bands[key].loc[short_timeindex] + def test_get_flexibility_bands_carries_forward_true_start_end(self): + """ + Regression test for ADR 0002: upper_energy/lower_energy must reflect + each event's TRUE park_start_timesteps/park_end_timesteps, even when + the active timeindex's window starts after (or ends before) that + true start/end - the bands must not be reset to zero / re-anchored + at the window's own edge. Verified by comparing the same event's + band value at a shared calendar timestamp, once built over the full + SimBEV span and once built over a window starting after the event's + true park_start_timesteps. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + full_timeindex = pd.date_range("1/1/2011", periods=200, freq="15min") + edisgo_obj.set_timeindex(full_timeindex) + full_bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, ["work", "public", "home", "hpc"] + ) + + # pick a charging point/timestamp where upper_energy is already + # elevated (i.e. charging started earlier and hasn't finished) - + # windowing from here on must preserve that value, not reset to 0 + upper_energy_full = full_bands["upper_energy"] + nonzero_mask = upper_energy_full > 0 + elevated_positions = nonzero_mask[nonzero_mask.any(axis=1)] + assert not elevated_positions.empty + window_start = elevated_positions.index[len(elevated_positions.index) // 2] + cp_id = elevated_positions.loc[window_start][ + elevated_positions.loc[window_start] + ].index[0] + expected_value = upper_energy_full.loc[window_start, cp_id] + assert expected_value > 0 + + edisgo_obj_windowed = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj_windowed, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj_windowed) + electromobility_import.integrate_charging_parks(edisgo_obj_windowed) + windowed_timeindex = full_timeindex[full_timeindex >= window_start] + edisgo_obj_windowed.set_timeindex(windowed_timeindex) + + windowed_bands = edisgo_obj_windowed.electromobility.get_flexibility_bands( + edisgo_obj_windowed, ["work", "public", "home", "hpc"] + ) + windowed_value = windowed_bands["upper_energy"].loc[window_start, cp_id] + + assert windowed_value == pytest.approx(expected_value) + + def test_get_flexibility_bands_reflects_partial_progress_for_unfinished_event(self): + """ + Regression test for ADR 0002: even though get_flexibility_bands now + filters out charging processes with zero overlap with the active + timeindex (see test_get_flexibility_bands_excludes_events_and_limits_ + array_size below), a RETAINED event that straddles the window + boundary must still report only the energy that charging-so-far + implies at each in-window timestep - never the event's full + (possibly not-yet-delivered) chargingdemand_kWh and never a naive + proportional share of it. upper_energy/lower_energy are cumulative + running totals of physically possible charging progress, not a + fixed total being allocated, and this must hold regardless of + whether the array is sized to SimBEV's full range or only to what's + needed. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + integrated = edisgo_obj.electromobility.integrated_charging_parks_df + park_id = integrated.index[0] + template = edisgo_obj.electromobility.charging_processes_df[ + edisgo_obj.electromobility.charging_processes_df.charging_park_id == park_id + ].iloc[0] + edisgo_id = integrated.at[park_id, "edisgo_id"] + + # event needs 60 steps of charging at 10 kW (150 kWh) to fulfil its + # demand, parked for 100 steps [50, 149] - charging is NOT finished + # by step 95 (needs steps 50-109) + edisgo_obj.electromobility.charging_processes_df = pd.DataFrame( + { + "ags": [template.ags], + "car_id": [0], + "destination": [template.destination], + "use_case": [template.use_case], + "nominal_charging_capacity_kW": [10.0], + "grid_charging_capacity_kW": [10.0], + "chargingdemand_kWh": [150.0], + "park_time_timesteps": [100], + "park_start_timesteps": [50], + "park_end_timesteps": [149], + "charging_park_id": [park_id], + "charging_point_id": [template.charging_point_id], + } + ) + + # active timeindex ends at step 95 - 46 steps into the 60-step + # charge (steps 50-95 inclusive = 46 steps) + timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") + edisgo_obj.set_timeindex(timeindex) + + bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, [template.use_case] + ) + upper_energy_at_cutoff = bands["upper_energy"][edisgo_id].iloc[-1] + + steps_charged_by_cutoff = 46 + expected_kWh = steps_charged_by_cutoff * 10.0 / 4 # 10 kW, 15-min steps + full_demand_kWh = 150.0 + proportional_share_kWh = full_demand_kWh * steps_charged_by_cutoff / 60 + + assert upper_energy_at_cutoff * 1e3 == pytest.approx(expected_kWh) + # must not be the full (not-yet-delivered) demand ... + assert upper_energy_at_cutoff * 1e3 != pytest.approx(full_demand_kWh) + # ... and not a naive proportional share of it either (they happen + # to coincide here only because chargingdemand_kWh/park_time_timesteps + # is linear - assert the true value directly, not this coincidence) + assert expected_kWh == pytest.approx(proportional_share_kWh) + + def test_get_flexibility_bands_excludes_events_and_limits_array_size(self): + """ + Regression test for ADR 0002: get_flexibility_bands must not build + over SimBEV's entire simulated range (nor over every charging + process) regardless of how much shorter the active timeindex is - + this was the actual "build full, then crop" waste this ADR targets, + distinct from (and originally mistaken for) mere value-correctness + after the final .loc[edisgo_timeindex] clip. A fully out-of-window + event must not extend the internal construction array at all, and + the returned bands must still be correct and exactly the active + timeindex's length. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + integrated = edisgo_obj.electromobility.integrated_charging_parks_df + park_id = integrated.index[0] + template = edisgo_obj.electromobility.charging_processes_df[ + edisgo_obj.electromobility.charging_processes_df.charging_park_id == park_id + ].iloc[0] + edisgo_id = integrated.at[park_id, "edisgo_id"] + + # one event fully inside the active window, one event far outside + # it (near the end of SimBEV's simulated week) that must not affect + # the construction array's size at all + edisgo_obj.electromobility.charging_processes_df = pd.DataFrame( + { + "ags": [template.ags, template.ags], + "car_id": [0, 1], + "destination": [template.destination, template.destination], + "use_case": [template.use_case, template.use_case], + "nominal_charging_capacity_kW": [10.0, 10.0], + "grid_charging_capacity_kW": [10.0, 10.0], + "chargingdemand_kWh": [10.0, 10.0], + "park_time_timesteps": [20, 20], + "park_start_timesteps": [10, 600], + "park_end_timesteps": [29, 619], + "charging_park_id": [park_id, park_id], + "charging_point_id": [ + template.charging_point_id, + template.charging_point_id, + ], + } + ) + + timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") # 1 day + edisgo_obj.set_timeindex(timeindex) + + orig_zeros = np.zeros + shapes = [] + + def spy_zeros(shape, *a, **kw): + shapes.append(shape) + return orig_zeros(shape, *a, **kw) + + np.zeros = spy_zeros + try: + bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, [template.use_case] + ) + finally: + np.zeros = orig_zeros + + # construction arrays must be far smaller than SimBEV's full + # simulated week (672 steps) - sized only around the active window + # and the in-window event, not the far-away out-of-window one + assert all(shape[0] < 200 for shape in shapes) + + # returned bands are still correct: exactly the active timeindex's + # length, and reflect only the in-window event's contribution + assert_index_equal(bands["upper_power"].index, timeindex) + assert bands["upper_power"][edisgo_id].sum() > 0 + + def test_get_flexibility_bands_clips_independently_per_gapped_interval(self): + """ + Regression test for ADR 0002: with a non-contiguous active + timeindex, each disjoint interval's bands must match exactly what a + standalone run scoped to just that interval would produce - i.e. + clipping the true, full band per interval, with no interaction + between intervals. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + run_1 = pd.date_range("1/1/2011", periods=24, freq="15min") + run_2 = pd.date_range("1/1/2011", periods=24, freq="15min") + pd.Timedelta( + days=3 + ) + gapped_timeindex = run_1.union(run_2) + edisgo_obj.set_timeindex(gapped_timeindex) + + gapped_bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, ["work", "public", "home", "hpc"] + ) + + edisgo_obj_run1 = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj_run1, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj_run1) + electromobility_import.integrate_charging_parks(edisgo_obj_run1) + edisgo_obj_run1.set_timeindex(run_1) + run1_bands = edisgo_obj_run1.electromobility.get_flexibility_bands( + edisgo_obj_run1, ["work", "public", "home", "hpc"] + ) + + for key in ("upper_power", "lower_energy", "upper_energy"): + assert_frame_equal( + gapped_bands[key].loc[run_1], + run1_bands[key], + check_freq=False, + ) + def test_get_flexibility_bands_empty_timeindex_is_a_no_op(self): """ With no timeindex set at all, get_flexibility_bands must return the From 617fcbba4a5f9687cf416976314c52d6718dfbdb Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 5 Aug 2026 10:06:39 +0000 Subject: [PATCH 65/66] fix: reinforce all line segments on path to critical node reinforce_lines_voltage_issues() determines the node at two thirds of the feeder length (node_2_3) and normally disconnects the line there and reconnects it to the station. If node_2_3 turns out to be the feeder representative - the bus directly connected to the station - no line can be disconnected and the measure falls back to reinforcing lines instead. In that fallback only the single line between the station and the representative was reinforced, not the remaining lines on the path to the critical node. Since the voltage deviation at the critical node is generally dominated by the segments further away from the station, the measure frequently did not change the reported voltage issue at all. Grid reinforcement then repeated the same ineffective measure in every iteration and aborted with MaximumIterationError, typically reporting a residual deviation in the per mille range. LVGrid_5 of the test grid shows this: for a voltage issue at Bus_BranchTee_LVGrid_5_2, node_2_3 is moved back to the representative Bus_BranchTee_LVGrid_5_1. Only Line_50000003 was reinforced, which is 0.56 m long, while Line_50000002 - 30 m of NAYY 4x1x35 causing nearly all of the voltage drop - was left untouched. All lines on the path from the station to the critical node are now reinforced. Feeders consisting of a single line are unaffected. Short feeders are affected most, which is why the error shows up predominantly in grids that went through spatial complexity reduction, where node_2_3 frequently coincides with the representative. Test expectations for the number of equipment changes and grid expansion costs are updated accordingly, as more lines are reinforced per measure. Applied directly on edisgo_run_edisgo instead of merging dev. Same change as 64f3ed62 on fix/reinforce-lines-voltage-issues-all-segments, without the whatsnew entry to keep a later merge with dev conflict-free. --- edisgo/flex_opt/reinforce_measures.py | 60 ++++++++++++++--------- tests/flex_opt/test_reinforce_measures.py | 30 ++++++++++-- tests/test_edisgo.py | 8 +-- 3 files changed, 65 insertions(+), 33 deletions(-) diff --git a/edisgo/flex_opt/reinforce_measures.py b/edisgo/flex_opt/reinforce_measures.py index 28ced73f7..b4e55cf93 100644 --- a/edisgo/flex_opt/reinforce_measures.py +++ b/edisgo/flex_opt/reinforce_measures.py @@ -497,32 +497,44 @@ def reinforce_lines_voltage_issues(edisgo_obj, grid, crit_nodes): # directly connected to the station), line cannot be # disconnected and must therefore be reinforced if node_2_3 in nodes_feeder.keys(): - crit_line_name = graph.get_edge_data(station_node, node_2_3)["branch_name"] - crit_line = grid.lines_df.loc[crit_line_name] - - # if critical line is already a standard line install one - # more parallel line - if crit_line.type_info == standard_line: - edisgo_obj.topology.update_number_of_parallel_lines( - pd.Series( - index=[crit_line_name], - data=[ - edisgo_obj.topology._lines_df.at[ - crit_line_name, "num_parallel" - ] - + 1 - ], + # all lines on the path from the station to the critical node are + # reinforced, not only the first line segment. As no line can be + # disconnected in this case, reinforcing the first segment alone + # does not necessarily reduce the voltage deviation at the critical + # node - the voltage drop is generally dominated by the segments + # further away from the station. Reinforcing only the first segment + # therefore leads to the same measure being repeated without effect + # in every iteration of the grid reinforcement, until it aborts with + # a MaximumIterationError. + for bus_0, bus_1 in zip(path[:-1], path[1:]): + crit_line_name = graph.get_edge_data(bus_0, bus_1)["branch_name"] + crit_line = grid.lines_df.loc[crit_line_name] + + # if critical line is already a standard line install one + # more parallel line + if crit_line.type_info == standard_line: + edisgo_obj.topology.update_number_of_parallel_lines( + pd.Series( + index=[crit_line_name], + data=[ + edisgo_obj.topology._lines_df.at[ + crit_line_name, "num_parallel" + ] + + 1 + ], + ) + ) + + # if critical line is not yet a standard line replace old + # line by a standard line + else: + # number of parallel standard lines could be calculated + # following [2] p.103; for now number of parallel + # standard lines is iterated + edisgo_obj.topology.change_line_type( + [crit_line_name], standard_line ) - ) - lines_changes[crit_line_name] = 1 - # if critical line is not yet a standard line replace old - # line by a standard line - else: - # number of parallel standard lines could be calculated - # following [2] p.103; for now number of parallel - # standard lines is iterated - edisgo_obj.topology.change_line_type([crit_line_name], standard_line) lines_changes[crit_line_name] = 1 # if node_2_3 is not a representative, disconnect line diff --git a/tests/flex_opt/test_reinforce_measures.py b/tests/flex_opt/test_reinforce_measures.py index 5ea720064..be0aede23 100644 --- a/tests/flex_opt/test_reinforce_measures.py +++ b/tests/flex_opt/test_reinforce_measures.py @@ -300,12 +300,19 @@ def test_reinforce_lines_voltage_issues(self): # LV: # * check where node_2_3 is in_building => problem at - # Bus_BranchTee_LVGrid_5_2, leads to reinforcement of line - # Line_50000003 (which is first line in feeder and not a - # standard line) + # Bus_BranchTee_LVGrid_5_2, node_2_3 is moved back to + # Bus_BranchTee_LVGrid_5_1, which is the feeder representative, so no + # line can be disconnected. All lines on the path from the station to + # the critical node are reinforced, i.e. Line_50000003 (first line in + # feeder, not a standard line) and Line_50000002 (line to the critical + # node, not a standard line). Line_50000003 is only 0.56 m long, while + # Line_50000002 is 30 m of NAYY 4x1x35 and therefore causes most of + # the voltage deviation - reinforcing the first line segment only + # would not resolve the voltage issue. # * check where node_2_3 is not in_building => problem at # Bus_BranchTee_LVGrid_5_5, leads to reinforcement of line - # Line_50000009 (which is first line in feeder and a standard line) + # Line_50000009 (which is the only line in the feeder and a standard + # line) crit_nodes = pd.DataFrame( { @@ -321,8 +328,9 @@ def test_reinforce_lines_voltage_issues(self): ) reinforced_lines = lines_changes.keys() - assert len(lines_changes) == 2 + assert len(lines_changes) == 3 assert "Line_50000003" in reinforced_lines + assert "Line_50000002" in reinforced_lines assert "Line_50000009" in reinforced_lines # check that LV station is one of the buses assert ( @@ -364,6 +372,18 @@ def test_reinforce_lines_voltage_issues(self): line.s_nom, np.sqrt(3) * grid.nominal_voltage * std_line.I_max_th ) assert line.num_parallel == 1 + # second line segment on the path to the critical node is reinforced as + # well + line = self.edisgo.topology.lines_df.loc["Line_50000002"] + assert line.type_info == std_line.name + assert np.isclose(line.r, std_line.R_per_km * line.length) + assert np.isclose( + line.x, std_line.L_per_km * line.length * 2 * np.pi * 50 / 1e3 + ) + assert np.isclose( + line.s_nom, np.sqrt(3) * grid.nominal_voltage * std_line.I_max_th + ) + assert line.num_parallel == 1 line = self.edisgo.topology.lines_df.loc["Line_50000009"] assert line.type_info == std_line.name assert line.num_parallel == 2 diff --git a/tests/test_edisgo.py b/tests/test_edisgo.py index 9184a44a7..b5617b7c3 100755 --- a/tests/test_edisgo.py +++ b/tests/test_edisgo.py @@ -608,7 +608,7 @@ def test_reinforce_catch_convergence(self): ) assert results.unresolved_issues.empty assert len(results.grid_expansion_costs) == 134 - assert len(results.equipment_changes) == 230 + assert len(results.equipment_changes) == 236 assert results.v_res.shape == (4, 142) # ############### test with catch convergence worst case true ################ @@ -619,7 +619,7 @@ def test_reinforce_catch_convergence(self): results = self.edisgo.reinforce(catch_convergence_problems=True) assert results.unresolved_issues.empty assert len(results.grid_expansion_costs) == 134 - assert len(results.equipment_changes) == 218 + assert len(results.equipment_changes) == 223 assert results.v_res.shape == (4, 142) @pytest.mark.slow @@ -640,8 +640,8 @@ def test_enhanced_reinforce_grid(self): results = edisgo_obj.results - assert len(results.grid_expansion_costs) == 454 - assert len(results.equipment_changes) == 892 + assert len(results.grid_expansion_costs) == 460 + assert len(results.equipment_changes) == 935 assert results.v_res.shape == (4, 148) edisgo_obj = copy.deepcopy(self.edisgo) From 9b472892d411d343114f32a12097f646084f3672 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 2 Sep 2026 07:50:53 +0000 Subject: [PATCH 66/66] fix(io): drop the dead bus-to-building-ID map in the PV rooftop import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picked from 8105c72a on dev (branch fix/ego-213-pv-rooftop-building-id-duplicate-bus), without the changelog entry — this branch has no 0.3.1 section to put it in. Importing generators failed for some grids with ValueError: cannot reindex on an axis with duplicate labels raised from `gens_df.update()` in `_integrate_pv_rooftop` (openego/eGo#213, reported for MV grids 33084 and 33695). The function derived a bus -> building ID map from the conventional loads. That map was deduplicated on "building_id" but indexed by "bus", so a bus carrying the loads of two different buildings kept both rows and produced a duplicated bus label. `gens_df.loc[:, ["bus"]].join(..., on="bus")` then emitted two rows per generator, and `DataFrame.update` reindexes its argument onto `self.index`, which rejects a non-unique index. The map is not repaired here, it is removed, because it has had no effect on the result for two and a half years. It was introduced in ae002033 to supply the key for matching existing PV rooftop plants against the scenario data, which merged `on="building_id"` at the time; the generators in a ding0 topology carry no building ID, so one was guessed from the conventional load sitting at the same bus. A ToDo above the block already asked for the match to use the unique MaStR source ID instead. d33e8515 did exactly that and removed the ToDo, but left the map behind. Since then the guessed value has been discarded: `gens_existing` is merged with `pv_rooftop_df` on "source_id" with `suffixes=("_old", "")`, so the building ID written back to `generators_df` comes from the scenario data and the guessed one survives only as the unread column `building_id_old`. Every PV rooftop generator is either decommissioned and removed, or matched and updated from the scenario, so there is no path on which the guess is read. Setting the building ID of all 2472 conventional loads in the test grid to a sentinel value changes nothing about the generators the function produces, and `generators_df` is byte-identical with and without the block. `loads_df` was read for the map alone, so the function no longer touches the loads at all. The regression test is kept as a guard against reintroducing a per-bus load lookup. The shipped test grid cannot produce the situation on its own: it has 70 buses with more than one conventional load, but all of them share a single building ID, so the second building is added explicitly. --- edisgo/io/generators_import.py | 15 -------- tests/io/test_generators_import.py | 60 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/edisgo/io/generators_import.py b/edisgo/io/generators_import.py index c22becae1..e5ea65731 100755 --- a/edisgo/io/generators_import.py +++ b/edisgo/io/generators_import.py @@ -1030,24 +1030,9 @@ def _integrate_pv_rooftop(edisgo_object, pv_rooftop_df): MaStR ID of the PV plant. """ - # match building ID to existing solar generators - loads_df = edisgo_object.topology.loads_df - busses_building_id = ( - loads_df[loads_df.type == "conventional_load"] - .drop_duplicates(subset=["building_id"]) - .set_index("bus") - .loc[:, ["building_id"]] - ) gens_df = edisgo_object.topology.generators_df[ edisgo_object.topology.generators_df.subtype == "pv_rooftop" ].copy() - gens_df_building_id = gens_df.loc[:, ["bus"]].join( - busses_building_id, how="left", on="bus" - ) - # using update to make sure to not overwrite existing building ID information - if "building_id" not in gens_df.columns: - gens_df["building_id"] = None - gens_df.update(gens_df_building_id, overwrite=False) # remove decommissioned PV rooftop plants gens_decommissioned = gens_df[ diff --git a/tests/io/test_generators_import.py b/tests/io/test_generators_import.py index 47765f703..6e06d2f6e 100644 --- a/tests/io/test_generators_import.py +++ b/tests/io/test_generators_import.py @@ -305,6 +305,66 @@ def test__integrate_pv_rooftop(self, caplog): "matched to an existing PV rooftop plant." in caplog.text ) + def test__integrate_pv_rooftop_two_buildings_on_one_bus(self): + """ + A bus carrying conventional loads of two different buildings must not + break the PV rooftop import. + + This used to raise "cannot reindex on an axis with duplicate labels" + (openego/eGo#213, seen on MV grids 33084 and 33695): the function built + a bus -> building ID map from the conventional loads, and that map was + deduplicated on "building_id" while being indexed by "bus". Two + buildings on one bus therefore left a duplicated bus label, the join + multiplied the generator rows and ``DataFrame.update`` rejected the + non-unique index. + + That map has since been removed entirely -- it fed a building-ID based + matching that was replaced by source-ID matching in d33e8515 -- so the + function no longer reads ``loads_df`` at all and the crash is + structurally impossible. This test is kept as a guard against + reintroducing a per-bus load lookup here. + + Note that the shipped test grid cannot produce the situation on its + own: it has 70 buses with more than one conventional load, but all of + them share a single building ID. The second building is therefore + added explicitly. + """ + edisgo = EDisGo( + ding0_grid=pytest.ding0_test_network_3_path, legacy_ding0_grids=False + ) + loads_df = edisgo.topology.loads_df + gens_df = edisgo.topology.generators_df + # a bus that carries both a PV rooftop generator and a conventional load + pv_buses = set(gens_df[gens_df.subtype == "pv_rooftop"].bus) + conv = loads_df[loads_df.type == "conventional_load"] + bus = conv[conv.bus.isin(pv_buses)].bus.iloc[0] + existing = conv[conv.bus == bus].iloc[0] + + # second building on the same bus + second = existing.copy() + second["building_id"] = int(existing.building_id) + 1_000_000 + edisgo.topology.loads_df.loc["Load_second_building_same_bus"] = second + + pv_df = pd.DataFrame( + data={ + "p_nom": [0.005], + "weather_cell_id": [11051], + "building_id": [430903], + "generator_id": [1], + "type": ["solar"], + "subtype": ["pv_rooftop"], + "source_id": ["SEE970362202254"], + }, + index=[1], + ) + + # used to raise ValueError: cannot reindex on an axis with duplicate labels + generators_import._integrate_pv_rooftop(edisgo, pv_df) + + # no generator was duplicated by the join (generators whose source_id + # is absent from the scenario are legitimately decommissioned here) + assert not edisgo.topology.generators_df.index.has_duplicates + def test__integrate_new_pv_rooftop_to_buildings(self, caplog): pv_df = pd.DataFrame( data={