diff --git a/.gitignore b/.gitignore index 9c649eb27..ed8ff1563 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,24 @@ eDisGo.egg-info/ .vscode/settings.json *OEP_TOKEN.* + +# local-only notes, not for sharing on this branch +/CONTEXT.md +/docs_notes/spatial_reduction_grilling_session.md +/docs_notes/issue_spatial_complexity_reduction_pipeline_integration.md +/docs_notes/issue_aggregation_mode_flexibility_bands_not_aggregated.md +/docs_notes/issue_temporal_reduction_flexibility_bands.md +analyse_spatial_reduction_executed_example.ipynb +analyse_spatial_reduction.ipynb +analyse_uc5_results copy.ipynb +analyse_uc5_results.ipynb +handson_edisgo_starter.ipynb +run_example_04.py +docs_notes/ego_timeseries_selection_plan.md +docs_notes/pr_description_spatial_complexity_reduction.md +docs_notes/timeseries_selection_remaining_tasks.md +results/uc4_example/main.zip +results/uc5_select_timesteps/32377_t_168_residual_load.csv +results/uc5_select_timesteps/main.zip +results/uc5_spatial_reduction/main.zip +results/uc6_spatial_reduction/main.zip diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index fe7fa6462..23bdf1fc0 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -64,10 +64,15 @@ from edisgo.tools import plots, tools from edisgo.tools.config import Config from edisgo.tools.geo import find_nearest_bus -from edisgo.tools.spatial_complexity_reduction import spatial_complexity_reduction +from edisgo.tools.spatial_complexity_reduction import ( + apply_reduced_results_to_full_grid, + spatial_complexity_reduction, +) from edisgo.tools.tools import ( + check_timeindex_coverage, determine_grid_integration_voltage_level, get_path_length_to_station, + reduce_timeseries_data_to_given_timeindex, ) if "READTHEDOCS" not in os.environ: @@ -243,6 +248,31 @@ def config(self): def config(self, kwargs): self._config = Config(**kwargs) + def run_pipeline(self, config, overlying_grid_data=None): + """ + Run a YAML/JSON task pipeline on this EDisGo instance. + + See :mod:`edisgo.run` for the config schema and task list. + + Parameters + ---------- + config : str, :class:`pathlib.Path`, or dict + Pipeline config as path to a YAML/JSON file or as a dict. + overlying_grid_data : dict, optional + Overlying-grid data (e.g. eTraGo results) consumed by the + ``import_overlying_grid_data`` task when + ``overlying_grid.source == "etrago"``. + + Returns + ------- + :class:`~.EDisGo` + The EDisGo instance after the pipeline has run. + + """ + from edisgo.run import _run_pipeline_on + + return _run_pipeline_on(self, config, overlying_grid_data=overlying_grid_data) + def import_ding0_grid(self, path, legacy_ding0_grids=True): """ Import ding0 topology data from csv files in the format as @@ -323,7 +353,11 @@ def set_time_series_manual( providing the input parameter 'timeindex' or using the function :attr:`~.edisgo.EDisGo.set_timeindex`. Also make sure that the time steps for which time series are provided include - the set time index. + the set time index - this is now enforced: a `ValueError` is raised if a + non-empty DataFrame is missing data for a time step in + :attr:`~.network.timeseries.TimeSeries.timeindex` when a time index is + already set. A DataFrame with no columns is exempt from this check (nothing + is being written, so there is nothing to validate coverage for). """ # check if time index is already set, otherwise raise warning @@ -334,6 +368,16 @@ def set_time_series_manual( "upon initialisation of the EDisGo object by providing the input " "parameter 'timeindex' or using the function EDisGo.set_timeindex()." ) + else: + for name, df in ( + ("generators_p", generators_p), + ("loads_p", loads_p), + ("storage_units_p", storage_units_p), + ("generators_q", generators_q), + ("loads_q", loads_q), + ("storage_units_q", storage_units_q), + ): + check_timeindex_coverage(self.timeseries.timeindex, name, df) self.timeseries.set_active_power_manual( self, ts_generators=generators_p, @@ -2162,6 +2206,14 @@ def apply_charging_strategy( match the SimBEV data frequency and after determining the charging demand time series resampled back to the original frequency. + The written charging point time series are trimmed to + :attr:`~.network.timeseries.TimeSeries.timeindex` when no such frequency + mismatch occurs. When it does occur, the resample round-trip above + currently fabricates a contiguous timeindex, which can reopen a gap + left by a prior manual/auto time step selection - the trim is skipped + in that case rather than risk operating on the wrong window. See + :func:`~.flex_opt.charging_strategies.charging_strategy` for details. + """ charging_strategy( self, strategy=strategy, charging_park_ids=charging_park_ids, **kwargs @@ -2340,6 +2392,16 @@ def apply_heat_pump_operating_strategy( pumps for which COP information in :attr:`~.edisgo.EDisGo.heat_pump` is given are used. Default: None. + Notes + ----- + The written load time series are scoped to + :attr:`~.network.timeseries.TimeSeries.timeindex`, regardless of + whether :attr:`~.edisgo.EDisGo.heat_pump`'s COP/heat demand time + series currently span a wider or different range. Raises + ``KeyError`` if they are missing data for a time step in the active + timeindex - see :func:`~.flex_opt.heat_pump_operation.operating_strategy` + for details. + """ hp_operating_strategy(self, strategy=strategy, heat_pump_names=heat_pump_names) @@ -3551,6 +3613,65 @@ def spatial_complexity_reduction( ) return edisgo_obj, busmap_df, linemap_df + def map_reduced_results_to_full_grid( + self, + reduced_grid: EDisGo, + flexible_cps: list | None = None, + flexible_hps: list | None = None, + flexible_loads: list | None = None, + flexible_storage_units: list | None = None, + ) -> EDisGo: + """ + Writes optimized flexible-component dispatch from a spatially-reduced + grid back onto this (full) grid. + + Counterpart to :meth:`spatial_complexity_reduction`: where that + method shrinks this grid for a faster OPF, this method maps the OPF's + active-power results from ``reduced_grid`` back onto ``self`` so + reinforcement can run on the full topology. Only components the OPF + actually rewrites are touched — flexible charging points, heat pumps, + DSM loads, and storage units. Inflexible loads/generators are + untouched, since the OPF never changed their series and ``self`` + already holds the correct values for them. + + See + :func:`~.tools.spatial_complexity_reduction.apply_reduced_results_to_full_grid` + for the full matching/disaggregation rules and the reactive-power + recompute this method triggers as a side effect. + + Parameters + ---------- + reduced_grid : :class:`~.EDisGo` + The spatially-reduced EDisGo instance the OPF ran on. Supplies + the optimized active-power series and, if aggregated, the + ``old_name`` provenance for disaggregation. + flexible_cps : list of str, optional + Names of flexible charging points in ``reduced_grid`` to map + back. + flexible_hps : list of str, optional + Names of flexible heat-pump loads in ``reduced_grid`` to map + back. + flexible_loads : list of str, optional + Names of flexible DSM loads in ``reduced_grid`` to map back. + flexible_storage_units : list of str, optional + Names of flexible storage units in ``reduced_grid`` to map back. + + Returns + ------- + :class:`~.EDisGo` + ``self``, with active power written for the given flexible + components and reactive power recomputed. + + """ + return apply_reduced_results_to_full_grid( + full_grid=self, + reduced_grid=reduced_grid, + flexible_cps=flexible_cps, + flexible_hps=flexible_hps, + flexible_loads=flexible_loads, + flexible_storage_units=flexible_storage_units, + ) + def check_integrity(self): """ Method to check the integrity of the EDisGo object. @@ -3691,8 +3812,8 @@ def resample_timeseries( """ Resamples time series data in :class:`~.network.timeseries.TimeSeries`, :class:`~.network.heat.HeatPump`, - :class:`~.network.electromobility.Electromobility` and - :class:`~.network.overlying_grid.OverlyingGrid`. + :class:`~.network.electromobility.Electromobility`, + :class:`~.network.dsm.DSM` and :class:`~.network.overlying_grid.OverlyingGrid`. Both up- and down-sampling methods are possible. @@ -3716,6 +3837,14 @@ def resample_timeseries( * :attr:`~.network.heat.HeatPump.heat_demand_df` + * :attr:`~.network.dsm.DSM.p_min` + + * :attr:`~.network.dsm.DSM.p_max` + + * :attr:`~.network.dsm.DSM.e_min` + + * :attr:`~.network.dsm.DSM.e_max` + * All data in :class:`~.network.overlying_grid.OverlyingGrid` Parameters @@ -3748,8 +3877,77 @@ def resample_timeseries( self.timeseries.resample(method=method, freq=freq) self.electromobility.resample(freq=freq) self.heat_pump.resample_timeseries(method=method, freq=freq) + self.dsm.resample(method=method, freq=freq) self.overlying_grid.resample(method=method, freq=freq) + def reduce_timeseries_data_to_given_timeindex( + self, + timeindex, + freq="1H", + timeseries=True, + electromobility=True, + save_ev_soc_initial=True, + heat_pump=True, + dsm=True, + overlying_grid=True, + ): + """ + Reduces timeseries data in this EDisGo object to given time index. + + Thin wrapper around + :func:`edisgo.tools.tools.reduce_timeseries_data_to_given_timeindex`, + exposed here for discoverability - the underlying implementation is + otherwise only importable directly from ``edisgo.tools.tools``, which + made it easy to miss for anyone using :class:`~.EDisGo` outside the + ``run`` pipeline (its only prior callers). + + Parameters + ----------- + timeindex : :pandas:`pandas.DatetimeIndex` + Time index to set. + freq : str or :pandas:`pandas.Timedelta`, optional + Frequency of time series data. This is only needed if it cannot be + inferred from the given `timeindex` and if electromobility data + and/or overlying grid data is reduced, as the initial SoC is + tried to be set using the time step before the first time step in + the given `timeindex`. Offset aliases can be found here: + https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases. + Default: '1H'. + timeseries : bool + Indicates whether timeseries in + :class:`~.network.timeseries.TimeSeries` are reduced to given + time index. Default: True. + electromobility : bool + Indicates whether timeseries in + :class:`~.network.electromobility.Electromobility` are reduced to + given time index. Default: True. + save_ev_soc_initial : bool + Indicates whether to save initial EV SOC from timestep before + first timestep of given time index. Default: True. + heat_pump : bool + Indicates whether timeseries in :class:`~.network.heat.HeatPump` + are reduced to given time index. Default: True. + dsm : bool + Indicates whether timeseries in :class:`~.network.dsm.DSM` are + reduced to given time index. Default: True. + overlying_grid : bool + Indicates whether timeseries in + :class:`~.network.overlying_grid.OverlyingGrid` are reduced to + given time index. Default: True. + + """ + reduce_timeseries_data_to_given_timeindex( + self, + timeindex, + freq=freq, + timeseries=timeseries, + electromobility=electromobility, + save_ev_soc_initial=save_ev_soc_initial, + heat_pump=heat_pump, + dsm=dsm, + overlying_grid=overlying_grid, + ) + def import_edisgo_from_pickle(filename, path=""): """ diff --git a/edisgo/flex_opt/charging_strategies.py b/edisgo/flex_opt/charging_strategies.py index dd260e6a9..6abbc3c35 100644 --- a/edisgo/flex_opt/charging_strategies.py +++ b/edisgo/flex_opt/charging_strategies.py @@ -89,7 +89,29 @@ def charging_strategy( :attr:`~.edisgo.EDisGo.apply_charging_strategy` for more information. Default: 0.1. + Notes + ----- + The written ``loads_active_power``/``loads_reactive_power`` are trimmed to + ``edisgo_obj.timeseries.timeindex`` when its frequency already matches the + SimBEV charging-process data's ``stepsize`` (the common case). When it + doesn't, this function internally resamples ``edisgo_obj.timeseries`` to + SimBEV's frequency and back (see the frequency-mismatch warning below); + that round-trip currently fabricates a contiguous timeindex, which can + reopen a gap left by ``select_timesteps`` (auto mode). The trim is + skipped in that case rather than risk operating on the wrong window - + tracked as a known limitation in + ``docs_notes/issue_temporal_reduction_flexibility_bands.md``. + """ + # Capture the target time index before any internal frequency resampling + # (below) can mutate it. `TimeSeries.resample` fabricates a contiguous + # index spanning first-to-last timestamp, which would silently reopen any + # gap `select_timesteps` (auto mode) deliberately left in the timeindex - + # trimming against this entry-time snapshot instead ensures only the + # steps actually selected by the caller are written. Only used when no + # internal resample round-trip happens (see Notes above). + target_timeindex = edisgo_obj.timeseries.timeindex + # get integrated charging parks integrated_parks = edisgo_obj.electromobility.integrated_charging_parks_df @@ -170,6 +192,27 @@ def charging_strategy( edisgo_obj.timeseries.resample(freq=simbev_timedelta) + # Map each SimBEV step position (0 .. len_ts - 1, the same positional + # space as park_start_timesteps/the placement slices below) to whether it + # is present in the active timeindex (`target_timeindex`). `dumb` and + # `reduced` place each event's demand deterministically at + # [start, start+stop) - rather than building the full-SimBEV-length + # series unconditionally and cropping the *output* down to the active + # timeindex afterwards (as before), the placement itself is now clipped + # to whatever of that interval is actually in-window, so an event's + # reported energy is a direct consequence of which positions get + # written, not a separate proration calculation (see ADR 0002). + # `resample=True` means `target_timeindex` predates an internal + # frequency round-trip and is no longer in the same step space as + # `park_start_timesteps` - the crop-after-build step already skips + # trimming in that case (see the module docstring), so this reduction is + # skipped here too and every step is treated as in-window, preserving + # today's (build-full) behavior only for that known limitation. + if resample: + step_in_window = np.ones(len_ts, dtype=bool) + else: + step_in_window = np.isin(timeindex, target_timeindex) + if strategy == "dumb": # "dumb" charging # Collect each charging park's series and add them to the time series in a @@ -192,7 +235,15 @@ def charging_strategy( for _, start, stop, cap in charging_processes_df[ RELEVANT_CHARGING_STRATEGIES_COLUMNS["dumb"] ].itertuples(): - dummy_ts[start : start + stop] += cap + # Write only to in-window positions of the deterministic + # charging interval [start, start+stop) - if the active + # timeindex has a gap inside this interval, every in-window + # sub-slice still gets the event's full, unscaled power (see + # ADR 0002); out-of-window positions are simply not written. + in_window_idx = ( + np.flatnonzero(step_in_window[start : start + stop]) + start + ) + dummy_ts[in_window_idx] += cap cp_ts[cp.edisgo_id] = dummy_ts @@ -231,12 +282,20 @@ def charging_strategy( ) in charging_processes_df[ RELEVANT_CHARGING_STRATEGIES_COLUMNS["reduced"] ].itertuples(): + # See the "dumb" branch above for why the placement slice + # itself (not a separate energy calculation) is clipped to + # in-window positions. if use_case == "public" or use_case == "hpc": # if the charging process takes place in a "public" setting # the charging is "dumb" - dummy_ts[start : start + stop_dumb] += cap_dumb + start_, stop_, cap = start, stop_dumb, cap_dumb else: - dummy_ts[start : start + stop_reduced] += cap_reduced + start_, stop_, cap = start, stop_reduced, cap_reduced + + in_window_idx = ( + np.flatnonzero(step_in_window[start_ : start_ + stop_]) + start_ + ) + dummy_ts[in_window_idx] += cap cp_ts[cp.edisgo_id] = dummy_ts @@ -264,35 +323,118 @@ def charging_strategy( eta_cp=eta_cp, ) - # get residual load - init_residual_load = edisgo_obj.timeseries.residual_load - len_residual_load = int(charging_processes_df.park_end_timesteps.max()) - if len(init_residual_load) >= len_residual_load: - init_residual_load = init_residual_load.loc[timeindex] + if not resample: + # The active timeindex can extend past the last charging event + # (e.g. a trailing gapped run with no events in it at all) - the + # step-space array built below must cover at least as far as + # target_timeindex itself, or the crop-after-build step later + # would reindex into positions that were never built, producing + # NaN rather than a legitimate zero. + target_span_steps = int( + (target_timeindex[-1] - target_timeindex[0]) + / pd.Timedelta(f"{edisgo_obj.electromobility.stepsize}min") + ) + len_residual_load = max(len_residual_load, target_span_steps) + + # Map each SimBEV step position (0 .. len_residual_load, the same + # positional space as park_start_timesteps/park_end_timesteps) to + # whether it is present in the active timeindex (`target_timeindex`). + # Real residual_load only exists for `target_timeindex`. Rather than + # tiling (cyclically repeating) it to cover steps beyond the active + # timeindex - which would rank timesteps against a fabricated, + # non-periodic-in-reality signal (see ADR 0001) - steps outside the + # active timeindex are simply marked as having no usable data. + # `resample=True` means `target_timeindex` predates an internal + # frequency round-trip and is no longer in the same step space as + # `park_start_timesteps` - the crop-after-build step already skips + # trimming in that case (see the module docstring), so this reduction + # is skipped here too and every step is treated as in-window, + # preserving today's (tiling) behavior only for that known + # limitation. + if resample: + step_in_window = np.ones(len_residual_load + 1, dtype=bool) else: - while len(init_residual_load) < len_residual_load: - len_rl = len(init_residual_load) - len_append = min(len_rl, len_residual_load - len_rl) - - s_append = init_residual_load.iloc[:len_append] - - init_residual_load = pd.concat( - [ - init_residual_load, - s_append, - ], - ignore_index=True, - ) + step_in_window = np.isin( + pd.date_range( + target_timeindex[0], + periods=len_residual_load + 1, + freq=f"{edisgo_obj.electromobility.stepsize}min", + ), + target_timeindex, + ) + in_window_steps_cumsum = np.concatenate(([0], np.cumsum(step_in_window))) + + if not resample: + # Events are reduced to the active timeindex before being + # scheduled: fully in-window events are untouched, fully + # out-of-window events are dropped, and boundary-straddling + # events have their charging demand prorated by how much of + # their parking time is actually observable. This mirrors + # `harmonize_charging_processes_df`'s own derivation of + # `minimum_charging_time` from demand and nominal power. + parking_time = ( + charging_processes_df.park_end_timesteps + - charging_processes_df.park_start_timesteps + + 1 + ) + overlap_steps = ( + in_window_steps_cumsum[ + charging_processes_df.park_end_timesteps.to_numpy() + 1 + ] + - in_window_steps_cumsum[ + charging_processes_df.park_start_timesteps.to_numpy() + ] + ) + + # drop events with zero overlap - nothing to schedule, no + # residual_load data exists for them at all + in_window = overlap_steps > 0 + charging_processes_df = charging_processes_df.loc[in_window] + in_window_fraction = ( + (overlap_steps[in_window]) / (parking_time.to_numpy()[in_window]) + ) + + scaled_demand_kWh = ( + charging_processes_df.harmonized_chargingdemand * in_window_fraction + ) + scaled_minimum_charging_time = ( + scaled_demand_kWh + / charging_processes_df.nominal_charging_capacity_kW + * 60 + / edisgo_obj.electromobility.stepsize + ) + scaled_minimum_charging_time = np.ceil(scaled_minimum_charging_time).astype( + np.uint16 + ) + + # defensive clamp: proration preserves + # minimum_charging_time <= parking_time, so this should only ever + # bind on pre-existing anomalous input (an event whose full, + # unscaled demand already didn't fit its own parking time) + scaled_minimum_charging_time = np.minimum( + scaled_minimum_charging_time, overlap_steps[in_window] + ) + + charging_processes_df = charging_processes_df.assign( + minimum_charging_time=scaled_minimum_charging_time, + flex_time=charging_processes_df.park_time_timesteps + - scaled_minimum_charging_time, + ) - init_residual_load = init_residual_load.to_numpy() + # get residual load; steps outside the active timeindex carry no + # real data (see above) and are set to NaN so they can never be + # selected as charging candidates below + init_residual_load = edisgo_obj.timeseries.residual_load timeindex_residual = pd.date_range( edisgo_obj.timeseries.timeindex[0], - periods=len(init_residual_load), + periods=len_residual_load + 1, freq=f"{edisgo_obj.electromobility.stepsize}min", ) + init_residual_load = init_residual_load.reindex(timeindex_residual).to_numpy() + init_residual_load[~step_in_window] = np.nan dummy_ts = pd.DataFrame( data=0.0, columns=[_.id for _ in charging_parks], index=timeindex_residual @@ -313,7 +455,15 @@ def charging_strategy( RELEVANT_CHARGING_STRATEGIES_COLUMNS["residual_dumb"] ].itertuples(): try: - dummy_ts.loc[:, cp_id].iloc[start : start + stop] += cap + # Write only to in-window positions of the deterministic + # charging interval [start, start+stop) - if the active + # timeindex has a gap inside this interval, every in-window + # sub-slice still gets the event's full, unscaled power (see + # ADR 0002); out-of-window positions are simply not written. + in_window_idx = ( + np.flatnonzero(step_in_window[start : start + stop]) + start + ) + dummy_ts.loc[:, cp_id].iloc[in_window_idx] += cap except Exception: maximum_ts = len(dummy_ts) @@ -329,11 +479,23 @@ def charging_strategy( for _, start, end, k, cp_id, cap in flex_charging_processes_df[ RELEVANT_CHARGING_STRATEGIES_COLUMNS["residual"] ].itertuples(): - flex_band = residual_load[start : end + 1] - - # get k time steps with the lowest residual load in the parking - # time - idx = np.argpartition(flex_band, k)[:k] + start + # Restrict ranking candidates to timesteps that are both within + # the parking window and present in the active timeindex - + # `residual_load` is NaN outside the active timeindex (no real + # data exists there, see above), so those positions must never + # be selected, even if the parking window itself spans a gap. + candidates = np.flatnonzero(step_in_window[start : end + 1]) + start + + if k >= len(candidates): + # k charging demand may (after proration/clamping) exactly + # saturate the available in-window candidates - nothing left + # to rank, every candidate is used. + idx = candidates + else: + flex_band = residual_load[candidates] + # get k time steps with the lowest residual load in the + # parking time, among the valid (in-window) candidates only + idx = candidates[np.argpartition(flex_band, k)[:k]] try: dummy_ts[cp_id].iloc[idx] += cap @@ -364,14 +526,48 @@ def charging_strategy( if resample: edisgo_obj.timeseries.resample(freq=edisgo_timedelta) + # `TimeSeries.resample` fabricates a contiguous index spanning + # first-to-last timestamp, which would reopen any gap + # `select_timesteps` (auto mode) left in `target_timeindex`. The trim + # below only removes *extra trailing* rows past `target_timeindex`'s + # own span - it does not (and, given the above, safely cannot) + # reintroduce a gap `resample` already closed. Fixing that root cause + # in `TimeSeries.resample` itself is tracked separately (see + # docs_notes/issue_temporal_reduction_flexibility_bands.md); until + # then, a `select_timesteps`-produced gap combined with a + # SimBEV/edisgo frequency mismatch is a known limitation here. + else: + # Trim the newly written columns down to the target time index. The + # writes above (all three strategies) span the full SimBEV + # simulation length rather than the active timeindex. + # `TimeSeries.loads_active_power` itself already scopes reads to + # `self.timeindex`, but the private `_loads_active_power` can still + # carry the untrimmed rows (visible to anything reading the private + # attribute directly, e.g. `reduce_timeseries_data_to_given_timeindex`) + # - rebuild just the touched columns via drop+add so only the extra + # rows for `edisgo_ids_to_update` are removed, leaving other + # components untouched. + trimmed_active_power = edisgo_obj.timeseries._loads_active_power.loc[ + :, edisgo_ids_to_update + ].reindex(target_timeindex) + edisgo_obj.timeseries.drop_component_time_series( + "loads_active_power", edisgo_ids_to_update + ) + edisgo_obj.timeseries.add_component_time_series( + "loads_active_power", trimmed_active_power + ) - # set reactive power time series to 0 Mvar + # set reactive power time series to 0 Mvar. Use `target_timeindex` only + # when it still matches `edisgo_obj.timeseries.timeindex` (i.e. no + # internal resample round-trip happened above) - see the comment on the + # active-power trim above for why a resampled, gap-closed timeindex isn't + # safely reconcilable with `target_timeindex` here yet. # fmt: off edisgo_obj.timeseries.add_component_time_series( "loads_reactive_power", pd.DataFrame( data=0.0, - index=edisgo_obj.timeseries.timeindex, + index=target_timeindex if not resample else edisgo_obj.timeseries.timeindex, columns=edisgo_ids_to_update, ), ) diff --git a/edisgo/flex_opt/heat_pump_operation.py b/edisgo/flex_opt/heat_pump_operation.py index e6a0d6fb9..1d31006c5 100644 --- a/edisgo/flex_opt/heat_pump_operation.py +++ b/edisgo/flex_opt/heat_pump_operation.py @@ -38,14 +38,33 @@ def operating_strategy( parameter in :attr:`~.edisgo.EDisGo.apply_heat_pump_operating_strategy` for more information. Default: None. + Notes + ----- + The written ``loads_active_power`` is scoped to + ``edisgo_obj.timeseries.timeindex``, regardless of whether + ``edisgo_obj.heat_pump.heat_demand_df``/``cop_df`` currently span a wider + or different range (e.g. because the timeindex changed after + :attr:`~.edisgo.EDisGo.import_heat_pumps` ran). Raises ``KeyError`` if + either is missing data for a time step in ``timeindex`` - this is + data-staleness the caller should fix (re-import or re-set the heat pump + time series for the active timeindex), not something to silently paper + over. + """ if heat_pump_names is None: heat_pump_names = edisgo_obj.heat_pump.cop_df.columns if strategy == "uncontrolled": + # Scope to the active timeindex explicitly rather than relying on + # heat_demand_df/cop_df already matching it - import_heat_pumps trims + # both to the timeindex active at import time, but nothing re-trims + # them if the timeindex changes afterward (e.g. a later + # select_timesteps step), which would otherwise silently write rows + # outside the current timeindex into loads_active_power. + timeindex = edisgo_obj.timeseries.timeindex ts = ( - edisgo_obj.heat_pump.heat_demand_df.loc[:, heat_pump_names] - / edisgo_obj.heat_pump.cop_df.loc[:, heat_pump_names] + edisgo_obj.heat_pump.heat_demand_df.loc[timeindex, heat_pump_names] + / edisgo_obj.heat_pump.cop_df.loc[timeindex, heat_pump_names] ) edisgo_obj.timeseries.add_component_time_series( "loads_active_power", diff --git a/edisgo/flex_opt/reinforce_measures.py b/edisgo/flex_opt/reinforce_measures.py index 28ced73f7..b4e55cf93 100644 --- a/edisgo/flex_opt/reinforce_measures.py +++ b/edisgo/flex_opt/reinforce_measures.py @@ -497,32 +497,44 @@ def reinforce_lines_voltage_issues(edisgo_obj, grid, crit_nodes): # directly connected to the station), line cannot be # disconnected and must therefore be reinforced if node_2_3 in nodes_feeder.keys(): - crit_line_name = graph.get_edge_data(station_node, node_2_3)["branch_name"] - crit_line = grid.lines_df.loc[crit_line_name] - - # if critical line is already a standard line install one - # more parallel line - if crit_line.type_info == standard_line: - edisgo_obj.topology.update_number_of_parallel_lines( - pd.Series( - index=[crit_line_name], - data=[ - edisgo_obj.topology._lines_df.at[ - crit_line_name, "num_parallel" - ] - + 1 - ], + # all lines on the path from the station to the critical node are + # reinforced, not only the first line segment. As no line can be + # disconnected in this case, reinforcing the first segment alone + # does not necessarily reduce the voltage deviation at the critical + # node - the voltage drop is generally dominated by the segments + # further away from the station. Reinforcing only the first segment + # therefore leads to the same measure being repeated without effect + # in every iteration of the grid reinforcement, until it aborts with + # a MaximumIterationError. + for bus_0, bus_1 in zip(path[:-1], path[1:]): + crit_line_name = graph.get_edge_data(bus_0, bus_1)["branch_name"] + crit_line = grid.lines_df.loc[crit_line_name] + + # if critical line is already a standard line install one + # more parallel line + if crit_line.type_info == standard_line: + edisgo_obj.topology.update_number_of_parallel_lines( + pd.Series( + index=[crit_line_name], + data=[ + edisgo_obj.topology._lines_df.at[ + crit_line_name, "num_parallel" + ] + + 1 + ], + ) + ) + + # if critical line is not yet a standard line replace old + # line by a standard line + else: + # number of parallel standard lines could be calculated + # following [2] p.103; for now number of parallel + # standard lines is iterated + edisgo_obj.topology.change_line_type( + [crit_line_name], standard_line ) - ) - lines_changes[crit_line_name] = 1 - # if critical line is not yet a standard line replace old - # line by a standard line - else: - # number of parallel standard lines could be calculated - # following [2] p.103; for now number of parallel - # standard lines is iterated - edisgo_obj.topology.change_line_type([crit_line_name], standard_line) lines_changes[crit_line_name] = 1 # if node_2_3 is not a representative, disconnect line diff --git a/edisgo/io/db.py b/edisgo/io/db.py index fd12673af..cc3b375d3 100644 --- a/edisgo/io/db.py +++ b/edisgo/io/db.py @@ -36,6 +36,33 @@ logger = logging.getLogger(__name__) +#: Default location of the egon-data SSH tunnel configuration file. Used when +#: no explicit config path is passed and the connection mode is not forced. +#: Can be overridden through the ``EGON_DATA_CONFIG`` environment variable. +DEFAULT_EGON_DATA_CONFIG = "~/.ssh/egon-data.configuration.yaml" + + +def default_config_path() -> Path | None: + """ + Return the path to the egon-data SSH configuration file, or ``None``. + + The location is read from the ``EGON_DATA_CONFIG`` environment variable and + falls back to :data:`DEFAULT_EGON_DATA_CONFIG` + (``~/.ssh/egon-data.configuration.yaml``). ``None`` is returned when the + resolved path does not point to an existing file, which callers use as the + signal to fall back to the Open Energy Platform (OEP). + + Returns + ------- + pathlib.Path or None + Path to an existing egon-data configuration file, or ``None`` if none + was found. + + """ + raw = os.environ.get("EGON_DATA_CONFIG", DEFAULT_EGON_DATA_CONFIG) + path = Path(raw).expanduser() + return path if path.is_file() else None + def config_settings(path: Path | str) -> dict[str, dict[str, str | int | Path]]: """ @@ -155,8 +182,16 @@ def ssh_tunnel(cred: dict) -> str: server = SSHTunnelForwarder( ssh_address_or_host=(cred["SSH_HOST"], 22), ssh_username=cred["SSH_USER"], - ssh_pkey=cred["SSH_PKEY"], + # SSHTunnelForwarder only accepts a string path (or a loaded paramiko + # PKey) here. Passing the pathlib.Path produced by credentials() makes + # sshtunnel silently ignore the key and fall back to the default keys + # in ~/.ssh, which fails authentication against the gateway. + ssh_pkey=str(cred["SSH_PKEY"]), remote_bind_address=(cred["PGRES_HOST"], cred["PORT"]), + # Keep the SSH transport alive during long idle periods (e.g. a + # multi-minute OPF between database queries in multi-grid eGo runs) so + # the tunnel is not torn down and connections stay usable. + set_keepalive=30.0, ) server.start() @@ -172,12 +207,17 @@ def engine( Parameters ---------- path : str or pathlib.Path, optional (default=None) - Path to configuration YAML file of egon-data database. + Path to configuration YAML file of egon-data database. Only used when + ``ssh=True``. If None, the default location is used + (``EGON_DATA_CONFIG`` environment variable or + ``~/.ssh/egon-data.configuration.yaml``, see + :func:`default_config_path`). ssh : bool (default=False) - If False, connects to the remote Open Energy Platform database (using the - token, see parameter `token`). If True, establishes an ssh tunnel to a local - egon-data database using the connection information in the configuration YAML - given through `path`. + If False, connects to the remote Open Energy Platform database (using + the token, see parameter `token`). If True, establishes an ssh tunnel + to a local egon-data database using the connection information in the + configuration YAML given through `path` (or the default location if + `path` is None). token : str or pathlib.Path, optional (default=None) Token for database connection or path to text file containing token. If empty the default token file in the config folder OEP_TOKEN.txt @@ -239,6 +279,15 @@ def engine( echo=False, ) + if path is None: + path = default_config_path() + if path is None: + raise ValueError( + "SSH connection requested but no egon-data configuration file " + "was found (checked the EGON_DATA_CONFIG environment variable " + f"and the default location {DEFAULT_EGON_DATA_CONFIG})." + ) + cred = credentials(path=path) local_port = ssh_tunnel(cred) @@ -247,9 +296,63 @@ def engine( f"{cred['POSTGRES_PASSWORD']}@{cred['PGRES_HOST']}:" f"{local_port}/{cred['POSTGRES_DB']}", echo=False, + # This engine is typically cached and reused across many long-running + # tasks/grids (e.g. one eGo run computes grid after grid, each with a + # multi-minute OPF during which the pooled connection sits idle). The + # server or SSH tunnel closes such idle connections, so a later grid + # would otherwise get a dead connection ("server closed the connection + # unexpectedly"). pool_pre_ping validates (and transparently replaces) + # a connection before use; pool_recycle proactively drops connections + # older than an hour. + pool_pre_ping=True, + pool_recycle=3600, ) +def engine_from_settings(database: dict | None = None) -> Engine: + """ + Build a database engine from a scenario ``database`` settings section. + + This maps the data source configured in the scenario JSON onto + :func:`engine`. Recognised keys of `database`: + + * ``source`` — ``"local"`` connects to a local egon-data database through + an SSH tunnel; ``"oep"`` (or a missing/empty value) connects to the + remote Open Energy Platform (OEP), i.e. the previous default behaviour. + * ``config_path`` — optional path to the egon-data configuration YAML. + Only relevant for ``source="local"``. If omitted, the default location + is used (``EGON_DATA_CONFIG`` environment variable or + ``~/.ssh/egon-data.configuration.yaml``, see :func:`default_config_path`). + + Parameters + ---------- + database : dict or None + The ``database`` section of the scenario configuration. If None or + empty, an OEP engine is returned. + + Returns + ------- + :sqlalchemy:`sqlalchemy.Engine` + Database engine. + + """ + database = database or {} + source = str(database.get("source") or "oep").lower() + + if source in ("local", "ssh", "egon-data", "egon_data"): + # config_path may be given explicitly; otherwise engine() falls back to + # the default location (~/.ssh/egon-data.configuration.yaml). + config_path = database.get("config_path") or database.get("credentials_path") + logger.info( + f"engine_from_settings: source='local', using egon-data database " + f"via SSH tunnel (config {config_path or 'default (~/.ssh/...)'})." + ) + return engine(path=config_path, ssh=True) + + logger.info("engine_from_settings: source='oep', connecting to the OEP.") + return engine(ssh=False) + + @contextmanager def session_scope_egon_data(engine: Engine): """Provide a transactional scope around a series of operations.""" diff --git a/edisgo/io/generators_import.py b/edisgo/io/generators_import.py index c22becae1..e5ea65731 100755 --- a/edisgo/io/generators_import.py +++ b/edisgo/io/generators_import.py @@ -1030,24 +1030,9 @@ def _integrate_pv_rooftop(edisgo_object, pv_rooftop_df): MaStR ID of the PV plant. """ - # match building ID to existing solar generators - loads_df = edisgo_object.topology.loads_df - busses_building_id = ( - loads_df[loads_df.type == "conventional_load"] - .drop_duplicates(subset=["building_id"]) - .set_index("bus") - .loc[:, ["building_id"]] - ) gens_df = edisgo_object.topology.generators_df[ edisgo_object.topology.generators_df.subtype == "pv_rooftop" ].copy() - gens_df_building_id = gens_df.loc[:, ["bus"]].join( - busses_building_id, how="left", on="bus" - ) - # using update to make sure to not overwrite existing building ID information - if "building_id" not in gens_df.columns: - gens_df["building_id"] = None - gens_df.update(gens_df_building_id, overwrite=False) # remove decommissioned PV rooftop plants gens_decommissioned = gens_df[ diff --git a/edisgo/io/heat_pump_import.py b/edisgo/io/heat_pump_import.py index c796f9f89..b95867a0b 100644 --- a/edisgo/io/heat_pump_import.py +++ b/edisgo/io/heat_pump_import.py @@ -337,8 +337,13 @@ def _get_individual_heat_pump_capacity(): ["egon_map_zensus_mvgd_buildings", "egon_map_zensus_weather_cell"], "boundaries", ) + # egon_etrago_bus/egon_etrago_link live in schema "grid" in a local + # egon-data database, whereas the OEP path resolves them via the table/schema + # alias mapping keyed on "supply". Only switch the schema for the local + # (SSH/psycopg2) backend; keep "supply" for the remote OEP. + etrago_schema = "supply" if "openenergyplatform" in str(engine.url) else "grid" egon_etrago_bus, egon_etrago_link = config.import_tables_from_oep( - engine, ["egon_etrago_bus", "egon_etrago_link"], "supply" + engine, ["egon_etrago_bus", "egon_etrago_link"], etrago_schema ) building_ids = edisgo_object.topology.loads_df.building_id.unique() diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index cba6ce944..fbf6457e1 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -311,17 +311,20 @@ def from_powermodels( ] results = pd.DataFrame(index=timesteps, columns=names, data=data) if (flex == "gen_nd") & (pm["nw"]["1"]["opf_version"] in [3, 4]): - edisgo_object.timeseries._generators_active_power.loc[:, names] = ( + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._generators_active_power.loc[ti, names] = ( edisgo_object.timeseries.generators_active_power.loc[:, names].values - results[names].values ) elif flex in ["heatpumps", "electromobility"]: - edisgo_object.timeseries._loads_active_power.loc[:, names] = results[ + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._loads_active_power.loc[ti, names] = results[ names ].values elif flex == "dsm": - edisgo_object.timeseries._loads_active_power.loc[:, names] = ( - edisgo_object.timeseries._loads_active_power.loc[:, names].values + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._loads_active_power.loc[ti, names] = ( + edisgo_object.timeseries._loads_active_power.loc[ti, names].values + results[names].values ) elif flex == "storage": @@ -333,8 +336,9 @@ def from_powermodels( data=results[names].values, ) else: + ti = edisgo_object.timeseries.timeindex edisgo_object.timeseries._storage_units_active_power.loc[ - :, names + ti, names ] = results[names].values except AttributeError: setattr( @@ -366,15 +370,27 @@ def from_powermodels( # calculate relative error df2 = deepcopy(df) for flex in df2.columns: - abs_error = abs(df2[flex].values - hv_flex_dict[flex].values) - rel_error = [ - ( + if isinstance(hv_flex_dict[flex], pd.Series): + abs_error = abs(df2[flex].values - hv_flex_dict[flex].values) + rel_error = [ abs_error[i] / hv_flex_dict[flex].iloc[i] if ((abs_error > 0.01)[i] & (hv_flex_dict[flex].iloc[i] != 0)) else 0 + for i in range(len(abs_error)) + ] + else: + abs_error = abs( + df2[flex].values - hv_flex_dict[flex].sum(axis=1).values ) - for i in range(len(abs_error)) - ] + rel_error = [ + abs_error[i] / hv_flex_dict[flex].sum(axis=1).iloc[i] + if ( + (abs_error > 0.01)[i] + & (hv_flex_dict[flex].sum(axis=1).iloc[i] != 0) + ) + else 0 + for i in range(len(abs_error)) + ] df2[flex] = rel_error # write results to edisgo object edisgo_object.opf_results.overlying_grid = pd.DataFrame( @@ -794,8 +810,8 @@ def _build_branch(edisgo_obj, psa_net, pm, flexible_storage_units, s_base): # only modify r, x and l values if min value is too small branches[par] = val.clip(lower=min_value) logger.warning( - f"Min value of {text} is too small. Lowest {100 * quant}% of {text} values will be set " - f"to {min_value} {unit}" + f"Min value of {text} is too small. Lowest {100 * quant}% of " + f"{text} values will be set to {min_value} {unit}" ) for branch_i in np.arange(len(branches.index)): @@ -940,8 +956,8 @@ def _build_load( pf, sign = _get_pf(edisgo_obj, pm, idx_bus, "charging_point") else: logger.warning( - f"No type specified for load {loads_df.index[load_i]}. Power factor and sign will" - "be set for conventional load." + f"No type specified for load {loads_df.index[load_i]}. " + "Power factor and sign will be set for conventional load." ) pf, sign = _get_pf(edisgo_obj, pm, idx_bus, "conventional_load") p_d = psa_net.loads_t.p_set[loads_df.index[load_i]] @@ -1018,9 +1034,18 @@ def _build_battery_storage( """ branches = pd.concat([psa_net.lines, psa_net.transformers]) if not edisgo_obj.overlying_grid.storage_units_soc.empty: + # Align the SOC series (which may use another year) onto the edisgo + # time index plus one end-of-period step. Uses reindex, so a missing + # step yields NaN instead of a KeyError. + from edisgo.tools.tools import align_series_to_timeindex + + soc_aligned = align_series_to_timeindex( + edisgo_obj.overlying_grid.storage_units_soc, + edisgo_obj.timeseries.timeindex, + extra_step=True, + ) data = pd.concat( - [edisgo_obj.overlying_grid.storage_units_soc] - * len(edisgo_obj.topology.storage_units_df), + [soc_aligned] * len(edisgo_obj.topology.storage_units_df), axis=1, ).values else: @@ -1041,6 +1066,18 @@ def _build_battery_storage( * edisgo_obj.topology.storage_units_df.max_hours ) + # The end-of-period SoC step (timeindex[-1] + freq) is only used as the OPF + # boundary (soc_end) and is not an optimized time step. When the time index + # is a reduced, non-contiguous selection, that step can fall in a gap and be + # missing from the source SoC series (which only carried a trailing step for + # the very last interval), leaving it NaN. A NaN boundary makes the Julia OPF + # fail with "Inf - Inf". Forward-fill (then back-fill) so the boundary takes + # the interval's last valid SoC — a harmless approximation for a throwaway + # scaffolding step. + edisgo_obj.overlying_grid.storage_units_soc = ( + edisgo_obj.overlying_grid.storage_units_soc.ffill().bfill() + ) + for stor_i in np.arange(len(flexible_storage_units)): idx_bus = _mapping( psa_net, @@ -1144,7 +1181,16 @@ def _build_electromobility(edisgo_obj, psa_net, pm, s_base, flexible_cps): Updated array containing all charging points that allow for flexible charging. """ - flex_bands_df = edisgo_obj.electromobility.flexibility_bands + # Align to the active timeindex explicitly rather than relying on + # flexibility_bands already being in the same order/length - correct + # regardless of what ran before this function (e.g. a second + # select_timesteps step, or direct EDisGo API use), not just when the + # standard run pipeline's task ordering happens to keep them aligned. + timeindex = edisgo_obj.timeseries.timeindex + flex_bands_df = { + key: df.loc[timeindex] + for key, df in edisgo_obj.electromobility.flexibility_bands.items() + } if (flex_bands_df["lower_energy"] > flex_bands_df["upper_energy"]).any().any(): logger.warning( "Upper energy level is smaller than lower energy level for " @@ -1226,9 +1272,10 @@ def _build_heatpump(psa_net, pm, edisgo_obj, s_base, flexible_hps): comparison = (heat_df2[hp_p_nom.index] > hp_cop * hp_p_nom.squeeze()).any() if comparison.any(): logger.warning( - "Heat demand is higher than rated heatpump power" - f" of heatpumps: {comparison.index[comparison.values].values}. Demand can not be covered if no sufficient" - " heat storage capacities are available." + "Heat demand is higher than rated heatpump power of heatpumps: " + f"{comparison.index[comparison.values].values}. " + "Demand can not be covered if no sufficient heat storage " + "capacities are available." ) for hp_i in np.arange(len(heat_df.index)): idx_bus = _mapping(psa_net, edisgo_obj, heat_df.bus.iloc[hp_i]) @@ -1336,6 +1383,12 @@ def _build_heat_storage(psa_net, pm, edisgo_obj, s_base, flexible_hps, opf_versi edisgo_obj.overlying_grid.heat_storage_units_soc = pd.concat( [df_decentral, df_central], axis=1 ) + # Fill the end-of-period boundary SoC step (see storage note above) so a + # reduced, non-contiguous time index does not leave a NaN boundary that + # breaks the Julia OPF. + edisgo_obj.overlying_grid.heat_storage_units_soc = ( + edisgo_obj.overlying_grid.heat_storage_units_soc.ffill().bfill() + ) heat_storage_df = heat_storage_df.loc[flexible_hps] for stor_i in np.arange(len(flexible_hps)): @@ -1595,11 +1648,18 @@ def _build_hv_requirements( ) for i in np.arange(len(opf_flex)): - pm["HV_requirements"][str(i + 1)] = { - "P": hv_flex_dict[opf_flex[i]].iloc[0], - "name": opf_flex[i], - "count": count, - } + if isinstance(hv_flex_dict[opf_flex[i]], pd.DataFrame): + pm["HV_requirements"][str(i + 1)] = { + "P": hv_flex_dict[opf_flex[i]].sum(axis=1).iloc[0], + "name": opf_flex[i], + "count": count, + } + else: + pm["HV_requirements"][str(i + 1)] = { + "P": hv_flex_dict[opf_flex[i]].iloc[0], + "name": opf_flex[i], + "count": count, + } def _build_timeseries( @@ -1869,21 +1929,27 @@ def _build_component_timeseries( } elif kind == "electromobility": if len(flexible_cps) > 0: + # Align to the active timeindex explicitly (see + # _build_electromobility for the same fix and its rationale) - + # also guards against a length mismatch with + # pm["time_series"]["num_steps"] (set from len(psa_net.snapshots) + # independently of flexibility_bands' own length). + timeindex = edisgo_obj.timeseries.timeindex p_set = ( - edisgo_obj.electromobility.flexibility_bands["upper_power"][ - flexible_cps + edisgo_obj.electromobility.flexibility_bands["upper_power"].loc[ + timeindex, flexible_cps ] / s_base ).round(20) e_min = ( - edisgo_obj.electromobility.flexibility_bands["lower_energy"][ - flexible_cps + edisgo_obj.electromobility.flexibility_bands["lower_energy"].loc[ + timeindex, flexible_cps ] / s_base ).round(20) e_max = ( - edisgo_obj.electromobility.flexibility_bands["upper_energy"][ - flexible_cps + edisgo_obj.electromobility.flexibility_bands["upper_energy"].loc[ + timeindex, flexible_cps ] / s_base ).round(20) @@ -1929,9 +1995,14 @@ def _build_component_timeseries( if (kind == "HV_requirements") & (pm["opf_version"] in [3, 4]): for i in np.arange(len(opf_flex)): - pm_comp[(str(i + 1))] = { - "P": hv_flex_dict[opf_flex[i]].round(20).tolist(), - } + if isinstance(hv_flex_dict[opf_flex[i]], pd.DataFrame): + pm_comp[(str(i + 1))] = { + "P": hv_flex_dict[opf_flex[i]].sum(axis=1).round(20).tolist(), + } + else: + pm_comp[(str(i + 1))] = { + "P": hv_flex_dict[opf_flex[i]].round(20).tolist(), + } pm["time_series"][kind] = pm_comp diff --git a/edisgo/network/dsm.py b/edisgo/network/dsm.py index a19258752..2b9c97838 100644 --- a/edisgo/network/dsm.py +++ b/edisgo/network/dsm.py @@ -19,6 +19,8 @@ import numpy as np import pandas as pd +from edisgo.tools import tools + logger = logging.getLogger(__name__) @@ -152,6 +154,37 @@ def _attributes(self): "e_max", ] + def resample(self, method: str = "ffill", freq: str | pd.Timedelta = "15min"): + """ + Resamples DSM potential time series to a desired resolution. + + Both up- and down-sampling methods are possible. + + Parameters + ---------- + method : str, optional + See :attr:`~.EDisGo.resample_timeseries` for more information. + + freq : str, optional + See :attr:`~.EDisGo.resample_timeseries` for more information. + + Notes + ----- + A gapped index is resampled per contiguous run - see + :func:`edisgo.tools.tools.split_into_contiguous_runs`. + + """ + for attr in self._attributes: + attr_index = getattr(self, attr).index + if len(attr_index) < 2: + logger.debug( + f"{attr} cannot be resampled as it contains less than two " + f"time steps." + ) + else: + freq_orig = attr_index[1] - attr_index[0] + tools.resample(self, freq_orig, method, freq, attr_to_resample=[attr]) + def reduce_memory( self, attr_to_reduce=None, diff --git a/edisgo/network/electromobility.py b/edisgo/network/electromobility.py index 8237febff..55e4d1a59 100644 --- a/edisgo/network/electromobility.py +++ b/edisgo/network/electromobility.py @@ -23,6 +23,7 @@ from sklearn import preprocessing from edisgo.network.components import PotentialChargingParks +from edisgo.tools.tools import align_series_to_timeindex if "READTHEDOCS" not in os.environ: import geopandas as gpd @@ -396,6 +397,21 @@ def get_flexibility_bands( for more information. To avoid this behaviour, set `tol` to 0.0. Default: 1e-6. + Notes + ----- + The bands are always built spanning SimBEV's own native calendar and + simulated date range (independent of ``edisgo_obj.timeseries.timeindex`` + - a charging process straddling a later window's boundary must still + count toward the band inside that window). If + ``edisgo_obj.timeseries.timeindex`` is non-empty, the returned/stored + bands are then year-aligned (SimBEV's calendar is commonly a fixed + reference year, independent of the scenario year) and trimmed to + exactly that timeindex - this is done regardless of `resample`, since + it is a correctness fix (avoiding a ``KeyError`` when a consumer later + indexes the bands by ``edisgo_obj.timeseries.timeindex``), not an + optional resampling convenience. When the timeindex is empty, the + bands are returned untouched, spanning SimBEV's own range/calendar. + Returns -------- dict(str, :pandas:`pandas.DataFrame`) @@ -473,6 +489,91 @@ def get_flexibility_bands( start=start_date, periods=t_max + 1, freq=f"{stepsize}min" ) + # Reduce to only the charging processes actually needed for + # edisgo_obj.timeseries.timeindex, instead of always building over + # every process regardless of how much shorter the active timeindex + # is (see ADR 0002). SimBEV's calendar (start_date) is typically a + # fixed reference year independent of the scenario year, so the + # active timeindex is inverse year-shifted onto that calendar to + # find which SimBEV steps are actually relevant - this mirrors the + # forward year-shift `align_series_to_timeindex` already applies to + # the built bands further down. Only events overlapping that window + # are kept (mirrors dumb/reduced/residual: zero overlap -> zero + # contribution, not a special case). + # + # The array itself is NOT truncated to the window's end, only + # (potentially) to the earliest relevant event's start: clamping a + # retained event's true end down to an artificial array boundary + # would push it onto the `end == n_steps - 1` edge case below (a + # pre-existing, unrelated exclusion for events ending exactly on the + # array's last row) for every boundary-straddling event, silently + # dropping their contribution entirely rather than only trimming it. + # The array is instead sized to comfortably cover every retained + # event's true end - the pre-existing final `.loc[edisgo_timeindex]` + # step below still discards whatever trailing rows aren't needed. + edisgo_timeindex = edisgo_obj.timeseries.timeindex + if len(edisgo_timeindex) > 0: + year_diff = flex_band_index[0].year - edisgo_timeindex[0].year + simbev_calendar_timeindex = edisgo_timeindex + pd.DateOffset( + years=year_diff + ) + window_start_step = int( + (simbev_calendar_timeindex.min() - flex_band_index[0]) + / pd.Timedelta(f"{stepsize}min") + ) + window_end_step = int( + (simbev_calendar_timeindex.max() - flex_band_index[0]) + / pd.Timedelta(f"{stepsize}min") + ) + + overlaps_window = ( + self.charging_processes_df.park_end_timesteps >= window_start_step + ) & (self.charging_processes_df.park_start_timesteps <= window_end_step) + relevant_processes = self.charging_processes_df.loc[overlaps_window] + + if relevant_processes.empty: + # No event overlaps the active window at all (e.g. the + # window falls entirely outside SimBEV's simulated range) - + # size the array to the window itself so the final + # `.loc[edisgo_timeindex]` step still produces the expected + # shape (filled with NaN via `align_series_to_timeindex`, + # exactly as the pre-existing full-span build already would + # have for this case), without ever touching t_max, which + # only bounds where real event data can be, not the window. + array_start_step = window_start_step + array_end_step = window_end_step + else: + array_start_step = min( + window_start_step, + int(relevant_processes.park_start_timesteps.min()), + ) + # Cover every retained event's true end (never clamped down + # below it) as well as the active window's own end. +1 extra + # step of padding mirrors the pre-existing `end_date + 1 day` + # padding on the full-span build (see above) - it exists so + # a retained event's true end never lands exactly on the + # array's last row, which would otherwise trip the + # pre-existing `end == n_steps - 1` exclusion below for + # every such event instead of just the rare full-span case + # it originally guarded against. + array_end_step = ( + max( + window_end_step, + int(relevant_processes.park_end_timesteps.max()), + ) + + 1 + ) + + flex_band_index = pd.date_range( + start=flex_band_index[0] + + pd.Timedelta(f"{array_start_step * stepsize}min"), + periods=max(array_end_step - array_start_step + 1, 0), + freq=f"{stepsize}min", + ) + else: + relevant_processes = self.charging_processes_df + array_start_step = 0 + # set up bands n_steps = len(flex_band_index) tmp_idx = range(n_steps) @@ -486,14 +587,21 @@ def get_flexibility_bands( # map every charging process to the column (charging point) it belongs to; # processes of charging points outside `cps` map to -1 and are dropped park_to_cp = self.integrated_charging_parks_df["edisgo_id"] - proc = self.charging_processes_df + proc = relevant_processes col = cps.index.get_indexer(proc["charging_park_id"].map(park_to_cp)) - end_all = proc["park_end_timesteps"].to_numpy() + # shift into the (possibly truncated) array's own step space - never + # clamped, since the array was sized to cover every retained event's + # true end above + start_all = proc["park_start_timesteps"].to_numpy() - array_start_step + end_all = proc["park_end_timesteps"].to_numpy() - array_start_step # the last time step can lead to problems --> skip those processes keep = (col >= 0) & (end_all != n_steps - 1) col = col[keep] - sub = proc.loc[keep] + sub = proc.loc[keep].assign( + park_start_timesteps=start_all[keep], + park_end_timesteps=end_all[keep], + ) start = sub["park_start_timesteps"].to_numpy().astype(int) end = sub["park_end_timesteps"].to_numpy().astype(int) power = sub["nominal_charging_capacity_kW"].to_numpy(dtype=float) @@ -582,15 +690,26 @@ def get_flexibility_bands( # sanity check self.check_integrity() - # check time index + + # Scope the bands to edisgo_obj's own timeindex, so this method is + # correct regardless of caller (not just the run pipeline, which + # previously had to patch this up itself via + # reduce_timeseries_data_to_given_timeindex right after calling this + # method). The bands built above always span SimBEV's own native + # calendar (its start_date, typically a fixed reference year like + # 2011) and simulated range - independent of edisgo_timeindex, which + # is why this can't just be a `.loc[edisgo_timeindex]` here: a year + # mismatch alone would raise KeyError, and a shorter/different-range + # edisgo_timeindex would too. align_series_to_timeindex year-shifts + # and reindexes (filling any still-missing steps with NaN rather than + # raising) before the final trim below. if len(edisgo_timeindex) > 0: - missing_indices = [_ for _ in edisgo_timeindex if _ not in flex_band_index] - if len(missing_indices) > 0: - logger.warning( - "There are time steps in timeindex of TimeSeries object that " - "are not in the index of the flexibility bands. This may lead " - "to problems." - ) + for key, df in self.flexibility_bands.items(): + if not df.empty: + self.flexibility_bands[key] = align_series_to_timeindex( + df, edisgo_timeindex + ).loc[edisgo_timeindex] + return self.flexibility_bands def fix_flexibility_bands_rounding_errors(self, tol=1e-6): diff --git a/edisgo/network/heat.py b/edisgo/network/heat.py index 414a1b5b3..624a2ac48 100644 --- a/edisgo/network/heat.py +++ b/edisgo/network/heat.py @@ -602,6 +602,11 @@ def resample_timeseries( freq : str, optional See :attr:`~.EDisGo.resample_timeseries` for more information. + Notes + ----- + A gapped index is resampled per contiguous run - see + :func:`edisgo.tools.tools.split_into_contiguous_runs`. + """ for attr in self._timeseries_attributes: attr_index = getattr(self, attr).index diff --git a/edisgo/network/overlying_grid.py b/edisgo/network/overlying_grid.py index df9339071..1eb5fd744 100644 --- a/edisgo/network/overlying_grid.py +++ b/edisgo/network/overlying_grid.py @@ -95,6 +95,18 @@ def __init__(self, **kwargs): "feedin_district_heating", pd.DataFrame(dtype="float64") ) + self.dispatchable_generators_active_power = kwargs.get( + "dispatchable_generators_active_power", pd.DataFrame(dtype="float64") + ) + + self.dispatchable_generators_reactive_power = kwargs.get( + "dispatchable_generators_reactive_power", pd.DataFrame(dtype="float64") + ) + + self.renewables_potential = kwargs.get( + "renewables_potential", pd.Series(dtype="float64") + ) + @property def _attributes(self): return [ @@ -108,6 +120,9 @@ def _attributes(self): "heat_pump_central_active_power", "thermal_storage_units_central_soc", "feedin_district_heating", + "dispatchable_generators_active_power", + "dispatchable_generators_reactive_power", + "renewables_potential", ] def reduce_memory(self, attr_to_reduce=None, to_type="float32"): @@ -256,6 +271,11 @@ def resample(self, method: str = "ffill", freq: str | pd.Timedelta = "15min"): freq : str, optional See :attr:`~.EDisGo.resample_timeseries` for more information. + Notes + ----- + A gapped index is resampled per contiguous run - see + :func:`edisgo.tools.tools.split_into_contiguous_runs`. + """ # get frequency of time series data timeindex = [] diff --git a/edisgo/network/timeseries.py b/edisgo/network/timeseries.py index 0c4bc9659..f07ebfd22 100644 --- a/edisgo/network/timeseries.py +++ b/edisgo/network/timeseries.py @@ -23,7 +23,12 @@ from edisgo.flex_opt import q_control from edisgo.io import timeseries_import -from edisgo.tools.tools import assign_voltage_level_to_component, resample +from edisgo.tools.tools import ( + assign_voltage_level_to_component, + check_timeindex_coverage, + resample, + split_into_contiguous_runs, +) if TYPE_CHECKING: from edisgo import EDisGo @@ -1264,6 +1269,14 @@ def predefined_fluctuating_generators_by_technology( `ts_generators` is 'oedb' and new ding0 grids with geo-referenced LV grids are used. + Notes + ----- + When `ts_generators` is a self-provided DataFrame and a timeindex is + already set on `edisgo_object`, its index must cover that timeindex - + a `ValueError` is raised naming any missing time steps, rather than + silently writing a partially-covering series. Not checked for the + `'oedb'` option, which is already scoped to the timeindex. + """ # in case time series from oedb are used, retrieve oedb time series if isinstance(ts_generators, str) and ts_generators == "oedb": @@ -1279,6 +1292,13 @@ def predefined_fluctuating_generators_by_technology( raise ValueError( "'ts_generators' must either be a pandas DataFrame or 'oedb'." ) + else: + # self-provided DataFrame - the oedb path above is already scoped + # to edisgo_object's timeindex by feedin_oedb/feedin_oedb_legacy + if not edisgo_object.timeseries.timeindex.empty: + check_timeindex_coverage( + edisgo_object.timeseries.timeindex, "ts_generators", ts_generators + ) # set generator_names if None if generator_names is None: @@ -1361,9 +1381,20 @@ def predefined_dispatchable_generators_by_technology( 'other', all dispatchable generators in the network (i.e. all but solar and wind generators) are used. + Notes + ----- + If a timeindex is already set on `edisgo_object`, `ts_generators`' + index must cover it - a `ValueError` is raised naming any missing + time steps, rather than silently writing a partially-covering + series. + """ if not isinstance(ts_generators, pd.DataFrame): raise ValueError("'ts_generators' must be a pandas DataFrame.") + if not edisgo_object.timeseries.timeindex.empty: + check_timeindex_coverage( + edisgo_object.timeseries.timeindex, "ts_generators", ts_generators + ) # write to TimeSeriesRaw for col in ts_generators: @@ -1444,6 +1475,14 @@ def predefined_conventional_loads_by_sector( in :func:`edisgo.io.timeseries_import.load_time_series_demandlib` for more information. + Notes + ----- + When `ts_loads` is a self-provided DataFrame and a timeindex is + already set on `edisgo_object`, its index must cover that timeindex - + a `ValueError` is raised naming any missing time steps, rather than + silently writing a partially-covering series. Not checked for the + `'demandlib'` option, which is already scoped to the timeindex. + """ # in case time series from demandlib are used, retrieve demandlib time series if isinstance(ts_loads, str) and ts_loads == "demandlib": @@ -1457,6 +1496,12 @@ def predefined_conventional_loads_by_sector( elif ts_loads.empty: logger.warning("The profile you entered is empty. Method is skipped.") return + elif not edisgo_object.timeseries.timeindex.empty: + # self-provided DataFrame - the demandlib path above is already + # scoped to edisgo_object's timeindex by load_time_series_demandlib + check_timeindex_coverage( + edisgo_object.timeseries.timeindex, "ts_loads", ts_loads + ) # write to TimeSeriesRaw for col in ts_loads: @@ -1520,12 +1565,23 @@ def predefined_charging_points_by_use_case( If None, all charging points of use cases for which use-case-specific time series are provided are used. + Notes + ----- + If a timeindex is already set on `edisgo_object`, `ts_loads`' index + must cover that timeindex - a `ValueError` is raised naming any + missing time steps, rather than silently writing a + partially-covering series. + """ if not isinstance(ts_loads, pd.DataFrame): raise ValueError("'ts_loads' must be a pandas DataFrame.") elif ts_loads.empty: logger.warning("The profile you entered is empty. Method is skipped.") return + elif not edisgo_object.timeseries.timeindex.empty: + check_timeindex_coverage( + edisgo_object.timeseries.timeindex, "ts_loads", ts_loads + ) # write to TimeSeriesRaw for col in ts_loads: @@ -2270,20 +2326,33 @@ def resample(self, method: str = "ffill", freq: str | pd.Timedelta = "15min"): resample(self, freq_orig, method, freq) - # create new index - if pd.Timedelta(freq) < freq_orig: # up-sampling - index = pd.date_range( - self.timeindex[0], - self.timeindex[-1] + freq_orig, - freq=freq, - inclusive="left", - ) - else: # down-sampling - index = pd.date_range( - self.timeindex[0], - self.timeindex[-1], - freq=freq, - ) + # Rebuild the new index per contiguous run of the original timeindex + # (mirroring how `resample()` above already resamples the data + # per-run) and union them back together, so a gap in the original + # timeindex (e.g. from `select_timesteps` in auto mode) is preserved + # here too, rather than bridged by one date_range(first, last, freq) + # span. + freq_td = pd.Timedelta(freq) + new_indices = [] + for run in split_into_contiguous_runs( + pd.DataFrame(index=self.timeindex), freq_orig + ): + if freq_td < freq_orig: # up-sampling + new_indices.append( + pd.date_range( + run.index[0], + run.index[-1] + freq_orig, + freq=freq, + inclusive="left", + ) + ) + else: # down-sampling + new_indices.append( + pd.date_range(run.index[0], run.index[-1], freq=freq) + ) + index = new_indices[0] + for other in new_indices[1:]: + index = index.union(other) # set new timeindex self._timeindex = index diff --git a/edisgo/opf/eDisGo_OPF.jl/Main.jl b/edisgo/opf/eDisGo_OPF.jl/Main.jl index 0084b7fd7..4b0413328 100644 --- a/edisgo/opf/eDisGo_OPF.jl/Main.jl +++ b/edisgo/opf/eDisGo_OPF.jl/Main.jl @@ -66,25 +66,34 @@ function optimize_edisgo() println("Starting convex SOC AC-OPF with Gurobi.") result_soc, pm = eDisGo_OPF.solve_mn_opf_bf_flex(data_edisgo_mn, SOCBFPowerModelEdisgo, gurobi) #println("Termination status: "*result_soc["termination_status"]) - if result_soc["termination_status"] != MOI.OPTIMAL - # if result_soc["termination_status"] == MOI.SUBOPTIMAL_TERMINATION - # PowerModels.update_data!(data_edisgo_mn, result_soc["solution"]) - # else + # A feasible solution exists if the solver proved optimality OR reports a + # feasible primal point (e.g. SUBOPTIMAL / ALMOST_OPTIMAL under the barrier + # tolerances set above). Only when there is genuinely no primal solution do + # we diagnose the infeasibility via an IIS conflict — calling + # compute_conflict! on a feasible model raises Gurobi error 10015. + has_solution = result_soc["termination_status"] == MOI.OPTIMAL || + MOI.get(pm.model, MOI.PrimalStatus()) == MOI.FEASIBLE_POINT + if !has_solution JuMP.compute_conflict!(pm.model) if MOI.get(pm.model, MOI.ConflictStatus()) == MOI.CONFLICT_FOUND iis_model, _ = copy_conflict(pm.model) print(iis_model) end - #end - elseif result_soc["termination_status"] == MOI.OPTIMAL - # Check if SOC constraint is tight - soc_tight, soc_dict = eDisGo_OPF.check_SOC_equality(result_soc, data_edisgo) - # Save SOC violations if SOC is not tight - if !soc_tight - open(joinpath(results_path, ding0_grid*"_"*join(data_edisgo["flexibilities"])*".json"), "w") do f - write(f, JSON.json(soc_dict)) + else + # Check if SOC constraint is tight (only meaningful for a proven optimum). + soc_tight = true + if result_soc["termination_status"] == MOI.OPTIMAL + soc_tight, soc_dict = eDisGo_OPF.check_SOC_equality(result_soc, data_edisgo) + # Save SOC violations if SOC is not tight + if !soc_tight + open(joinpath(results_path, ding0_grid*"_"*join(data_edisgo["flexibilities"])*".json"), "w") do f + write(f, JSON.json(soc_dict)) + end + println("SOC solution is not tight!") end - println("SOC solution is not tight!") + else + println("SOC model terminated feasible but not optimal ("* + string(result_soc["termination_status"])*"); using the solution.") end PowerModels.update_data!(data_edisgo_mn, result_soc["solution"]) data_edisgo_mn["solve_time"] = result_soc["solve_time"] diff --git a/edisgo/opf/powermodels_opf.py b/edisgo/opf/powermodels_opf.py index 22841f78b..a22b9c3e6 100644 --- a/edisgo/opf/powermodels_opf.py +++ b/edisgo/opf/powermodels_opf.py @@ -9,6 +9,7 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later +import copy import json import logging import os @@ -16,6 +17,7 @@ import sys import numpy as np +import pandas as pd from edisgo.flex_opt import exceptions from edisgo.io.powermodels_io import from_powermodels @@ -23,6 +25,120 @@ logger = logging.getLogger(__name__) +# Time-indexed opf_results attributes that from_powermodels overwrites on each +# call. When the OPF is run separately per interval, these must be concatenated +# across intervals so opf_results covers the full (reduced) time index. Nested +# containers (LineVariables etc.) are listed via their sub-frame attribute names. +_OPF_FLAT_TIME_FRAMES = ( + "slack_generator_t", + "hv_requirement_slacks_t", +) +_OPF_NESTED_TIME_FRAMES = ( + "lines_t", + "heat_storage_t", + "grid_slacks_t", + "battery_storage_t", +) + + +def _with_freq(index): + """Return the DatetimeIndex with its frequency inferred/attached if regular. + + Reducing/uniting time indices drops the ``freq`` attribute; several + downstream consumers (notably the powermodels OPF) do + ``timeindex[-1] + timeindex.freq`` and break on ``freq is None``. This + re-attaches the freq when the index is regularly spaced (a no-op otherwise). + """ + if index.freq is not None or len(index) < 2: + return index + inferred = pd.infer_freq(index) + if inferred is not None: + try: + return pd.DatetimeIndex(index, freq=inferred) + except (ValueError, TypeError): + return index + return index + + +def _contiguous_intervals(timeindex): + """ + Split a time index into contiguous intervals. + + Automatic timestep selection can reduce the time index to disconnected + intervals (e.g. one load-case and one feed-in-case week). This helper detects + the gap(s) so the OPF can be run separately per interval — storage/heat state + does not carry across a gap, so a single OPF over the concatenated steps would + be wrong. + + A boundary is placed wherever the spacing between two consecutive time steps + exceeds the regular step (the smallest spacing in the index). A contiguous + index therefore yields a single interval. Each returned interval has its + ``freq`` restored (set operations that produced the reduced index drop it). + + Parameters + ---------- + timeindex : pandas.DatetimeIndex + + Returns + ------- + list of pandas.DatetimeIndex + One entry per contiguous interval, in chronological order. Empty index + in -> empty list out; a single time step -> one interval. + """ + timeindex = timeindex.sort_values() + if len(timeindex) <= 1: + return [timeindex] if len(timeindex) else [] + diffs = timeindex[1:] - timeindex[:-1] + step = diffs.min() + breaks = [i + 1 for i, d in enumerate(diffs) if d > step] + starts = [0] + breaks + ends = breaks + [len(timeindex)] + return [_with_freq(timeindex[s:e]) for s, e in zip(starts, ends)] + + +def _snapshot_opf_time_frames(opf_results): + """Copy the time-indexed opf_results frames produced by one interval's OPF.""" + snap = {} + for attr in _OPF_FLAT_TIME_FRAMES: + snap[attr] = getattr(opf_results, attr).copy() + for attr in _OPF_NESTED_TIME_FRAMES: + container = getattr(opf_results, attr) + snap[attr] = { + sub: getattr(container, sub).copy() for sub in container._attributes() + } + return snap + + +def _merge_opf_time_frames(opf_results, snapshots): + """ + Concatenate per-interval opf_results snapshots by time index and write them + back onto ``opf_results``, so its detailed frames cover the full reduced + index rather than only the last interval's. + """ + + def _concat(frames): + frames = [f for f in frames if f is not None and not f.empty] + if not frames: + return pd.DataFrame() + return pd.concat(frames).sort_index() + + for attr in _OPF_FLAT_TIME_FRAMES: + setattr(opf_results, attr, _concat([s[attr] for s in snapshots])) + for attr in _OPF_NESTED_TIME_FRAMES: + container = getattr(opf_results, attr) + for sub in container._attributes(): + setattr(container, sub, _concat([s[attr][sub] for s in snapshots])) + + # Recompute the overlying_grid summary (opf_version 3/4) from the merged HV + # requirement slacks, since it is a reduction over the whole time index. + hv = opf_results.hv_requirement_slacks_t + if not hv.empty: + opf_results.overlying_grid = pd.DataFrame( + columns=["Highest error", "Mean error", "Sum error"], + index=hv.columns, + data=pd.concat([hv.max(), hv.mean(), hv.sum()], axis=1).values, + ) + def pm_optimize( edisgo_obj, @@ -37,8 +153,162 @@ def pm_optimize( silence_moi: bool = False, ) -> None: """ - Run OPF for edisgo object in julia subprocess and write results of OPF to edisgo - object. Results of OPF are time series of operation schedules of flexibilities. + Run OPF for the edisgo object and write results back to it. + + If the time index is a single contiguous interval, this runs one OPF + (:func:`_pm_optimize_single`). If the time index is NON-contiguous + (disconnected intervals, e.g. from automatic timestep selection), each + contiguous interval is optimized separately and independently — storage/heat + state does not carry across the gap — and the results are combined: + + * per-interval operation schedules accumulate in ``edisgo.timeseries``; + * the detailed ``edisgo.opf_results`` frames are merged by time index; + * a per-interval solve report is stored in + ``edisgo.opf_results.interval_results``; + * if any interval was infeasible, an + :class:`~.flex_opt.exceptions.InfeasibleModelError` is raised after the + feasible intervals' results have been stored. + + The overlying-grid SOC attributes and reactive-power time series (which + ``to_powermodels`` / ``from_powermodels`` mutate or replace on the current + interval) are snapshotted and restored pristine before each interval so a + later interval sees intact input. Parameters are as for + :func:`_pm_optimize_single`. + """ + opf_kwargs = dict( + s_base=s_base, + flexible_cps=flexible_cps, + flexible_hps=flexible_hps, + flexible_loads=flexible_loads, + flexible_storage_units=flexible_storage_units, + opf_version=opf_version, + method=method, + warm_start=warm_start, + silence_moi=silence_moi, + ) + + intervals = _contiguous_intervals(edisgo_obj.timeseries.timeindex) + if len(intervals) <= 1: + # single contiguous optimization. Re-set the (freq-restored) interval so + # the OPF sees a time index with a frequency — set operations upstream + # (e.g. timestep selection) drop it, and the OPF needs timeindex.freq. + if intervals: + edisgo_obj.set_timeindex(intervals[0]) + _pm_optimize_single(edisgo_obj, **opf_kwargs) + return + + logger.info( + f"pm_optimize: time index has {len(intervals)} disconnected intervals; " + f"running a separate OPF per interval." + ) + full_timeindex = edisgo_obj.timeseries.timeindex + + # Snapshot the shared input state that per-interval OPF runs mutate: + # * overlying-grid SOC attributes are rewritten in place by to_powermodels; + # * the reactive-power time series are fully REPLACED (not .loc-updated) by + # the set_time_series_reactive_power_control() call inside from_powermodels. + # Reactive power was set on the full reduced index before this call; restore + # this input pristine before each interval. Active-power frames are NOT + # restored — they accumulate each interval's OPF results via .loc. + og = edisgo_obj.overlying_grid + og_snapshot = {attr: copy.deepcopy(getattr(og, attr)) for attr in og._attributes} + reactive_attrs = [ + "_generators_reactive_power", + "_loads_reactive_power", + "_storage_units_reactive_power", + ] + reactive_snapshot = { + attr: copy.deepcopy(getattr(edisgo_obj.timeseries, attr, None)) + for attr in reactive_attrs + } + + def _restore_pristine_inputs(): + for attr, value in og_snapshot.items(): + setattr(og, attr, copy.deepcopy(value)) + for attr, value in reactive_snapshot.items(): + if value is not None: + setattr(edisgo_obj.timeseries, attr, copy.deepcopy(value)) + + # Pre-allocate the storage active-power schedule over the FULL reduced index + # so from_powermodels .loc-accumulates each interval's storage result instead + # of replacing the frame with an interval-only one (which would drop earlier + # intervals' storage schedules). + su_names = edisgo_obj.topology.storage_units_df.index + if len(su_names) > 0 and edisgo_obj.timeseries.storage_units_active_power.empty: + edisgo_obj.timeseries.storage_units_active_power = pd.DataFrame( + 0.0, index=full_timeindex, columns=su_names + ) + + snapshots = [] + report = [] + try: + for interval in intervals: + _restore_pristine_inputs() + edisgo_obj.set_timeindex(interval) + entry = { + "start": interval[0], + "end": interval[-1], + "status": None, + "solver": None, + "solution_time": None, + } + try: + _pm_optimize_single(edisgo_obj, **opf_kwargs) + entry["status"] = edisgo_obj.opf_results.status + entry["solver"] = edisgo_obj.opf_results.solver + entry["solution_time"] = edisgo_obj.opf_results.solution_time + snapshots.append(_snapshot_opf_time_frames(edisgo_obj.opf_results)) + except exceptions.InfeasibleModelError: + entry["status"] = "infeasible" + logger.warning( + f"pm_optimize: OPF infeasible for interval " + f"{interval[0]}..{interval[-1]}." + ) + report.append(entry) + finally: + # restore the full (reduced) index so all intervals' schedules are exposed + # and undo the per-interval mutations of the overlying-grid/reactive input. + _restore_pristine_inputs() + edisgo_obj.set_timeindex(full_timeindex) + + _merge_opf_time_frames(edisgo_obj.opf_results, snapshots) + edisgo_obj.opf_results.interval_results = report + solution_times = [ + e["solution_time"] for e in report if e["solution_time"] is not None + ] + edisgo_obj.opf_results.solution_time = ( + sum(solution_times) if solution_times else None + ) + statuses = [e["status"] for e in report] + infeasible = [e for e in report if e["status"] == "infeasible"] + edisgo_obj.opf_results.status = ( + "infeasible" if infeasible else (statuses[0] if statuses else None) + ) + if infeasible: + raise exceptions.InfeasibleModelError( + f"OPF infeasible for {len(infeasible)} of {len(intervals)} time " + f"intervals; see edisgo.opf_results.interval_results. Results for " + f"feasible intervals have been stored." + ) + + +def _pm_optimize_single( + edisgo_obj, + s_base: int = 1, + flexible_cps: np.ndarray | None = None, + flexible_hps: np.ndarray | None = None, + flexible_loads: np.ndarray | None = None, + flexible_storage_units: np.ndarray | None = None, + opf_version: int = 1, + method: str = "soc", + warm_start: bool = False, + silence_moi: bool = False, +) -> None: + """ + Run a single-interval OPF for the edisgo object in a julia subprocess and + write results back to the edisgo object. Assumes the time index is a single + contiguous interval; :func:`pm_optimize` is the public entry point that + handles non-contiguous indices by calling this per interval. Parameters ---------- diff --git a/edisgo/opf/results/opf_result_class.py b/edisgo/opf/results/opf_result_class.py index f109c681e..83ac1e1a7 100644 --- a/edisgo/opf/results/opf_result_class.py +++ b/edisgo/opf/results/opf_result_class.py @@ -176,6 +176,13 @@ class OPFResults: Aggregated exchange with the overlying grid. battery_storage_t : :class:`~.opf.results.opf_result_class.BatteryStorage` Battery-storage results. + interval_results : list of dict + Per-interval solve report, populated when the OPF is run separately over + several disconnected time intervals (see the ``optimize`` pipeline task, + which splits a non-contiguous time index — e.g. from automatic timestep + selection — into independent optimizations). Each entry has keys + ``start``, ``end``, ``status``, ``solver`` and ``solution_time``. Empty + for a single contiguous optimization. """ @@ -190,6 +197,7 @@ def __init__(self): self.grid_slacks_t = GridSlacks() self.overlying_grid = pd.DataFrame() self.battery_storage_t = BatteryStorage() + self.interval_results = [] def to_csv(self, directory, attributes=None): """ diff --git a/edisgo/run/__init__.py b/edisgo/run/__init__.py new file mode 100644 index 000000000..cd202ddef --- /dev/null +++ b/edisgo/run/__init__.py @@ -0,0 +1,31 @@ +""" +YAML/JSON-driven pipeline runner for eDisGo. + +Two entry points share the same core: + + from edisgo.run import run_edisgo + edisgo = run_edisgo("presets/uc2_flex_opf.yaml") + + # or, on an existing EDisGo instance: + edisgo = EDisGo(ding0_grid="30879") + edisgo.run_pipeline("my_run.yaml") + +Pipelines are lists of named tasks from :mod:`edisgo.run.tasks`. Each step +is either a string (``worst_case_ts``) or a single-key mapping with +parameters (``import_electromobility: {charging_strategy: dumb}``). Tasks +can be grouped into ordered ``stages`` that can save artifacts and reload +them with ``load_from``, enabling two-phase workflows (base reinforce + +per-scenario reinforce). +""" + +from edisgo.run.context import RunContext +from edisgo.run.registry import known_tasks, register_task +from edisgo.run.runner import _run_pipeline_on, run_edisgo + +__all__ = [ + "RunContext", + "_run_pipeline_on", + "known_tasks", + "register_task", + "run_edisgo", +] diff --git a/edisgo/run/config.py b/edisgo/run/config.py new file mode 100644 index 000000000..ed50438ff --- /dev/null +++ b/edisgo/run/config.py @@ -0,0 +1,421 @@ +""" +Config loader and schema normalizer for the eDisGo pipeline runner. + +The loader turns a YAML file, JSON file, or Python dict into the +canonical internal schema consumed by :mod:`edisgo.run.runner`. It +handles four concerns in a fixed order: + +1. **Read** — parse YAML/JSON (auto-detected by extension; unknown + extensions are tried as JSON first, then YAML). +2. **extends** — resolve a ``extends:`` key recursively into the + parent config and deep-merge; the child overrides parent keys. The + ``extends:`` value may be a path (relative to the including file) + or a bare preset name (resolved against + :mod:`edisgo.run.presets`). +3. **external_config** — merge machine-specific overrides from an + ``external_config:`` path (typically ``~/.edisgo/secrets.json`` + with DB credentials). Keys in the external file override keys in + the main config. +4. **eGo-legacy adaptation** — if the config looks like an eGo + ``scenario_setting_*.json`` (has top-level ``eDisGo.tasks``), map + it onto the new schema so old eGo configs run unchanged. +5. **Stage normalization** — collapse a flat ``pipeline:`` into a + single-stage ``stages: [{name: main, pipeline: [...]}]`` so the + runner only ever deals with the stage form. + +Only :func:`load_config` is public. Everything else is implementation +detail. +""" +from __future__ import annotations + +import copy +import json +import logging +import os + +from pathlib import Path +from typing import Any + +import yaml + +logger = logging.getLogger("edisgo.run.config") + + +def load_config(cfg_or_path) -> dict[str, Any]: + """ + Load, merge, adapt, and normalize a pipeline config. + + Accepts a path to a YAML/JSON file or a dict. The returned dict + always has the normalized shape expected by the runner: + + * top-level ``stages`` (list of ``{name, pipeline, ...}``) + * ``scenario`` (may be ``None``) + * optional ``grid``, ``database``, ``results`` sections + * no ``pipeline``, ``extends``, or ``external_config`` keys + (they have been consumed) + + Parameters + ---------- + cfg_or_path : str, pathlib.Path, or dict + Either a path to a YAML/JSON config file, or a dict already + holding the config. A dict is deep-copied so the caller's + dict is not mutated. + + Returns + ------- + dict + The fully resolved, normalized config. + + Raises + ------ + FileNotFoundError + If the given path (or an ``extends`` reference) does not + exist. + ValueError + If the config has both ``pipeline`` and ``stages``, missing + ``pipeline``/``stages``, duplicate stage names, or a stage + without ``name``/``pipeline``. + + """ + if isinstance(cfg_or_path, (dict,)): + cfg = copy.deepcopy(cfg_or_path) + base_dir = Path.cwd() + else: + path = Path(cfg_or_path).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f"Config file not found: {path}") + cfg = _read_file(path) + base_dir = path.parent + + cfg = _resolve_extends(cfg, base_dir) + cfg = _apply_external_config(cfg) + cfg = _adapt_ego_legacy(cfg) + cfg = _normalize_stages(cfg) + return cfg + + +def _read_file(path: Path) -> dict[str, Any]: + """ + Parse a YAML or JSON file into a dict. + + Parameters + ---------- + path : pathlib.Path + File path. Extension (``.json``, ``.yaml``, ``.yml``) selects + the parser. Unknown extensions fall back to JSON first, then + YAML. + + Returns + ------- + dict + Parsed config contents. + + """ + text = path.read_text() + suffix = path.suffix.lower() + if suffix == ".json": + return json.loads(text) + if suffix in (".yaml", ".yml"): + return yaml.safe_load(text) + try: + return json.loads(text) + except json.JSONDecodeError: + return yaml.safe_load(text) + + +def _resolve_extends(cfg: dict, base_dir: Path) -> dict: + """ + Resolve an ``extends:`` reference and deep-merge parent into child. + + The parent is loaded recursively, so a chain of ``extends:`` works. + A relative reference is looked up as (1) a path relative to + ``base_dir``, (2) a bundled preset name under + :mod:`edisgo.run.presets`. The child's keys override the parent's on + conflicts. + + Parameters + ---------- + cfg : dict + Child config (may contain ``extends:``). + base_dir : pathlib.Path + Directory against which relative ``extends`` paths are + resolved (usually the directory of the child config). + + Returns + ------- + dict + Merged config with ``extends`` consumed. + + Raises + ------ + FileNotFoundError + If the referenced parent config file does not exist. + + """ + ext = cfg.pop("extends", None) + if ext is None: + return cfg + ext_path = Path(ext).expanduser() + if not ext_path.is_absolute(): + # Resolve relative to the including file first (least surprise: a + # local file next to the config wins), then fall back to a bundled + # preset of that name. + local_path = (base_dir / ext_path).resolve() + if local_path.is_file(): + ext_path = local_path + else: + preset_path = _preset_path(str(ext_path)) + ext_path = preset_path if preset_path is not None else local_path + if not ext_path.is_file(): + raise FileNotFoundError(f"extends: file not found: {ext_path}") + parent = _read_file(ext_path) + parent = _resolve_extends(parent, ext_path.parent) + return _deep_merge(parent, cfg) + + +def _preset_path(name: str) -> Path | None: + """ + Look up a preset YAML/JSON by bare name. + + Searches the ``edisgo/run/presets/`` directory for a file matching + ``name``, ``name.yaml``, ``name.yml``, or ``name.json`` (in that + order). + + Parameters + ---------- + name : str + Preset identifier, e.g. ``"uc2_flex_opf"`` or + ``"presets/uc2_flex_opf.yaml"``. + + Returns + ------- + pathlib.Path or None + The resolved preset path, or ``None`` if no match is found. + + """ + presets_dir = Path(__file__).parent / "presets" + candidates = [ + presets_dir / name, + presets_dir / f"{name}.yaml", + presets_dir / f"{name}.yml", + presets_dir / f"{name}.json", + ] + for c in candidates: + if c.is_file(): + return c + return None + + +def _apply_external_config(cfg: dict) -> dict: + """ + Merge an ``external_config:`` file on top of the current config. + + Used to keep machine-specific secrets (DB credentials, result + directories) out of versioned scenario configs. If the referenced + file does not exist, a warning is logged but the config is used + as-is. + + Parameters + ---------- + cfg : dict + Config possibly containing an ``external_config:`` key. + + Returns + ------- + dict + Merged config with ``external_config`` consumed. + + """ + ext = cfg.pop("external_config", None) + if ext is None: + return cfg + path = Path(os.path.expanduser(ext)) + if not path.is_file(): + logger.warning(f"external_config file not found, skipping: {path}") + return cfg + override = _read_file(path) + return _deep_merge(cfg, override) + + +def _deep_merge(base: dict, override: dict) -> dict: + """ + Recursively merge two dicts, with ``override`` winning on conflicts. + + Nested dicts are merged key-by-key. Non-dict values (including + lists) are replaced wholesale — lists are NOT concatenated, to + keep the merge semantics predictable (otherwise a preset could + silently extend the child's pipeline). + + Parameters + ---------- + base : dict + Parent / lower-priority dict. + override : dict + Child / higher-priority dict. + + Returns + ------- + dict + A new dict holding the merge result. Inputs are not mutated. + + """ + out = copy.deepcopy(base) if base else {} + for key, val in (override or {}).items(): + if ( + key in out + and isinstance(out[key], dict) + and isinstance(val, dict) + ): + out[key] = _deep_merge(out[key], val) + else: + out[key] = copy.deepcopy(val) + return out + + +def _normalize_stages(cfg: dict) -> dict: + """ + Collapse a flat ``pipeline:`` into the canonical ``stages`` shape. + + After this step the runner only has to iterate ``cfg["stages"]``; + flat configs become a single stage named ``main``. + + Parameters + ---------- + cfg : dict + Config with either ``pipeline`` or ``stages`` at the top + level. + + Returns + ------- + dict + Config with ``stages`` guaranteed to be present and + ``pipeline`` removed. + + Raises + ------ + ValueError + If both ``pipeline`` and ``stages`` are present, if neither + is present, if any stage is missing ``name``/``pipeline``, or + if stage names are not unique. + + """ + if "stages" in cfg and "pipeline" in cfg: + raise ValueError( + "Config has both top-level 'pipeline' and 'stages'. " + "Use only one." + ) + if "stages" not in cfg: + pipeline = cfg.pop("pipeline", None) + if pipeline is None: + raise ValueError( + "Config must define either 'pipeline' or 'stages'." + ) + cfg["stages"] = [{"name": "main", "pipeline": pipeline}] + + seen = set() + for stage in cfg["stages"]: + if "name" not in stage: + raise ValueError("Every stage needs a 'name' key.") + if stage["name"] in seen: + raise ValueError( + f"Duplicate stage name: {stage['name']}" + ) + seen.add(stage["name"]) + if "pipeline" not in stage: + raise ValueError( + f"Stage '{stage['name']}' is missing 'pipeline'." + ) + return cfg + + +_EGO_TASK_MAP = { + "1_setup_grid": "setup_grid", + "5_grid_reinforcement": "reinforce", + "4_optimisation": "optimize", + "worst_case_ts": "worst_case_ts", + "base_reinforce": "base_reinforce", + "oedb_ts": "oedb_ts", + "import_heat_pumps_from_db": "import_heat_pumps", + "import_home_batteries_from_db": "import_home_batteries", + "import_dsm_from_db": "import_dsm", + "import_electromobility_from_db": "import_electromobility", + "load_charging_from_files": "load_charging_from_files", + "load_from_base": "load_from_base", +} +"""Mapping from eGo task names to edisgo.run task names. eGo-specific +tasks with no eDisGo equivalent (e.g. ``2_specs_overlying_grid``, +``3_temporal_complexity_reduction``) are intentionally missing — they +require eTraGo and are logged as "skipped" when adapted.""" + + +def _adapt_ego_legacy(cfg: dict) -> dict: + """ + Map an eGo-style ``scenario_setting_*.json`` onto the new schema. + + Recognizes an eGo config by the presence of an ``eDisGo.tasks`` + key at the top level together with the absence of + ``pipeline``/``stages``. Translates: + + * ``eDisGo.grid_path`` → ``grid.ding0_path`` + * ``eDisGo.results`` → ``results.directory`` + * ``eTraGo.scn_name`` → ``scenario`` + * ``eDisGo.tasks`` → ``pipeline`` (via :data:`_EGO_TASK_MAP`) + * top-level ``database``/``ssh`` kept under ``database`` + + eGo-only tasks (overlying grid / temporal reduction) are + dropped with a warning. Cosmetic keys (``eGo``, ``eTraGo``, + ``_comment``, ``_workflow``) are stripped. + + Parameters + ---------- + cfg : dict + Possibly-legacy config. + + Returns + ------- + dict + Adapted config. If the input is not an eGo-legacy config, it + is returned unchanged. + + """ + if "eDisGo" not in cfg or "pipeline" in cfg or "stages" in cfg: + return cfg + + edisgo_cfg = cfg["eDisGo"] + tasks = edisgo_cfg.get("tasks") + if tasks is None: + return cfg + + logger.info( + "Detected legacy eGo config schema — adapting to edisgo.run." + ) + mapped = [] + for t in tasks: + if t not in _EGO_TASK_MAP: + logger.warning( + f"eGo task '{t}' has no eDisGo equivalent — skipping " + "(likely eTraGo-specific)." + ) + continue + mapped.append(_EGO_TASK_MAP[t]) + + adapted: dict[str, Any] = { + "scenario": cfg.get("eTraGo", {}).get("scn_name", "eGon2035"), + "grid": {"ding0_path": edisgo_cfg.get("grid_path")}, + "results": {"directory": edisgo_cfg.get("results")}, + "pipeline": mapped, + "overlying_grid": { + "path": edisgo_cfg.get("overlying_grid_source"), + "selection": edisgo_cfg.get("overlying_grid"), + }, + } + if "database" in cfg: + # Deep-copy so injecting ssh below does not mutate the caller's + # cfg["database"] (which is merged again in _deep_merge afterwards). + adapted["database"] = copy.deepcopy(cfg["database"]) + if "ssh" in cfg: + adapted["database"]["ssh"] = copy.deepcopy(cfg["ssh"]) + for side_key in ("eGo", "eTraGo", "ssh", "_comment", "_workflow"): + cfg.pop(side_key, None) + cfg.pop("eDisGo", None) + return _deep_merge(adapted, cfg) diff --git a/edisgo/run/context.py b/edisgo/run/context.py new file mode 100644 index 000000000..88cc18c7a --- /dev/null +++ b/edisgo/run/context.py @@ -0,0 +1,167 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +""" +Runtime context passed to every task during pipeline execution. + +The context is a small mutable object that threads shared state between +tasks without polluting the :class:`~edisgo.EDisGo` instance itself. +Typical uses: + +* ``scenario`` — the active eGon scenario name (``eGon2035``, + ``eGon100RE``, …) so tasks don't have to re-read it from the config. +* ``engine`` — a SQLAlchemy engine, lazily created on first DB access + via :meth:`RunContext.ensure_engine`. Tasks that don't touch the + database never pay connection cost. +* ``results_dir`` — base directory for stage artifacts and ``save``. +* ``flags`` — free-form boolean/state flags tasks set to coordinate + with each other (``has_heat_pumps``, ``timeseries_set``, …). +* ``stage_artifacts`` — map ``stage_name -> path`` of zip/dir artifacts + emitted by ``save``, consumed by later stages via ``load_from``. + +Tasks should treat ``flags`` as advisory — they MAY short-circuit based +on a flag but MUST NOT assume a flag is present. +""" + +from __future__ import annotations + +import logging + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass +class RunContext: + """ + Mutable per-run state shared across all tasks of a pipeline. + + Attributes + ---------- + scenario : str or None + Active scenario name from the top-level ``scenario:`` key. + engine : sqlalchemy.engine.Engine or None + Database engine for oedb-backed imports. Created lazily; + see :meth:`ensure_engine`. + results_dir : pathlib.Path or None + Base directory for stage outputs. Resolved from + ``results.directory`` in the config. + logger : logging.Logger + Logger instance used by tasks and the runner. Defaults to + the ``edisgo.run`` logger. + flags : dict + Free-form state flags that tasks use to communicate. Common + keys: ``grid_loaded``, ``timeseries_set``, + ``reactive_power_set``, ``has_heat_pumps``, ``has_dsm``, + ``has_home_batteries``, ``has_electromobility``, + ``base_reinforced``, ``last_saved``. + stage_artifacts : dict + Map ``stage_name -> Path`` of save-artifacts. Populated by the + ``save`` task when running inside a named stage, consumed by + subsequent stages that set ``load_from:``. + current_stage : str or None + Name of the stage currently executing. Set by the runner. + raw_config : dict + The fully resolved pipeline config (after ``extends``, + ``external_config``, and eGo-legacy adaptation). Tasks can + read supplementary keys like ``database.*`` from here. + overlying_grid_data : dict or None + Overlying-grid data (e.g. eTraGo results) injected via the + ``overlying_grid_data=`` argument of :func:`edisgo.run.run_edisgo`. + Consumed by the ``import_overlying_grid_data`` task when + ``overlying_grid.source == "etrago"``. + full_grid_stash : edisgo.EDisGo or None + The pre-reduction :class:`~edisgo.EDisGo` instance, deepcopied and + stashed by the ``spatial_reduce`` task before it spatially reduces + the working object. Consumed (and cleared back to ``None``) by + ``spatial_restore``. ``None`` when spatial reduction is not in use. + + """ + + scenario: str | None = None + engine: Any = None + results_dir: Path | None = None + logger: logging.Logger = field( + default_factory=lambda: logging.getLogger("edisgo.run") + ) + flags: dict[str, Any] = field(default_factory=dict) + stage_artifacts: dict[str, Path] = field(default_factory=dict) + current_stage: str | None = None + raw_config: dict[str, Any] = field(default_factory=dict) + overlying_grid_data: Any = None + full_grid_stash: Any = None + + def ensure_engine(self): + """ + Return a database engine, creating it on first call. + + The data source is chosen from the ``database`` section of + :attr:`raw_config`: + + * ``source: "local"`` — egon-data database via SSH tunnel, using + ``config_path`` if given, otherwise the default location + (``~/.ssh/egon-data.configuration.yaml``). + * ``source: "oep"`` or no ``database`` section — remote Open Energy + Platform (previous default behaviour). + + A legacy explicit direct-local database (``host`` given with SSH + disabled) is still honoured for backward compatibility. The engine is + cached on the context so subsequent calls reuse the same connection. + + Returns + ------- + sqlalchemy.engine.Engine + The active database engine. + + """ + if self.engine is not None: + return self.engine + db_cfg = self.raw_config.get("database") or {} + source = str(db_cfg.get("source") or "").lower() + + # Legacy explicit direct local database: SSH disabled and explicit + # connection parameters given (host/port/user/password as passed by + # eGo). Connect straight to that postgres via psycopg2. + ssh_cfg = db_cfg.get("ssh") or {} + ssh_enabled = bool(ssh_cfg.get("enabled", False)) + host = db_cfg.get("host") + if source not in ("local", "oep") and host and not ssh_enabled: + from sqlalchemy import create_engine + + user = db_cfg.get("user") + password = db_cfg.get("password") + port = db_cfg.get("port") + name = db_cfg.get("database_name") or db_cfg.get("database") + self.logger.info( + f"ensure_engine: using local database " + f"{user}@{host}:{port}/{name} (no OEP, no SSH tunnel)." + ) + self.engine = create_engine( + f"postgresql+psycopg2://{user}:{password}@{host}:{port}/{name}", + connect_args={"connect_timeout": 10}, + # The engine is cached and reused across long-running tasks + # (e.g. electromobility can idle the connection for many + # minutes). pool_pre_ping detects connections the server/SSH + # tunnel dropped while idle and transparently reconnects, + # avoiding "server closed the connection unexpectedly". + pool_pre_ping=True, + ) + return self.engine + + # Source-driven engine: source="local" -> egon-data via SSH tunnel + # (config_path or ~/.ssh default), source="oep"/absent -> OEP. + from edisgo.io.db import engine_from_settings + + # A legacy ssh.enabled flag maps to source "local". + if not source and ssh_enabled: + db_cfg = {**db_cfg, "source": "local"} + self.engine = engine_from_settings(db_cfg) + return self.engine diff --git a/edisgo/run/presets/basic.yaml b/edisgo/run/presets/basic.yaml new file mode 100644 index 000000000..136161855 --- /dev/null +++ b/edisgo/run/presets/basic.yaml @@ -0,0 +1,22 @@ +_comment: | + Basic preset: worst-case pre-reinforce → reinforce. + Minimal end-to-end example with no database dependency. + Reproduces the core of example_01 without flex imports. + +_workflow: + - setup_grid: load ding0 topology + - worst_case_ts: set worst-case time series (feed-in + load) + - reactive_power: fix reactive power control + - check_integrity: validate grid consistency + - reinforce: run grid reinforcement + - save: persist topology + timeseries + results + +scenario: eGon2035 + +pipeline: + - setup_grid + - worst_case_ts + - reactive_power + - check_integrity + - reinforce + - save diff --git a/edisgo/run/presets/r4mu_base_and_scenario.yaml b/edisgo/run/presets/r4mu_base_and_scenario.yaml new file mode 100644 index 000000000..6412f36ca --- /dev/null +++ b/edisgo/run/presets/r4mu_base_and_scenario.yaml @@ -0,0 +1,49 @@ +_comment: | + R4MU — two-stage base + scenario reinforcement: + Stage 1 produces a base-reinforced grid (generators + heat pumps) + and saves it as an artifact. Stage 2 loads that artifact, integrates + scenario-specific charging stations from a GeoPackage/CSV directory, + applies worst-case time series, and runs a scenario-specific + reinforce. Cost delta = extra reinforcement caused by the charging + scenario. + +_workflow: + - stage base: + - setup_grid: load ding0 topology + import generators + - import_heat_pumps: from egon_data + - worst_case_ts + - reactive_power + - reinforce + - save (artifact consumed by next stage) + - stage scenario: + - load_from: base + - load_charging_from_files: integrate scenario charging + - worst_case_ts + - reactive_power + - reinforce (delta only) + - save + +scenario: eGon2035 + +stages: + - name: base + pipeline: + - setup_grid: {import_generators: true} + - import_heat_pumps + - worst_case_ts + - reactive_power + - reinforce + - save + - name: scenario + load_from: base + params: + charging_dir: "./charging_scenario_1" + mv_threshold_kw: 100 + pipeline: + - load_charging_from_files: + charging_dir: "{{params.charging_dir}}" + mv_threshold_kw: "{{params.mv_threshold_kw}}" + - worst_case_ts + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc1_loads_worst_case.yaml b/edisgo/run/presets/uc1_loads_worst_case.yaml new file mode 100644 index 000000000..2204d3ce9 --- /dev/null +++ b/edisgo/run/presets/uc1_loads_worst_case.yaml @@ -0,0 +1,32 @@ +_comment: | + UC1 — worst-case flexibility loads: + load grid, base-reinforce (generators only), then import flex assets + (heat pumps, home batteries, DSM, electromobility) and apply worst-case + time series before a final reinforce. Cost delta = extra reinforcement + caused by the new assets under worst-case conditions. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging) + - worst_case_ts: synthetic worst case incl. new assets + - reactive_power: fix reactive power control + - reinforce: final reinforcement — delta only + - save: persist topology + results + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - worst_case_ts + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc2_flex_opf.yaml b/edisgo/run/presets/uc2_flex_opf.yaml new file mode 100644 index 000000000..c09cae87e --- /dev/null +++ b/edisgo/run/presets/uc2_flex_opf.yaml @@ -0,0 +1,43 @@ +_comment: | + UC2 — OPF with full flexibility: + Like UC1 but loads real egon_data time series (oedb) and runs a + powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) + before the final reinforce. Cost delta = extra reinforcement needed + under optimal flex dispatch. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (168 h, 2035) + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - reactive_power + - check_integrity + - optimize: pm_optimize with flex assets (SOC, opf v2) + - reinforce: final reinforcement + - save + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - oedb_ts: + timeindex: {start: "2035-01-01", periods: 168, freq: h} + dispatchable: {other: 0.7} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - check_integrity + - optimize: + flexible: [heat_pumps, storage] + method: soc + opf_version: 2 + - reinforce + - save diff --git a/edisgo/run/presets/uc3_oedb_ts.yaml b/edisgo/run/presets/uc3_oedb_ts.yaml new file mode 100644 index 000000000..59c184cdd --- /dev/null +++ b/edisgo/run/presets/uc3_oedb_ts.yaml @@ -0,0 +1,36 @@ +_comment: | + UC3 — real-world time series without OPF: + Like UC1 but uses real egon_data time series (oedb) instead of + synthetic worst cases. No optimization, no eTraGo. Difference to + UC1 is the data source for the final TS; difference to UC2 is no + OPF. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging) + - oedb_ts: real egon_data time series + - apply_heat_pump_strategy: uncontrolled + - reactive_power + - reinforce: final reinforcement + - save + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - oedb_ts: + timeindex: {start: "2035-01-01", periods: 168, freq: h} + dispatchable: {other: 0.7} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc4_example.yaml b/edisgo/run/presets/uc4_example.yaml new file mode 100644 index 000000000..02b26ee53 --- /dev/null +++ b/edisgo/run/presets/uc4_example.yaml @@ -0,0 +1,65 @@ +_comment: | + UC4 — OPF with full flexibility: + Loads real egon_data time series (oedb) and runs a powermodels OPF + over flexibilities (heat pumps, EV, DSM, storage) with HV requirements + from overlying grid. opf_version 3 activates HV-constraints from + overlying_grid CSV directory. + +_workflow: + - setup_grid: load ding0 topology + - import_generators: from egon_data + - import_home_batteries: from egon_data + - import_heat_pumps: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_charging_strategy: dumb + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - import_overlying_grid_data: HV constraints from CSV dir + - optimize: pm_optimize with flex assets (SOC, opf v3 = HV constraints) + +scenario: eGon2035 + +grid: + ding0_path: "/path/to/ding0_grid" + legacy_ding0_grids: false + +database: + ssh: + enabled: false + +timeindex: {start: "2035-01-01", periods: 24, freq: h} + +overlying_grid: + enabled: true # master switch — set true to activate import_overlying_grid_data + source: csv # "csv" (load from path) or "etrago" (consume overlying_grid_data kwarg) + path: "/storage/JoDa/edisgo_playground/overlying_grid_data" # required when source == csv; full leaf dir for ONE grid (like ding0_path) + +results: + directory: results/uc4_example + + +pipeline: + - setup_grid + - import_generators + - import_home_batteries + - import_heat_pumps + - import_dsm + - import_electromobility: + charging_strategy: null + flexibility_bands_ucs: ["home", "work", "public", "hpc"] + - oedb_ts: + dispatchable: {other: 0.7} + timeindex: {start: "2035-01-01", periods: 24, freq: h} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - import_overlying_grid_data + - reactive_power + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - reinforce + - save: + archive: true + save_opf_results: true diff --git a/edisgo/run/presets/uc5_select_timesteps.yaml b/edisgo/run/presets/uc5_select_timesteps.yaml new file mode 100644 index 000000000..82431bc7d --- /dev/null +++ b/edisgo/run/presets/uc5_select_timesteps.yaml @@ -0,0 +1,117 @@ +_comment: | + UC5 — OPF with configurable timestep selection (manual OR auto): + Like UC4 (full flexibility OPF with HV requirements), but the time index + is reduced to a selected subset instead of a fixed window. The mode is + chosen in the timeseries_selection block below (typically overridden in + the run script). + + The pipeline carries TWO select_timesteps steps, each with a `position`: + - position: pre_import (before import_heat_pumps) — acts only in MANUAL + mode. It sets the explicit time index, which the heat-pump/DSM imports + and oedb_ts then use to fetch only the selected steps (cheap). + - position: post_grid (after import_overlying_grid_data, before + reactive_power) — acts only in AUTO mode. It needs all active-power + time series (incl. overlying-grid generation) set to run the scoring + power flow via get_most_critical_time_intervals. + Whichever mode is configured, the other positioned step is a no-op. + + Auto mode normally yields two disconnected intervals (one overloading, + one voltage). They are kept separate (a gap in the time index); if they + overlap, a non-overlapping pair is chosen if possible, otherwise they are + concatenated into one interval. A later optimize step can detect the gap + and run separate optimizations per interval. + +_workflow: + - setup_grid: load ding0 topology + - import_generators / import_home_batteries + - select_timesteps (pre_import): manual only — set explicit time index + - import_heat_pumps / import_dsm: fetch only selected steps (manual) + - import_electromobility: dumb charging, flex bands + - oedb_ts: real wind/solar + load time series + - apply_charging_strategy / apply_heat_pump_strategy + - build_flexibility_bands: EV bands on the fixed hourly index + - import_overlying_grid_data: HV constraints from CSV dir + - select_timesteps (post_grid): auto only — reduce to critical intervals + - reactive_power: fixed cosphi on the reduced index + - optimize: pm_optimize with flex assets + - reinforce / save + +# Self-contained (no `extends`): everything uc4_example provided is inlined +# below, so this preset can be run on its own via +# run_edisgo({"extends": "uc5_select_timesteps", "grid": {"ding0_path": ...}}) +scenario: eGon2035 + +grid: + ding0_path: "/path/to/ding0_grid" + legacy_ding0_grids: false + +database: + source: local + + +# No explicit base time index is set. oedb_ts falls back to a full year derived +# from the scenario when none is given, which is what auto interval selection +# needs (week-long critical intervals to pick from). For manual selection the +# pre-import select_timesteps step sets the index instead. + +overlying_grid: + enabled: true # set true to activate import_overlying_grid_data + source: csv # "csv" (load from path) or "etrago" (kwarg) + path: "/storage/JoDa/edisgo_playground/overlying_grid_data" + +results: + directory: results/uc5_select_timesteps + +# Top-level block read by the select_timesteps task via ctx.raw_config. +# eGo can inject this block the same way it injects overlying_grid. +# Set `mode` (and its parameters) here or override it in the run script. +timeseries_selection: + mode: auto + # auto method: "power_flow" (default, scores intervals via a power flow) or + # "residual_load" (no power flow — the weeks ending at the max/min residual-load + # time steps; requires overlying-grid data). + method: residual_load + # --- shared auto parameters (both methods) --- + time_steps_per_time_interval: 168 # one week (must be a multiple of 24) + time_step_day_start: 4 # hour of day the intervals start/end on + # --- power_flow method parameters --- + percentage: 1.0 + save_steps: true # write selected intervals CSV to results_dir + use_troubleshooting_mode: true # handle power-flow non-convergence + overloading_factor: 0.95 + voltage_deviation_factor: 0.95 + # --- manual parameters (used when mode: manual) --- + # timestamps: ["2035-01-15 08:00", "2035-01-15 9:00", "2035-01-15 10:00"] + # or a range instead of `timestamps`: + start: "2035-01-15 00:00" + periods: 24 + freq: h + +pipeline: + - setup_grid + - import_generators + - import_home_batteries + - select_timesteps: {position: pre_import} # acts in manual mode only + - import_heat_pumps + - import_dsm + - import_electromobility: + charging_strategy: null + # flexibility bands are built later (build_flexibility_bands), once the + # analysis time index is fixed, so they are resampled to it + - oedb_ts: + dispatchable: {other: 0.7} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - build_flexibility_bands # hourly bands on the 2035 index + - import_overlying_grid_data + - select_timesteps: {position: post_grid} # acts in auto mode only + - reactive_power + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - reinforce: + catch_convergence_problems: true + - save: + archive: true + save_opf_results: true diff --git a/edisgo/run/presets/uc6_spatial_reduction.yaml b/edisgo/run/presets/uc6_spatial_reduction.yaml new file mode 100644 index 000000000..767327552 --- /dev/null +++ b/edisgo/run/presets/uc6_spatial_reduction.yaml @@ -0,0 +1,149 @@ +_comment: | + UC6 — OPF with configurable timestep selection (manual OR auto), PLUS spatial + complexity reduction bracketing the OPF step. Standalone copy of + uc5_select_timesteps.yaml (not an `extends` overlay) with the spatial_reduce / + spatial_restore bracket added around `optimize`. + + The pipeline carries TWO select_timesteps steps, each with a `position`: + - position: pre_import (before import_heat_pumps) — acts only in MANUAL + mode. It sets the explicit time index, which the heat-pump/DSM imports + and oedb_ts then use to fetch only the selected steps (cheap). + - position: post_grid (after import_overlying_grid_data, before + reactive_power) — acts only in AUTO mode. It needs all active-power + time series (incl. overlying-grid generation) set to run the scoring + power flow via get_most_critical_time_intervals. + Whichever mode is configured, the other positioned step is a no-op. + + Auto mode normally yields two disconnected intervals (one overloading, + one voltage). They are kept separate (a gap in the time index); if they + overlap, a non-overlapping pair is chosen if possible, otherwise they are + concatenated into one interval. A later optimize step can detect the gap + and run separate optimizations per interval. + + Spatial complexity reduction (spatial_reduce / spatial_restore) brackets + `optimize` only: + - spatial_reduce (before optimize): deepcopies and stashes the full grid on + ctx, then spatially reduces the working object so `optimize` runs on a + smaller grid. + - spatial_restore (after optimize): writes the optimized flexible-component + dispatch back onto the stashed full grid (by name, or disaggregated onto + `old_name` members if aggregation_mode is true), recomputes reactive power + for those components, and makes the full grid active again for `reinforce`. + Both are no-ops when `spatial_reduction.enabled` is false (default), so + `reinforce` then runs on the same grid `optimize` used, same as + uc5_select_timesteps.yaml. `reactive_power` (pre-OPF, full time series) stays + where it already was, unaffected by the spatial bracket. + Reinforcement always runs on the full topology, regardless of the flag. + +_workflow: + - setup_grid: load ding0 topology + - import_generators / import_home_batteries + - select_timesteps (pre_import): manual only — set explicit time index + - import_heat_pumps / import_dsm: fetch only selected steps (manual) + - import_electromobility: dumb charging, flex bands + - oedb_ts: real wind/solar + load time series + - apply_charging_strategy / apply_heat_pump_strategy + - build_flexibility_bands: EV bands on the fixed hourly index + - import_overlying_grid_data: HV constraints from CSV dir + - select_timesteps (post_grid): auto only — reduce to critical intervals + - reactive_power: fixed cosphi on the reduced index + - spatial_reduce: no-op unless spatial_reduction.enabled — stash full grid, + reduce working object + - optimize: pm_optimize with flex assets, on the (possibly) reduced grid + - spatial_restore: no-op unless spatial_reduction.enabled — write dispatch + back onto the stashed full grid, recompute reactive power + - reinforce / save: always on the full topology + +# Self-contained (no `extends`): everything uc4_example provided is inlined +# below, so this preset can be run on its own via +# run_edisgo({"extends": "uc6_spatial_reduction", "grid": {"ding0_path": ...}}) +scenario: eGon2035 + +grid: + ding0_path: "/path/to/ding0_grid" + legacy_ding0_grids: false + +database: + source: local + + +# No explicit base time index is set. oedb_ts falls back to a full year derived +# from the scenario when none is given, which is what auto interval selection +# needs (week-long critical intervals to pick from). For manual selection the +# pre-import select_timesteps step sets the index instead. + +overlying_grid: + enabled: true # set true to activate import_overlying_grid_data + source: csv # "csv" (load from path) or "etrago" (kwarg) + path: "/home/gurobi/.edisgo_input/overlying_grid" + +results: + directory: results/uc6_spatial_reduction + +# Top-level block read by the select_timesteps task via ctx.raw_config. +# eGo can inject this block the same way it injects overlying_grid. +# Set `mode` (and its parameters) here or override it in the run script. +timeseries_selection: + mode: manual + # auto method: "power_flow" (default, scores intervals via a power flow) or + # "residual_load" (no power flow — the weeks ending at the max/min residual-load + # time steps; requires overlying-grid data). + method: residual_load + # --- shared auto parameters (both methods) --- + time_steps_per_time_interval: 168 # one week (must be a multiple of 24) + time_step_day_start: 4 # hour of day the intervals start/end on + # --- power_flow method parameters --- + percentage: 1.0 + save_steps: true # write selected intervals CSV to results_dir + use_troubleshooting_mode: true # handle power-flow non-convergence + overloading_factor: 0.95 + voltage_deviation_factor: 0.95 + # --- manual parameters (used when mode: manual) --- + # timestamps: ["2035-01-15 08:00", "2035-01-15 9:00", "2035-01-15 10:00"] + # or a range instead of `timestamps`: + start: "2035-01-15 00:00" + periods: 24 + freq: h + +# Top-level block read by the spatial_reduce/spatial_restore tasks via +# ctx.raw_config. eGo can inject this block the same way it injects +# overlying_grid/timeseries_selection (global `spatial_reduction` default + +# per-grid `spatial_reduction_per_grid` override keyed by mv_grid_id). +spatial_reduction: + enabled: true # set true to activate spatial_reduce/spatial_restore + mode: kmeansdijkstra # clustering mode for spatial_complexity_reduction + cluster_area: feeder + reduction_factor: 0.3 + reduction_factor_not_focused: False + aggregation_mode: false # start with false; true enables load/generator merging + +pipeline: + - setup_grid + - import_generators + - import_home_batteries + - select_timesteps: {position: pre_import} # acts in manual mode only + - import_heat_pumps + - import_dsm + - import_electromobility: + charging_strategy: null + # flexibility bands are built later (build_flexibility_bands), once the + # analysis time index is fixed, so they are resampled to it + - oedb_ts: + dispatchable: {other: 0.7} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - build_flexibility_bands # hourly bands on the 2035 index + - import_overlying_grid_data + - select_timesteps: {position: post_grid} # acts in auto mode only + - reactive_power + - spatial_reduce # no-op unless spatial_reduction.enabled + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - spatial_restore # no-op unless spatial_reduction.enabled + - reinforce: + catch_convergence_problems: true + - save: + archive: true + save_opf_results: true diff --git a/edisgo/run/registry.py b/edisgo/run/registry.py new file mode 100644 index 000000000..c7ffca338 --- /dev/null +++ b/edisgo/run/registry.py @@ -0,0 +1,178 @@ +""" +Task registry for the eDisGo pipeline runner. + +This module holds the global, process-wide mapping of task names to task +functions. Tasks are registered via the :func:`register_task` decorator +and looked up by name at pipeline execution time by the runner. Keeping +the registry separate from both the runner and the task implementations +lets external projects add their own tasks without patching eDisGo — +just import ``register_task`` and decorate a function. + +Registered tasks all share the signature ``(edisgo, ctx, **params)`` +where ``edisgo`` is the current :class:`~edisgo.EDisGo` instance (or +``None`` before it has been created by the first task), ``ctx`` is a +:class:`~edisgo.run.context.RunContext`, and ``**params`` are the +parameters passed from the YAML/JSON step definition. A task may return +an updated ``edisgo`` object (e.g. ``setup_grid`` creates it, ``load_*`` +replaces it); otherwise the runner keeps using the same instance. +""" +from __future__ import annotations + +from typing import Callable, NamedTuple + +_TASKS: dict[str, Callable] = {} + + +class TaskMeta(NamedTuple): + """ + Declarative metadata describing a task's pipeline pre-/post-conditions. + + Attributes + ---------- + requires : frozenset of str + Capabilities that must already be satisfied in the stage before + this task runs (e.g. ``{"grid"}``, ``{"timeseries"}``, ``{"flex"}``). + provides : frozenset of str + Capabilities this task establishes for later tasks in the stage. + ts_altering : bool + Whether the task sets/alters the active-power time series. Such + tasks must not appear after ``reactive_power``. The validator uses + this metadata so it stays in sync with the actual tasks instead of + maintaining a parallel hard-coded list. + """ + + requires: frozenset = frozenset() + provides: frozenset = frozenset() + ts_altering: bool = False + + +_META: dict[str, TaskMeta] = {} + + +def register_task( + name: str, + *, + requires=frozenset(), + provides=frozenset(), + ts_altering: bool = False, +) -> Callable[[Callable], Callable]: + """ + Decorator to register a task function under the given name. + + The decorated function becomes addressable from YAML/JSON pipelines + as either a plain string ``name`` or a single-key mapping + ``name: {param: value, ...}``. The name must be unique globally — + re-registering raises :class:`ValueError` to prevent silent + overrides across plugins. + + Parameters + ---------- + name : str + Unique task name used in pipeline definitions. + requires : iterable of str, optional + Capabilities the task needs (see :class:`TaskMeta`). Used by the + validator for static ordering checks. + provides : iterable of str, optional + Capabilities the task establishes for later tasks. + ts_altering : bool, optional + Whether the task alters the active-power time series (must precede + ``reactive_power``). + + Returns + ------- + Callable + A decorator that registers ``fn`` and returns it unchanged. + + Raises + ------ + ValueError + If ``name`` is already registered. + + Examples + -------- + >>> @register_task("set_timeindex_weekly", provides={"timeseries"}, + ... ts_altering=True) + ... def task_weekly(edisgo, ctx, *, start): + ... import pandas as pd + ... edisgo.set_timeindex(pd.date_range(start, periods=168, freq="h")) + + """ + def deco(fn: Callable) -> Callable: + if name in _TASKS: + raise ValueError( + f"Task '{name}' is already registered " + f"(existing={_TASKS[name].__qualname__}, " + f"new={fn.__qualname__})." + ) + _TASKS[name] = fn + _META[name] = TaskMeta( + requires=frozenset(requires), + provides=frozenset(provides), + ts_altering=ts_altering, + ) + return fn + + return deco + + +def get_task_meta(name: str) -> TaskMeta: + """ + Return the :class:`TaskMeta` for a registered task. + + Parameters + ---------- + name : str + Task name. + + Returns + ------- + TaskMeta + The task's declared metadata. Unregistered names yield an empty + :class:`TaskMeta` (no requirements, no provided capabilities). + + """ + return _META.get(name, TaskMeta()) + + +def get_task(name: str) -> Callable: + """ + Look up a registered task function by name. + + Parameters + ---------- + name : str + Task name as used in pipeline definitions. + + Returns + ------- + Callable + The task function registered under ``name``. + + Raises + ------ + KeyError + If ``name`` is not registered. The error message lists all + known task names to aid typo debugging. + + """ + if name not in _TASKS: + raise KeyError( + f"Unknown task: '{name}'. Known tasks: {sorted(_TASKS)}" + ) + return _TASKS[name] + + +def known_tasks() -> list[str]: + """ + Return a sorted list of all registered task names. + + Useful for error messages, CLI completion, and tests that assert + core tasks exist. + + Returns + ------- + list of str + All registered task names in alphabetical order. + + """ + return sorted(_TASKS) diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py new file mode 100644 index 000000000..cf1f531cc --- /dev/null +++ b/edisgo/run/runner.py @@ -0,0 +1,270 @@ +""" +Pipeline execution engine for the eDisGo runner. + +This module ties the other three pieces — :mod:`edisgo.run.config` +(loader), :mod:`edisgo.run.validator` (static checks), and +:mod:`edisgo.run.registry` (task lookup) — together into a linear +stage-by-stage executor. + +The execution model: + +1. Load and validate the config. +2. Build a :class:`~edisgo.run.context.RunContext`. +3. For each stage, if the stage declares ``load_from: X``, reload + the EDisGo object from stage ``X``'s save-artifact (topology + + results only; time series are dropped to let the new stage set + fresh ones). +4. For each step in the stage's pipeline, look up the task function + in the registry and call it with the current EDisGo object and + the context. A task may return a new EDisGo object (``setup_grid``, + ``load_from_base``) which then replaces the current one. +5. Repeat for all stages, finally return the EDisGo object. + +Two entry points are exposed: + +* :func:`run_edisgo` — starts from no EDisGo object; the first task + must create one (usually ``setup_grid``). +* :func:`_run_pipeline_on` — starts from an existing EDisGo instance; + used by :meth:`edisgo.EDisGo.run_pipeline`. +""" + +from __future__ import annotations + +import logging + +from pathlib import Path +from typing import Any + +from edisgo.run import tasks as _tasks # noqa: F401 — triggers registration +from edisgo.run.config import load_config +from edisgo.run.context import RunContext +from edisgo.run.registry import get_task +from edisgo.run.validator import _split_step, validate + +logger = logging.getLogger("edisgo.run.runner") + + +def run_edisgo(config, overlying_grid_data=None, engine=None) -> Any: + """ + Run an eDisGo pipeline from a YAML/JSON config or dict. + + This is the standalone entry point. The pipeline's first task is + typically ``setup_grid`` or ``load_from_base`` to bootstrap the + :class:`~edisgo.EDisGo` instance. If you already have one, + prefer :meth:`edisgo.EDisGo.run_pipeline` instead. + + Parameters + ---------- + config : str, pathlib.Path, or dict + Path to a YAML/JSON pipeline config, or an in-memory dict of + the same shape. + overlying_grid_data : dict, optional + Overlying-grid data (e.g. eTraGo results) consumed by the + ``import_overlying_grid_data`` task. + engine : sqlalchemy.engine.Engine, optional + Pre-built database engine to use for all DB-backed tasks. When + given, it is cached on the :class:`~edisgo.run.context.RunContext` + so every task reuses it (via :meth:`RunContext.ensure_engine`) + instead of building its own from the config. This lets a caller + (e.g. eGo) supply a single connection that overrides the + ``database`` section of the config/preset. + + Returns + ------- + :class:`~edisgo.EDisGo` + The EDisGo instance after the last stage has run. For + multi-stage configs this is the object produced by the final + stage. + + """ + return _run_pipeline_on( + None, config, overlying_grid_data=overlying_grid_data, engine=engine + ) + + +def _run_pipeline_on(edisgo, config, overlying_grid_data=None, engine=None): + """ + Internal runner shared by :func:`run_edisgo` and the EDisGo method. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Existing EDisGo instance to operate on, or ``None`` to have + the first task create one. + config : str, pathlib.Path, or dict + Config to execute. Passed through to + :func:`edisgo.run.config.load_config`. + + Returns + ------- + edisgo.EDisGo + The final EDisGo instance. + + Raises + ------ + RuntimeError + If a stage declares ``load_from: X`` but ``X`` produced no + artifact (typically because validate() was skipped). + + """ + cfg = load_config(config) + validate(cfg) + ctx = _build_context(cfg) + ctx.overlying_grid_data = overlying_grid_data + # A caller-supplied engine (e.g. from eGo) overrides the config/preset + # database section: caching it on the context makes ensure_engine() return + # it for every DB-backed task. + if engine is not None: + ctx.engine = engine + ctx.logger.info( + f"run_edisgo: using caller-supplied database engine " + f"'{getattr(engine.url, 'database', engine)}' for all tasks." + ) + + for stage in cfg["stages"]: + ctx.current_stage = stage["name"] + ctx.logger.info(f"=== stage '{stage['name']}' ===") + + load_from = stage.get("load_from") + if load_from is not None: + artifact = ctx.stage_artifacts.get(load_from) + if artifact is None: + raise RuntimeError( + f"Stage '{stage['name']}' wants to load from " + f"'{load_from}' but no artifact is registered." + ) + edisgo = _load_artifact(str(artifact)) + + params = stage.get("params", {}) or {} + for step in stage["pipeline"]: + name, step_params = _split_step(step) + step_params = _resolve_templating(step_params, params) + ctx.logger.info(f" -> task '{name}'") + task_fn = get_task(name) + result = task_fn(edisgo, ctx, **step_params) + if result is not None: + edisgo = result + + return edisgo + + +def _build_context(cfg: dict) -> RunContext: + """ + Build a :class:`~edisgo.run.context.RunContext` from a config. + + Wires ``scenario`` and ``results.directory`` into the context and + stores the full config under :attr:`RunContext.raw_config` so + tasks can read supplementary sections. + + Parameters + ---------- + cfg : dict + Normalized config. + + Returns + ------- + RunContext + Initialized context with no engine, no artifacts, empty flags. + + """ + results_cfg = cfg.get("results") or {} + results_dir = results_cfg.get("directory") + return RunContext( + scenario=cfg.get("scenario"), + results_dir=Path(results_dir) if results_dir else None, + raw_config=cfg, + ) + + +def _load_artifact(path: str): + """ + Reload an EDisGo instance from a save-artifact for a ``load_from``. + + Loads topology + results only; time series and flex data are + dropped so the consuming stage can set them fresh. Equipment + changes are reset so the next stage's reinforce accounts only + for its own scenario. + + Parameters + ---------- + path : str + Path to a directory or ``.zip`` produced by the ``save`` + task. + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + from edisgo.run.tasks.grid import load_saved_edisgo + + # Topology + results only; time series and flex data are dropped so the + # consuming stage sets them fresh, and equipment_changes is reset. + return load_saved_edisgo(path, import_results=True) + + +def _resolve_templating(step_params: dict, stage_params: dict) -> dict: + """ + Substitute ``{{params.x}}`` placeholders in step parameters. + + Stage-level ``params:`` allows a preset to expose a few knobs that + individual step parameters can reference. Only simple + ``{{params.KEY}}`` expansions inside string values are supported + (no filters, no conditionals, no nested expressions) — deliberately + kept trivial to avoid a Jinja dependency. + + Parameters + ---------- + step_params : dict + Keyword arguments for a single step. + stage_params : dict + Stage-level ``params:`` dict. + + Returns + ------- + dict + ``step_params`` with template strings resolved. + + """ + if not stage_params or not step_params: + return step_params + out = {} + for k, v in step_params.items(): + if isinstance(v, str) and "{{" in v: + out[k] = _render_template(v, stage_params) + else: + out[k] = v + return out + + +def _render_template(s: str, stage_params: dict) -> str: + """ + Expand ``{{params.KEY}}`` references in a single string. + + Parameters + ---------- + s : str + Source string. + stage_params : dict + Mapping of stage-level parameters. + + Returns + ------- + str + Rendered string. Unknown keys are left in place (the original + placeholder remains) so downstream errors point at the + typo-ed key rather than silently turning into an empty + string. + + """ + import re + + def repl(match): + expr = match.group(1).strip() + if expr.startswith("params."): + key = expr.split(".", 1)[1] + return str(stage_params.get(key, match.group(0))) + return match.group(0) + + return re.sub(r"\{\{\s*([^}]+)\s*\}\}", repl, s) diff --git a/edisgo/run/tasks/__init__.py b/edisgo/run/tasks/__init__.py new file mode 100644 index 000000000..a0ef5d22c --- /dev/null +++ b/edisgo/run/tasks/__init__.py @@ -0,0 +1,44 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +""" +Task implementations for the eDisGo pipeline runner. + +Importing this package as a side effect registers every task defined +in its submodules with :func:`edisgo.run.registry.register_task`, so +that the runner sees them at execution time. The submodules are: + +* :mod:`.grid` — ``setup_grid``, ``load_from_base`` +* :mod:`.timeseries` — ``worst_case_ts``, ``oedb_ts``, ``manual_ts``, + ``set_timeindex``, ``reactive_power`` +* :mod:`.flex` — flex imports + (``import_heat_pumps``, ``import_home_batteries``, ``import_dsm``, + ``import_electromobility``, ``import_generators``), + ``build_flexibility_bands``, and operating strategies + (``apply_charging_strategy``, ``apply_heat_pump_strategy``) +* :mod:`.analysis` — ``check_integrity``, ``analyze``, ``reinforce``, + ``base_reinforce``, ``optimize`` +* :mod:`.io` — ``save``, ``load_charging_from_files`` +* :mod:`.spatial` — ``spatial_reduce``, ``spatial_restore`` + +Task signature convention: ``(edisgo, ctx, **params)``. A task may +mutate ``edisgo`` in place and/or return a new EDisGo instance (the +returned value, if non-None, replaces the current one in the runner's +loop). +""" + +from edisgo.run.tasks import ( # noqa: F401 + analysis, + flex, + grid, + io, + spatial, + timeseries, +) diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py new file mode 100644 index 000000000..9f8ec57ec --- /dev/null +++ b/edisgo/run/tasks/analysis.py @@ -0,0 +1,385 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +""" +Power-flow, reinforcement, and optimization tasks. + +The three analysis layers: + +* :func:`task_analyze` (``analyze``) — non-linear AC load flow over + the active time series; does not modify the topology. +* :func:`task_reinforce` (``reinforce``) — iterative reinforcement + that adds/upgrades equipment until all technical constraints are + met. Populates ``results.equipment_changes``. +* :func:`task_optimize` (``optimize``) — powermodels OPF over + flexibilities (heat pumps, EV, DSM, storage) to minimize + reinforcement need. + +In addition: + +* :func:`task_check_integrity` (``check_integrity``) — a cheap + sanity check before the expensive steps. +* :func:`task_base_reinforce` (``base_reinforce``) — two-phase helper: + worst-case TS → reinforce → reset ``equipment_changes``. Used to + produce a "base" grid whose subsequent reinforce costs reflect + only a scenario overlay. +""" + +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + + +@register_task("check_integrity") +def task_check_integrity(edisgo, ctx): + """ + Run EDisGo's integrity checks on the topology and time series. + + Catches bus mismatches, missing time series for components, and + similar structural problems. Raises if something is off — do not + swallow it silently. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to check. + ctx : RunContext + Run context (unused). + + Returns + ------- + edisgo.EDisGo + The unchanged EDisGo instance. + + """ + edisgo.check_integrity() + return edisgo + + +@register_task("analyze", requires={"timeseries"}) +def task_analyze( + edisgo, + ctx, + *, + mode=None, + timesteps=None, + raise_not_converged=False, + troubleshooting_mode=None, +): + """ + Run AC power flow over the active time series. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to analyze. + ctx : RunContext + Run context. Stores the number of non-converged time steps + under ``ctx.flags['not_converged_steps']`` and warns if any. + mode : str, optional + ``None`` (default) runs the full grid; ``"mv"`` runs only the + medium-voltage level; ``"lv"`` runs only LV. + timesteps : pandas.DatetimeIndex, optional + Restrict the analysis to these time steps. + raise_not_converged : bool, optional + If ``True``, raise on non-convergence. Default ``False`` so + the pipeline can continue and ``reinforce`` can attempt to + resolve the issue. + troubleshooting_mode : str, optional + Extra diagnostic mode passed through to + :meth:`EDisGo.analyze`. + + Returns + ------- + edisgo.EDisGo + The analyzed EDisGo instance. + + """ + result = edisgo.analyze( + mode=mode, + timesteps=timesteps, + raise_not_converged=raise_not_converged, + troubleshooting_mode=troubleshooting_mode, + ) + if isinstance(result, tuple) and len(result) == 2: + converged, not_converged = result + ctx.flags["not_converged_steps"] = len(not_converged) + if len(not_converged) > 0: + ctx.logger.warning( + f"Power flow did not converge for {len(not_converged)} time steps." + ) + return edisgo + + +@register_task("reinforce", requires={"timeseries"}) +def task_reinforce( + edisgo, + ctx, + *, + timesteps_pfa=None, + reduced_analysis=False, + copy_grid=False, + max_while_iterations=20, + split_voltage_band=True, + mode=None, + without_generator_import=False, + n_minus_one=False, + catch_convergence_problems=False, +): + """ + Run iterative grid reinforcement. + + Adds/upgrades lines and transformers until voltage and loading + constraints are met for all time steps. Results accumulate in + :attr:`EDisGo.results.equipment_changes` and + :attr:`~EDisGo.results.grid_expansion_costs`. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to reinforce. + ctx : RunContext + Run context (unused beyond logging). + timesteps_pfa : pandas.DatetimeIndex, optional + Restrict the reinforcement's analysis to these time steps. + reduced_analysis : bool, optional + If ``True``, use a cheaper convergence check during + reinforcement. + copy_grid : bool, optional + If ``True``, operate on a copy and return it as a new + instance (default ``False``). + max_while_iterations : int, optional + Cap on the outer iteration loop. + split_voltage_band : bool, optional + Split the allowed voltage deviation between MV and LV + (typical MV/LV coupling rule). + mode : str, optional + ``None``, ``"mv"``, ``"lv"``, or ``"mvlv"``. Restricts + reinforcement to a voltage level. + without_generator_import : bool, optional + Skip the implicit generator import step. + n_minus_one : bool, optional + Enable (N-1) contingency reinforcement. Expensive. + catch_convergence_problems : bool, optional + Wrap in the catch-convergence helper for troublesome grids. + + Returns + ------- + edisgo.EDisGo + The reinforced EDisGo instance. + + """ + edisgo.reinforce( + timesteps_pfa=timesteps_pfa, + reduced_analysis=reduced_analysis, + copy_grid=copy_grid, + max_while_iterations=max_while_iterations, + split_voltage_band=split_voltage_band, + mode=mode, + without_generator_import=without_generator_import, + n_minus_one=n_minus_one, + catch_convergence_problems=catch_convergence_problems, + ) + return edisgo + + +@register_task("base_reinforce", requires={"grid"}) +def task_base_reinforce( + edisgo, ctx, *, cases=None, reset_equipment_changes=True, save_artifact=True +): + """ + Produce a base-reinforced grid and reset the cost accumulator. + + This is the composite step ported from eGo's two-phase reinforce + workflow: + + 1. Set synthetic worst-case time series (``feed-in_case`` + + ``load_case``). + 2. Run :meth:`EDisGo.reinforce` to bring the grid to a neutral + baseline. + 3. Optionally save the resulting grid so downstream stages can + ``load_from: ...``. + 4. Clear :attr:`Results.equipment_changes` so the next reinforce + captures only scenario-specific deltas. + 5. Restore the prior time index so the next TS-setting task + starts from a clean state. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to base-reinforce. + ctx : RunContext + Run context. ``ctx.results_dir`` is the artifact destination. + Sets ``ctx.flags['base_reinforced'] = True`` and + ``ctx.stage_artifacts['__base_reinforce__']`` on save. + cases : list of str, optional + Which worst cases to set (subset of + ``{"load_case", "feed-in_case"}``). Default is both. + reset_equipment_changes : bool, optional + Clear the equipment-changes DataFrame after reinforcement. + save_artifact : bool, optional + Write a ``grid_data_base_reinforcement.zip`` next to the + other results. + + Returns + ------- + edisgo.EDisGo + The base-reinforced EDisGo instance. + + """ + import os + + prev_timeindex = edisgo.timeseries.timeindex + + edisgo.set_time_series_worst_case_analysis(cases=cases) + edisgo.reinforce() + + if save_artifact and ctx.results_dir is not None: + artifact_dir = os.path.join( + str(ctx.results_dir), "grid_data_base_reinforcement" + ) + edisgo.save( + directory=artifact_dir, + save_topology=True, + save_timeseries=False, + save_results=True, + archive=True, + archive_type="zip", + parameters={"grid_expansion_results": ["equipment_changes"]}, + ) + ctx.stage_artifacts["__base_reinforce__"] = artifact_dir + ".zip" + + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + + if len(prev_timeindex) > 0: + edisgo.set_timeindex(prev_timeindex) + + ctx.flags["base_reinforced"] = True + return edisgo + + +@register_task( + "optimize", requires={"timeseries", "flex"}, provides={"optimized_dispatch"} +) +def task_optimize( + edisgo, + ctx, + *, + flexible=None, + flexible_cps=None, + flexible_hps=None, + flexible_loads=None, + flexible_storage_units=None, + opf_version=2, + method="soc", + warm_start=False, + s_base=1, +): + """ + Run a powermodels optimal-power-flow (OPF) over flexibilities. + + If ``flexible`` is given (high-level shortcut), it expands to the + lower-level ``flexible_*`` lists automatically: + + * ``"heat_pumps"`` → all loads of type ``heat_pump`` + * ``"charging_points"`` → all loads of type ``charging_point`` + * ``"storage"`` → all storage-unit indices + * ``"loads"`` → all DSM-ready load indices + + Explicit ``flexible_*`` kwargs override the shortcut. + + This task only performs the ``flexible`` shortcut expansion (mode selection) + and calls :meth:`EDisGo.pm_optimize`. Handling of a non-contiguous (reduced) + time index — running a separate OPF per contiguous interval and merging the + results — lives in :func:`~.opf.powermodels_opf.pm_optimize`. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to optimize. + ctx : RunContext + Run context. Used for logging and, for multi-interval runs, nothing + else is required from it. The resolved ``flexible_*`` name lists are + written to ``ctx.flags['flexible_cps']`` / ``ctx.flags['flexible_hps']`` + / ``ctx.flags['flexible_loads']`` / ``ctx.flags['flexible_storage_units']`` + so a later ``spatial_restore`` step knows which components' dispatch + needs mapping back onto the full grid. + flexible : list of str, optional + High-level selector, subset of ``{"heat_pumps", + "charging_points", "storage"}``. If ``None``, nothing is + auto-populated. + flexible_cps : list of str, optional + Explicit list of flexible charging-point names. + flexible_hps : list of str, optional + Explicit list of flexible heat-pump load names. + flexible_loads : list of str, optional + Explicit list of flexible DSM load names. + flexible_storage_units : list of str, optional + Explicit list of flexible storage-unit names. + opf_version : int, optional + Powermodels OPF formulation version (1 or 2, default 2). + method : str, optional + OPF relaxation method, e.g. ``"soc"`` (second-order cone). + warm_start : bool, optional + Reuse a previous solution as the starting point. + s_base : float, optional + Per-unit base power for normalization. + + Returns + ------- + edisgo.EDisGo + The optimized EDisGo instance. + + """ + flexible = flexible or [] + + if flexible_hps is None and "heat_pumps" in flexible: + flexible_hps = edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "heat_pump" + ].index.tolist() + if flexible_cps is None and "charging_points" in flexible: + flexible_cps = edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "charging_point" + ].index.tolist() + if flexible_storage_units is None and "storage" in flexible: + flexible_storage_units = edisgo.topology.storage_units_df.index.tolist() + if flexible_loads is None and "dsm" in flexible: + flexible_loads = edisgo.dsm.p_min.columns.values + + if flexible_cps is None: + flexible_cps = [] + if flexible_hps is None: + flexible_hps = [] + if flexible_loads is None: + flexible_loads = [] + if flexible_storage_units is None: + flexible_storage_units = [] + + ctx.flags["flexible_cps"] = flexible_cps + ctx.flags["flexible_hps"] = flexible_hps + ctx.flags["flexible_loads"] = flexible_loads + ctx.flags["flexible_storage_units"] = flexible_storage_units + + # pm_optimize handles a non-contiguous (reduced) time index internally: + # it runs one OPF per contiguous interval and merges the results. + edisgo.pm_optimize( + flexible_cps=flexible_cps, + flexible_hps=flexible_hps, + flexible_loads=flexible_loads, + flexible_storage_units=flexible_storage_units, + opf_version=opf_version, + method=method, + warm_start=warm_start, + s_base=s_base, + ) + return edisgo diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py new file mode 100644 index 000000000..541b4abe4 --- /dev/null +++ b/edisgo/run/tasks/flex.py @@ -0,0 +1,347 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +""" +Flex-asset import and operation-strategy tasks. + +These tasks either pull flex assets (heat pumps, home batteries, DSM, +electromobility, generators) from egon_data / OEP into the topology, +or apply an operating strategy on assets already present. They must +run AFTER the grid is loaded (``setup_grid`` or ``load_from_base``) +and typically BEFORE the time-series step, so the time series can +cover the new assets. +""" + +from __future__ import annotations + +from edisgo.run.registry import register_task + + +@register_task("import_heat_pumps", requires={"grid"}, provides={"flex"}) +def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): + """ + Import heat pumps from egon_data into the topology. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets + ``ctx.flags['has_heat_pumps']`` to the observed count. + import_types : list of str, optional + Subset of ``["individual_heat_pumps", "central_heat_pumps"]``; + default imports both. + timeindex : pandas.DatetimeIndex, optional + Restrict COP / heat-demand time series to this index. If None, + falls back to ``ctx.flags['selected_timeindex']`` (set by a + preceding ``select_timesteps`` manual step) so the download is + restricted to the selected steps. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if timeindex is None: + timeindex = ctx.flags.get("selected_timeindex") + edisgo.import_heat_pumps( + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + timeindex=timeindex, + import_types=import_types, + ) + ctx.flags["has_heat_pumps"] = ( + len(edisgo.topology.loads_df.loc[edisgo.topology.loads_df.type == "heat_pump"]) + > 0 + ) + return edisgo + + +@register_task("import_home_batteries", requires={"grid"}, provides={"flex"}) +def task_import_home_batteries(edisgo, ctx): + """ + Import home batteries from egon_data into the topology. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets + ``ctx.flags['has_home_batteries']``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_home_batteries(scenario=ctx.scenario, engine=ctx.ensure_engine()) + ctx.flags["has_home_batteries"] = not edisgo.topology.storage_units_df.empty + return edisgo + + +@register_task("import_dsm", requires={"grid"}, provides={"flex"}) +def task_import_dsm(edisgo, ctx, *, timeindex=None): + """ + Import demand-side-management potential from egon_data. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets ``ctx.flags['has_dsm']``. + timeindex : pandas.DatetimeIndex, optional + Restrict DSM availability time series to this index. If None, + falls back to ``ctx.flags['selected_timeindex']`` (set by a + preceding ``select_timesteps`` manual step) so the download is + restricted to the selected steps. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if timeindex is None: + timeindex = ctx.flags.get("selected_timeindex") + edisgo.import_dsm( + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + timeindex=timeindex, + ) + ctx.flags["has_dsm"] = edisgo.dsm.p_max is not None and not edisgo.dsm.p_max.empty + return edisgo + + +@register_task("import_electromobility", requires={"grid"}, provides={"flex"}) +def task_import_electromobility( + edisgo, + ctx, + *, + data_source="oedb", + charging_strategy="dumb", + flexibility_bands_ucs=None, + import_electromobility_data_kwds=None, + allocate_charging_demand_kwds=None, +): + """ + Import electromobility data (charging processes + parks). + + Optionally applies a charging strategy directly after import to + turn the raw charging processes into active-power time series on + the charging points. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()`` (for ``data_source='oedb'``). Sets + ``ctx.flags['has_electromobility'] = True``. + data_source : str, optional + ``"oedb"`` (egon_data) or ``"directory"`` (requires + ``import_electromobility_data_kwds={"charging_processes_dir": + ..., "potential_charging_points_dir": ...}``). + charging_strategy : str or None, optional + Charging strategy applied right after import. ``"dumb"`` + (uncontrolled, default), ``"reduced"``, ``"residual"``, or + ``None`` to skip. + flexibility_bands_ucs : str or list of str, optional + Charging-point use case(s) to compute flexibility bands for + via :meth:`Electromobility.get_flexibility_bands` after import + and charging-strategy application. Valid entries: + ``"home"``, ``"work"``, ``"public"``, ``"hpc"``. Pass a single + string for one use case or a list for multiple. ``None`` + (default) skips flexibility-band computation — build them later + with the standalone :func:`task_build_flexibility_bands` once the + analysis time index is fixed, so the bands are resampled to it + (mirrors heat-pump handling, where the HP time series are not set + inside ``import_heat_pumps``). + import_electromobility_data_kwds : dict, optional + Extra kwargs passed through to the underlying importer. + allocate_charging_demand_kwds : dict, optional + Extra kwargs for charging-demand allocation. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_electromobility( + data_source=data_source, + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + import_electromobility_data_kwds=import_electromobility_data_kwds, + allocate_charging_demand_kwds=allocate_charging_demand_kwds, + ) + if charging_strategy: + edisgo.apply_charging_strategy(strategy=charging_strategy) + if flexibility_bands_ucs is not None: + edisgo.electromobility.get_flexibility_bands( + edisgo, + use_case=flexibility_bands_ucs, + ) + ctx.flags["has_electromobility"] = True + return edisgo + + +@register_task("build_flexibility_bands", requires={"flex"}) +def task_build_flexibility_bands(edisgo, ctx, *, use_case=None): + """ + Build EV charging flexibility bands from imported electromobility data. + + Standalone variant of the band computation that + :func:`task_import_electromobility` can do inline. Running it as a + separate step lets it execute *after* the analysis time index is fixed + (e.g. after ``oedb_ts`` / timestep selection), so + :meth:`Electromobility.get_flexibility_bands` resamples/scopes the bands + to the edisgo time-series frequency and timeindex instead of leaving + them at the raw SimBEV resolution and range. This mirrors how the + heat-pump time series are set outside ``import_heat_pumps``, and is more + efficient than building bands over a non-final index. + + ``get_flexibility_bands`` itself year-aligns and trims the bands down to + ``edisgo.timeseries.timeindex`` (see its docstring) whenever that + timeindex is non-empty, so ``electromobility.flexibility_bands`` always + matches it after this step runs - no separate trim call needed here. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + use_case : str or list of str, optional + Charging-point use case(s) to compute bands for. Valid entries: + ``"home"``, ``"work"``, ``"public"``, ``"hpc"``. Defaults to all + four. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + """ + if use_case is None: + use_case = ["home", "work", "public", "hpc"] + edisgo.electromobility.get_flexibility_bands(edisgo, use_case=use_case) + return edisgo + + +@register_task("apply_charging_strategy") +def task_apply_charging_strategy( + edisgo, ctx, *, strategy="dumb", charging_park_ids=None +): + """ + Apply a charging strategy to the already-imported EV fleet. + + Standalone variant of the step that ``import_electromobility`` + does inline. Useful when you want to import once and then try + multiple strategies in different runs. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + strategy : str, optional + Strategy name (``"dumb"`` / ``"reduced"`` / ``"residual"``). + charging_park_ids : list of int, optional + Restrict the strategy to these charging-park IDs. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.apply_charging_strategy( + strategy=strategy, charging_park_ids=charging_park_ids + ) + return edisgo + + +@register_task("apply_heat_pump_strategy") +def task_apply_heat_pump_strategy( + edisgo, ctx, *, strategy="uncontrolled", heat_pump_names=None +): + """ + Apply a heat-pump operating strategy. + + Skipped with an info-log if no heat pumps are present + (``ctx.flags['has_heat_pumps']`` is falsy), so pipelines can + safely include this step without a conditional guard. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + strategy : str, optional + Operating strategy (``"uncontrolled"``, ``"flexible"``, …). + heat_pump_names : list of str, optional + Restrict to specific heat-pump load names; default is all. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if not ctx.flags.get("has_heat_pumps"): + ctx.logger.info("Skipping 'apply_heat_pump_strategy': no heat pumps present.") + return edisgo + edisgo.apply_heat_pump_operating_strategy( + strategy=strategy, heat_pump_names=heat_pump_names + ) + return edisgo + + +@register_task("import_generators") +def task_import_generators(edisgo, ctx, *, generator_scenario=None): + """ + Import future generators for the active scenario. + + Thin wrapper around :meth:`EDisGo.import_generators`. Mostly + useful when you want to split grid loading and generator import + into two separate pipeline steps. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. ``ctx.scenario`` is used if + ``generator_scenario`` is not given. + generator_scenario : str, optional + Scenario name, e.g. ``"nep2035"`` or ``"ego100"``. Defaults + to ``ctx.scenario``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_generators( + generator_scenario=generator_scenario or ctx.scenario, + engine=ctx.ensure_engine(), + ) + return edisgo diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py new file mode 100644 index 000000000..a8474c142 --- /dev/null +++ b/edisgo/run/tasks/grid.py @@ -0,0 +1,263 @@ +""" +Grid loading tasks — bring an EDisGo instance into existence. + +Two ways to start a pipeline: + +* :func:`task_setup_grid` (``setup_grid``) — read a ding0 topology + from disk. This is the typical first step of every pipeline. +* :func:`task_load_from_base` (``load_from_base``) — reload a + previously saved EDisGo instance. Used to split a computation into + a slow "base" phase and one or more fast "scenario" phases that + reuse the base-reinforced grid. +""" + +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + +import pandas as pd + +@register_task("setup_grid", provides={"grid"}) +def task_setup_grid( + edisgo, + ctx, + *, + timeindex=None, + ding0_path=None, + legacy_ding0_grids=None, + import_generators=False, + generator_scenario=None, +): + """ + Load a ding0 grid into an EDisGo instance. + + If the runner was started without an EDisGo object (via + :func:`edisgo.run.run_edisgo`) this task creates one from the + ding0 CSV directory. If an EDisGo object is already present (via + :meth:`edisgo.EDisGo.run_pipeline`), it imports the topology into + that existing instance. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Current EDisGo instance, or ``None`` to create a fresh one. + ctx : RunContext + Run context. ``ctx.raw_config['grid']`` is consulted when + parameters are not passed explicitly. + ding0_path : str, optional + Path to the ding0 grid directory. Falls back to + ``ctx.raw_config['grid']['ding0_path']``. + legacy_ding0_grids : bool, optional + Whether to treat the ding0 directory as the legacy format. + Falls back to ``ctx.raw_config['grid']['legacy_ding0_grids']`` + and ultimately to ``False``. + import_generators : bool, optional + If ``True``, call :meth:`EDisGo.import_generators` after + loading the grid. + generator_scenario : str, optional + Generator scenario name passed to + :meth:`EDisGo.import_generators` (only if + ``import_generators=True``). + + Returns + ------- + edisgo.EDisGo + The EDisGo instance with the ding0 topology loaded. + + Raises + ------ + ValueError + If no ``ding0_path`` is given either as a task parameter or + under ``config.grid.ding0_path``. + + """ + from edisgo import EDisGo + import os + + grid_cfg = ctx.raw_config.get("grid", {}) + ding0_path = ding0_path or grid_cfg.get("ding0_path") + if ding0_path is None: + raise ValueError( + "Task 'setup_grid' requires 'ding0_path' either as task " + "parameter or under config.grid.ding0_path." + ) + if legacy_ding0_grids is None: + legacy_ding0_grids = grid_cfg.get("legacy_ding0_grids", False) + + if edisgo is None: + # Check if topology-subfolder is part of ding0 grids + ding0_path_str = str(ding0_path) + if not os.path.exists(os.path.join(ding0_path_str, "buses.csv")): + topology_path = os.path.join(ding0_path_str, "topology") + if os.path.exists(os.path.join(topology_path, "buses.csv")): + ding0_path_str = topology_path + + edisgo = EDisGo( + ding0_grid=ding0_path_str, + legacy_ding0_grids=legacy_ding0_grids, + ) + else: + edisgo.import_ding0_grid( + path=str(ding0_path), legacy_ding0_grids=legacy_ding0_grids + ) + + if import_generators: + edisgo.import_generators( + generator_scenario=generator_scenario, + engine=ctx.ensure_engine(), + ) + + if timeindex is not None: + ti_df = pd.date_range( + start=timeindex["start"], + periods=timeindex["periods"], + freq=timeindex.get("freq", "h"), + ) + edisgo.set_timeindex(ti_df) + + ctx.flags["grid_loaded"] = True + return edisgo + + +def load_saved_edisgo( + path, + *, + reset_equipment_changes=True, + import_timeseries=False, + import_results=False, + import_electromobility=False, + import_heat_pump=False, + import_dsm=False, + import_overlying_grid=False, +): + """ + Reload a previously saved EDisGo object from a directory or ``.zip``. + + Shared by the ``load_from_base`` task and the runner's stage-level + ``load_from`` handling so both load artifacts with the same policy. + Topology is always imported; time series and flex data default to off + (the consuming stage sets them fresh). ``legacy_grids`` is cleared and, + by default, ``results.equipment_changes`` is reset so a subsequent + reinforce reflects only the current scenario. + + Parameters + ---------- + path : str or pathlib.Path + Directory or ``.zip`` produced by the ``save`` task. + reset_equipment_changes : bool, optional + If ``True`` (default), clear ``results.equipment_changes``. + import_timeseries, import_results, import_electromobility, \ + import_heat_pump, import_dsm, import_overlying_grid : bool, optional + Which saved sub-datasets to import (all off by default except as + overridden by the caller). + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import os + + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + path = str(path) + from_zip = path.endswith(".zip") or not os.path.isdir(path) + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=import_timeseries, + import_results=import_results, + import_electromobility=import_electromobility, + import_heat_pump=import_heat_pump, + import_dsm=import_dsm, + import_overlying_grid=import_overlying_grid, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + return edisgo + + +@register_task("load_from_base", provides={"grid"}) +def task_load_from_base( + edisgo, + ctx, + *, + path=None, + reset_equipment_changes=True, + import_timeseries=False, + import_results=False, + import_electromobility=False, + import_heat_pump=False, + import_dsm=False, + import_overlying_grid=False, +): + """ + Reload an EDisGo instance from a previously saved directory/zip. + + This is the two-phase R4MU workflow's entry point: stage 1 + produces a base-reinforced grid and saves it, stage 2 (or N) + starts from ``load_from_base`` to pick up that grid and apply + scenario-specific modifications. The cost of the scenario then + shows up cleanly in ``equipment_changes`` because we reset it on + load. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Unused — the task always replaces whatever was there. + ctx : RunContext + Run context (logger only). + path : str + Directory or ``.zip`` produced by :func:`task_save`. + reset_equipment_changes : bool, optional + If ``True`` (default), clear + :attr:`Results.equipment_changes` so only the scenario's + reinforce is tracked. + import_timeseries : bool, optional + Whether to import the saved time series. Default: ``False`` + so the next stage sets its own. + import_results : bool, optional + Whether to import saved results. Default: ``False``. + import_electromobility : bool, optional + Whether to import saved electromobility data. + import_heat_pump : bool, optional + Whether to import saved heat-pump data. + import_dsm : bool, optional + Whether to import saved DSM data. + import_overlying_grid : bool, optional + Whether to import saved overlying-grid data (eTraGo + specifications). + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + if path is None: + grid_cfg = ctx.raw_config.get("grid", {}) or {} + path = grid_cfg.get("ding0_path") + if path is None: + raise ValueError( + "Task 'load_from_base' requires 'path' either as task " + "parameter or under config.grid.ding0_path." + ) + edisgo = load_saved_edisgo( + path, + reset_equipment_changes=reset_equipment_changes, + import_timeseries=import_timeseries, + import_results=import_results, + import_electromobility=import_electromobility, + import_heat_pump=import_heat_pump, + import_dsm=import_dsm, + import_overlying_grid=import_overlying_grid, + ) + ctx.flags["grid_loaded"] = True + return edisgo diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py new file mode 100644 index 000000000..045b824e3 --- /dev/null +++ b/edisgo/run/tasks/io.py @@ -0,0 +1,325 @@ +""" +Input/output tasks — persisting results and ingesting external files. + +* :func:`task_save` (``save``) — persist topology, time series, and + results to disk (directory or zip). Also publishes the artifact + path into ``ctx.stage_artifacts`` so a later stage can + ``load_from:``. +* :func:`task_load_charging_from_files` + (``load_charging_from_files``) — R4MU-specific placeholder for + integrating scenario charging stations from a directory of CSV / + GeoPackage files; implementation is deferred until needed. +""" + +from __future__ import annotations + +import os + +from edisgo.run.registry import register_task + + +@register_task("save") +def task_save( + edisgo, + ctx, + *, + directory=None, + save_topology=True, + save_timeseries=True, + save_results=True, + save_electromobility=None, + save_opf_results=False, + save_heatpump=None, + save_overlying_grid=False, + save_dsm=None, + archive=False, + archive_type="zip", + reduce_memory=False, + parameters=None, +): + """ + Save the current EDisGo state to disk. + + If ``directory`` is not given, the artifact is written under + ``ctx.results_dir / `` so every stage gets its own + subdirectory. When ``archive=True`` the result is a single zip; + the artifact path (including ``.zip``) is recorded in + ``ctx.stage_artifacts[]`` so a downstream stage can + declare ``load_from: ``. + + Flags drive smart defaults for the optional ``save_*`` switches: + if flex data is absent (per ``ctx.flags``), saving it is skipped. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to persist. + ctx : RunContext + Run context. Uses ``ctx.results_dir``, ``ctx.current_stage``, + and reads ``has_heat_pumps`` / ``has_dsm`` / + ``has_electromobility`` flags. + directory : str, optional + Absolute target directory. If omitted, derived from + ``ctx.results_dir / ctx.current_stage``. + save_topology : bool, optional + Write the topology CSVs. Default ``True``. + save_timeseries : bool, optional + Write time-series CSVs. Default ``True``. + save_results : bool, optional + Write the results CSVs (equipment changes, expansion costs, + etc.). Default ``True``. + save_electromobility : bool or None, optional + If ``None``, auto-enabled iff + ``ctx.flags['has_electromobility']`` is truthy. + save_opf_results : bool, optional + Write OPF results if present. + save_heatpump : bool or None, optional + If ``None``, auto-enabled iff ``ctx.flags['has_heat_pumps']`` + is truthy. + save_overlying_grid : bool, optional + Write overlying-grid (eTraGo) specs if present. + save_dsm : bool or None, optional + If ``None``, auto-enabled iff ``ctx.flags['has_dsm']`` is + truthy. + archive : bool, optional + Pack the directory into a single ``.zip`` archive. + archive_type : str, optional + Archive format (currently only ``"zip"``). + reduce_memory : bool, optional + Downcast float time-series to ``float32`` to save disk. + parameters : dict, optional + Fine-grained selection of which results fields to write, + e.g. ``{"grid_expansion_results": ["equipment_changes"]}``. + + Returns + ------- + edisgo.EDisGo + The unchanged EDisGo instance. + + Raises + ------ + ValueError + If no ``directory`` is given and ``ctx.results_dir`` is also + unset. + + """ + if directory is None: + if ctx.results_dir is None: + raise ValueError( + "Task 'save' needs a 'directory' parameter or config.results.directory." + ) + stage = ctx.current_stage or "main" + directory = os.path.join(str(ctx.results_dir), stage) + + if save_heatpump is None: + save_heatpump = ctx.flags.get("has_heat_pumps", False) + if save_dsm is None: + save_dsm = ctx.flags.get("has_dsm", False) + if save_electromobility is None: + save_electromobility = ctx.flags.get("has_electromobility", False) + + kwargs = dict( + directory=directory, + save_topology=save_topology, + save_timeseries=save_timeseries, + save_results=save_results, + save_electromobility=save_electromobility, + save_opf_results=save_opf_results, + save_heatpump=save_heatpump, + save_overlying_grid=save_overlying_grid, + save_dsm=save_dsm, + ) + if archive: + kwargs["archive"] = True + kwargs["archive_type"] = archive_type + if reduce_memory: + kwargs["reduce_memory"] = True + if parameters is not None: + kwargs["parameters"] = parameters + + edisgo.save(**kwargs) + + saved_path = directory + (".zip" if archive else "") + if ctx.current_stage: + ctx.stage_artifacts[ctx.current_stage] = saved_path + ctx.flags["last_saved"] = saved_path + return edisgo + + +@register_task("load_charging_from_files") +def task_load_charging_from_files( + edisgo, ctx, *, charging_dir, use_case_to_sector=None, mv_threshold_kw=100.0 +): + """ + Integrate scenario charging stations from files (R4MU workflow). + + PLACEHOLDER — the full implementation lives in eGo's + ``_run_edisgo_task_load_charging_from_files`` and needs to be + ported when R4MU is prioritised. The eGo version reads a + GeoPackage / CSV of charging locations, filters by the MV grid + district geometry, and integrates them into the topology via + :func:`find_nearest_bus` / ``integrate_component_based_on_geolocation`` + with a use-case-to-sector mapping and an MV/LV connection + threshold. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + charging_dir : str + Directory containing the charging-station source files. + use_case_to_sector : dict, optional + Maps raw use-case labels (``"home_detached"`` etc.) to + eDisGo sector names (``"home"``, ``"work"``, …). + mv_threshold_kw : float, optional + Capacity threshold above which stations connect to an MV + bus; below connect to LV. + + Raises + ------ + NotImplementedError + Always — port the eGo implementation before using. + + """ + raise NotImplementedError( + "Task 'load_charging_from_files' is a placeholder port from " + "eGo R4MU. Port the logic from eGo's " + "_run_edisgo_task_load_charging_from_files when R4MU is " + "needed." + ) + + +@register_task("import_overlying_grid_data") +def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): + """ + Import overlying grid data into the EDisGo instance. + + Behavior controlled by ``ctx.raw_config['overlying_grid']``: + + * ``enabled`` (bool) — master switch. Falsy → task no-ops. + * ``source`` (str) — ``"etrago"`` or ``"csv"``. + + ``source: etrago`` consumes ``ctx.overlying_grid_data`` (a dict of + DataFrames as returned by ``get_etrago_results_per_bus``), injected + via the ``overlying_grid_data=`` kwarg of + :func:`edisgo.run.run_edisgo`. Sets overlying-grid attributes and + dispatchable/fluctuating generator time series from it. + + ``source: csv`` loads overlying-grid attributes from CSVs in + ``overlying_grid.path`` (full directory path for ONE grid — same + leaf-dir convention as ``grid.ding0_path``; callers handling many + grids must compose the per-grid subdirectory themselves). + ``dispatchable_generators_active_power.csv`` and + ``renewables_potential.csv``, if present in that dir, are applied + as generator time series. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Reads ``raw_config['overlying_grid']`` and + ``overlying_grid_data`` attribute. + overlying_grid_path : str, optional + CSV directory override (takes precedence over + ``overlying_grid.path`` from the config) when ``source='csv'``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + import pandas as pd + + og_cfg = ctx.raw_config.get("overlying_grid") or {} + if not og_cfg.get("enabled"): + return edisgo + + source = og_cfg.get("source") + overlying_grid_data = ctx.overlying_grid_data + edisgo_ti = edisgo.timeseries.timeindex + + soc_attrs = { + "storage_units_soc", + "thermal_storage_units_decentral_soc", + "thermal_storage_units_central_soc", + } + + from edisgo.tools.tools import align_series_to_timeindex + + def _to_edisgo_timeindex(ts, extra_step=False): + # bind the stage's edisgo time index to the shared aligner + return align_series_to_timeindex(ts, edisgo_ti, extra_step=extra_step) + + if source not in ("etrago", "csv"): + ctx.logger.warning( + f"task 'import_overlying_grid_data': unknown source={source!r} " + "(expected 'etrago' or 'csv') — skipping." + ) + return edisgo + + # --- 1) load the overlying-grid attributes for the chosen source --- + if source == "etrago": + if overlying_grid_data is None: + ctx.logger.warning( + "task 'import_overlying_grid_data': source='etrago' but no " + "overlying_grid_data passed to run_edisgo — skipping." + ) + return edisgo + for attr in edisgo.overlying_grid._attributes: + if attr in overlying_grid_data: + setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + else: # source == "csv" + overlying_grid_path = overlying_grid_path or og_cfg.get("path") + if overlying_grid_path is None: + ctx.logger.warning( + "task 'import_overlying_grid_data': source='csv' but no " + "overlying_grid.path configured — skipping." + ) + return edisgo + edisgo.overlying_grid.from_csv(overlying_grid_path) + + # --- 2) reindex the overlying-grid attributes onto the edisgo timeindex + # (data may use a different year; SOC series carry one extra end step) --- + for attr in edisgo.overlying_grid._attributes: + ts = getattr(edisgo.overlying_grid, attr) + if ts is None or ts.empty: + continue + setattr( + edisgo.overlying_grid, + attr, + _to_edisgo_timeindex(ts, extra_step=attr in soc_attrs), + ) + + # --- 3) set dispatchable/fluctuating generator time series --- + if source == "etrago": + disp_ts = edisgo.overlying_grid.dispatchable_generators_active_power + pot_ts = edisgo.overlying_grid.renewables_potential + if disp_ts is not None and not disp_ts.empty: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + ) + if pot_ts is not None and not pot_ts.empty: + edisgo.set_time_series_active_power_predefined( + fluctuating_generators_ts=_to_edisgo_timeindex(pot_ts), + ) + else: # source == "csv": load the two generator-TS CSVs from the dir + def _load_generator_ts(filename): + path = os.path.join(overlying_grid_path, filename) + if not os.path.isfile(path): + return None + ts = pd.read_csv(path, index_col=0, parse_dates=True) + return _to_edisgo_timeindex(ts) + + disp_ts = _load_generator_ts("dispatchable_generators_active_power.csv") + pot_ts = _load_generator_ts("renewables_potential.csv") + if disp_ts is not None or pot_ts is not None: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + fluctuating_generators_ts=pot_ts, + ) + + return edisgo diff --git a/edisgo/run/tasks/spatial.py b/edisgo/run/tasks/spatial.py new file mode 100644 index 000000000..c7c893b08 --- /dev/null +++ b/edisgo/run/tasks/spatial.py @@ -0,0 +1,148 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +""" +Spatial complexity reduction tasks bracketing ``optimize``. + +* :func:`task_spatial_reduce` (``spatial_reduce``) — stashes a deepcopy of + the full grid on ``ctx`` and spatially reduces the working object so + ``optimize`` runs on a smaller grid. +* :func:`task_spatial_restore` (``spatial_restore``) — writes the optimized + flexible-component dispatch back onto the stashed full grid and makes it + the active object again, so ``reinforce`` runs on the full topology. + +Both are no-ops when ``spatial_reduction.enabled`` is false (the default), +so a pipeline that carries this bracket behaves exactly like one that +doesn't when spatial reduction is turned off. +""" + +from __future__ import annotations + +import copy + +from edisgo.run.registry import register_task + + +@register_task("spatial_reduce", requires={"grid"}, provides={"reduced_grid"}) +def task_spatial_reduce(edisgo, ctx, **overrides): + """ + Deepcopy and stash the full grid, then spatially reduce the working + object. + + Configuration is read from the top-level ``spatial_reduction:`` config + block (so eGo can inject it the same way it injects + ``timeseries_selection``); inline step params override individual keys + of that block. + + A no-op when ``enabled`` is not true — ``edisgo`` is returned unchanged + and ``ctx.full_grid_stash`` is left ``None``, so a downstream + ``spatial_restore`` also no-ops (see its docstring) and ``optimize``/ + ``reinforce`` run on the same, unreduced grid as if this task were + absent from the pipeline. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to spatially reduce in place. + ctx : RunContext + Run context. Reads ``ctx.raw_config['spatial_reduction']``. Sets + ``ctx.full_grid_stash`` to the pre-reduction deepcopy. + **overrides + Inline step params overriding keys of the ``spatial_reduction`` + block. Recognized keys: ``enabled`` (bool, default ``False``), + ``mode``, ``cluster_area``, ``reduction_factor``, + ``reduction_factor_not_focused``, ``aggregation_mode``, and the + aggregation sub-modes ``load_aggregation_mode`` / + ``generator_aggregation_mode`` — forwarded to + :meth:`~.EDisGo.spatial_complexity_reduction`. + + Returns + ------- + edisgo.EDisGo + The (possibly) spatially-reduced EDisGo instance. + + """ + cfg = {**ctx.raw_config.get("spatial_reduction", {}), **overrides} + if not cfg.get("enabled", False): + return edisgo + + ctx.full_grid_stash = copy.deepcopy(edisgo) + + kwargs = { + k: v + for k, v in cfg.items() + if k + in ( + "mode", + "cluster_area", + "reduction_factor", + "reduction_factor_not_focused", + "apply_pseudo_coordinates", + "aggregation_mode", + "load_aggregation_mode", + "generator_aggregation_mode", + "line_naming_convention", + "mv_pseudo_coordinates", + ) + } + edisgo.spatial_complexity_reduction(copy_edisgo=False, **kwargs) + return edisgo + + +@register_task("spatial_restore", requires={"reduced_grid", "optimized_dispatch"}) +def task_spatial_restore(edisgo, ctx, **overrides): + """ + Write optimized flexible-component dispatch back onto the stashed full + grid, and make it the active object again. + + Reads the flexible-component name lists ``optimize`` wrote to + ``ctx.flags`` and passes them, together with ``edisgo`` (the reduced, + just-optimized grid) and ``ctx.full_grid_stash`` (the pre-reduction + grid), to :meth:`~.EDisGo.map_reduced_results_to_full_grid`. See that + method (and the core function it wraps, + :func:`~.tools.spatial_complexity_reduction.apply_reduced_results_to_full_grid`) + for the matching/disaggregation rules. + + A no-op when ``ctx.full_grid_stash`` is ``None`` — i.e. when + ``spatial_reduce`` did not run or ran disabled — so ``edisgo`` (the + grid ``optimize`` already ran on) is returned unchanged. + + Parameters + ---------- + edisgo : edisgo.EDisGo + The reduced EDisGo instance ``optimize`` ran on. + ctx : RunContext + Run context. Reads ``ctx.full_grid_stash`` and the + ``flexible_cps`` / ``flexible_hps`` / ``flexible_loads`` / + ``flexible_storage_units`` flags ``optimize`` set. Clears + ``ctx.full_grid_stash`` back to ``None`` after restoring. + **overrides + Unused; accepted for signature consistency with other tasks. + + Returns + ------- + edisgo.EDisGo + The full-grid EDisGo instance with flexible dispatch restored, or + ``edisgo`` unchanged if there is no stash to restore from. + + """ + full_grid = ctx.full_grid_stash + if full_grid is None: + return edisgo + + full_grid.map_reduced_results_to_full_grid( + reduced_grid=edisgo, + flexible_cps=ctx.flags.get("flexible_cps"), + flexible_hps=ctx.flags.get("flexible_hps"), + flexible_loads=ctx.flags.get("flexible_loads"), + flexible_storage_units=ctx.flags.get("flexible_storage_units"), + ) + ctx.full_grid_stash = None + return full_grid diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py new file mode 100644 index 000000000..bbe776398 --- /dev/null +++ b/edisgo/run/tasks/timeseries.py @@ -0,0 +1,629 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +""" +Time-series tasks — set active/reactive power profiles on EDisGo. + +Time series drive every downstream step: ``analyze``, ``reinforce`` +and ``optimize`` all operate on the time index and power time series +attached to the EDisGo object. The order inside a stage matters: + +1. Set the time index and active-power profiles with one of + :func:`task_worst_case_ts`, :func:`task_oedb_ts`, + :func:`task_manual_ts`, possibly :func:`task_set_timeindex`. +2. Optionally reduce the time index to a selected subset with + :func:`task_select_timesteps` (manual before the imports, auto after + ``import_overlying_grid_data``). +3. Finally call :func:`task_reactive_power` to fix reactive power + control — this MUST come last because it overwrites whatever + reactive power was set by the earlier steps. +""" + +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + + +@register_task("worst_case_ts", provides={"timeseries"}, ts_altering=True) +def task_worst_case_ts( + edisgo, + ctx, + *, + cases=None, + generators_names=None, + loads_names=None, + storage_units_names=None, +): + """ + Set synthetic worst-case active-power time series. + + Produces two snapshots (load case and feed-in case) that + represent the network's extremes. Useful for a coarse first + reinforce that does not require real load/generation data. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['timeseries_set'] = True``. + cases : list of str, optional + Subset of ``{"load_case", "feed-in_case"}``. Default is both. + generators_names : list of str, optional + Restrict to these generator names; default is all. + loads_names : list of str, optional + Restrict to these load names; default is all. + storage_units_names : list of str, optional + Restrict to these storage units; default is all. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.set_time_series_worst_case_analysis( + cases=cases, + generators_names=generators_names, + loads_names=loads_names, + storage_units_names=storage_units_names, + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("set_timeindex", provides={"timeseries"}, ts_altering=True) +def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, freq="h"): + """ + Set the time index on the EDisGo object. + + Useful as a stand-alone step when you want a specific hourly + range without immediately attaching time-series data (the + ``oedb_ts`` task already accepts a ``timeindex`` argument and + does this internally). + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + start : str or pandas.Timestamp + First timestamp of the range. + periods : int, optional + Number of periods; mutually exclusive with ``end``. + end : str or pandas.Timestamp, optional + Last timestamp; mutually exclusive with ``periods``. + freq : str, optional + pandas frequency string, default hourly (``"h"``). + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + Raises + ------ + ValueError + If neither ``periods`` nor ``end`` is provided. + + """ + from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex + + if end is not None: + timeindex = pd.date_range(start=start, end=end, freq=freq) + else: + if periods is None: + raise ValueError("set_timeindex needs either 'periods' or 'end'.") + timeindex = pd.date_range(start=start, periods=periods, freq=freq) + if edisgo.timeseries.timeindex.empty: + edisgo.set_timeindex(timeindex) + else: + reduce_timeseries_data_to_given_timeindex(edisgo, timeindex) + return edisgo + + +@register_task("oedb_ts", provides={"timeseries"}, ts_altering=True) +def task_oedb_ts( + edisgo, + ctx, + *, + timeindex=None, + dispatchable=None, + fluctuating="oedb", + conventional_loads="oedb", + charging_points_ts=None, +): + """ + Set active-power time series from egon_data (OEP) plus overrides. + + This is the "real data" path: wind and solar profiles come from + ``egon_era5_renewable_feedin``, conventional loads come from the + egon demand tables. Dispatchable generators (conventional, + etc.) are set via a per-technology-type profile since egon_data + does not dispatch them. Storage units default to zero if not + already set. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()`` when any source is ``"oedb"``. Sets + ``ctx.flags['timeseries_set'] = True``. + timeindex : dict, optional + ``{"start": ..., "periods": N, "freq": "h"}``. If present, a + matching :class:`~pandas.DatetimeIndex` is set before + importing data. + dispatchable : dict, optional + Per-technology scaling factors, e.g. ``{"other": 0.7}`` → + constant profile of 0.7 p.u. for all non-fluctuating + generators of type "other". + fluctuating : str or pandas.DataFrame, optional + How to populate wind/solar. ``"oedb"`` pulls egon_data, + ``"default"`` uses bundled standard profiles, or a DataFrame + with columns "solar" / "wind" is passed through. + conventional_loads : str, optional + Source for conventional loads (not heat pumps / charging + points). ``"oedb"`` or ``"demandlib"``. + charging_points_ts : pandas.DataFrame, optional + Explicit active-power profile for charging points; default + ``None`` leaves them untouched so + :func:`task_apply_charging_strategy` can set them. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if timeindex is not None: + ti_df = pd.date_range( + start=timeindex["start"], + periods=timeindex["periods"], + freq=timeindex.get("freq", "h"), + ) + edisgo.set_timeindex(ti_df) + elif edisgo.timeseries.timeindex.empty: + # No explicit timeindex and none set yet (e.g. no manual time series + # earlier): fall back to a full year derived from the scenario, the + # same default the flex imports use. + from edisgo.tools.tools import get_year_based_on_scenario + + year = get_year_based_on_scenario(ctx.scenario) + if year is None: + raise ValueError( + f"Cannot derive a default time index: invalid scenario " + f"{ctx.scenario!r}. Provide a 'timeindex' or a valid scenario " + f"('eGon2035', 'eGon100RE')." + ) + edisgo.set_timeindex(pd.date_range(f"1/1/{year}", periods=8760, freq="h")) + + dispatchable_df = None + if dispatchable is not None: + ti = edisgo.timeseries.timeindex + dispatchable_df = pd.DataFrame(dispatchable, index=ti) + + conv_loads_names = None + if conventional_loads == "oedb": + conv_loads_names = edisgo.topology.loads_df.loc[ + ~edisgo.topology.loads_df.type.isin(["heat_pump", "charging_point"]) + ].index.tolist() + + edisgo.set_time_series_active_power_predefined( + fluctuating_generators_ts=fluctuating, + conventional_loads_ts=conventional_loads, + conventional_loads_names=conv_loads_names, + dispatchable_generators_ts=dispatchable_df, + charging_points_ts=charging_points_ts, + scenario=ctx.scenario, + engine=ctx.ensure_engine() + if fluctuating == "oedb" or conventional_loads == "oedb" + else None, + ) + + su_names = edisgo.topology.storage_units_df.index + if len(su_names) > 0 and edisgo.timeseries.storage_units_active_power.empty: + edisgo.timeseries.storage_units_active_power = pd.DataFrame( + 0.0, + index=edisgo.timeseries.timeindex, + columns=su_names, + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("manual_ts", provides={"timeseries"}, ts_altering=True) +def task_manual_ts( + edisgo, + ctx, + *, + generators_active_power=None, + generators_reactive_power=None, + loads_active_power=None, + loads_reactive_power=None, + storage_units_active_power=None, + storage_units_reactive_power=None, +): + """ + Set active/reactive power time series from explicit DataFrames. + + Used when the caller already has the raw profiles (e.g. from a + coupled run) and wants to inject them directly. Any argument left + at ``None`` is not touched. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['timeseries_set'] = True``. + generators_active_power : dict or pandas.DataFrame, optional + Generator active-power profile(s). Converted via + :class:`pandas.DataFrame`. + generators_reactive_power : dict or pandas.DataFrame, optional + Generator reactive-power profile(s). + loads_active_power : dict or pandas.DataFrame, optional + Load active-power profile(s). + loads_reactive_power : dict or pandas.DataFrame, optional + Load reactive-power profile(s). + storage_units_active_power : dict or pandas.DataFrame, optional + Storage-unit active-power profile(s). + storage_units_reactive_power : dict or pandas.DataFrame, optional + Storage-unit reactive-power profile(s). + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + + def _as_df(obj): + return pd.DataFrame(obj) if obj is not None else None + + edisgo.set_time_series_manual( + generators_p=_as_df(generators_active_power), + generators_q=_as_df(generators_reactive_power), + loads_p=_as_df(loads_active_power), + loads_q=_as_df(loads_reactive_power), + storage_units_p=_as_df(storage_units_active_power), + storage_units_q=_as_df(storage_units_reactive_power), + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +def _set_default_full_year_timeindex(edisgo, ctx): + """ + Set a full-year hourly time index derived from the scenario. + + Used as a fallback so time-index-dependent imports (notably the EV + flexibility bands built in ``import_electromobility``) run on an hourly, + full-year index rather than their raw source resolution. The year is only a + label — the DB imports fetch scenario-correct data regardless — and the index + can be overridden later (e.g. by ``oedb_ts`` or the auto ``select_timesteps`` + step). + """ + from edisgo.tools.tools import get_year_based_on_scenario + + year = get_year_based_on_scenario(ctx.scenario) or 2011 + edisgo.set_timeindex(pd.date_range(f"1/1/{year}", periods=8760, freq="h")) + ctx.logger.info( + f"select_timesteps: no time index set; using default full year " + f"{year} (8760 h) so imports build hourly full-year data." + ) + + +@register_task("select_timesteps", provides={"timeseries"}, ts_altering=True) +def task_select_timesteps(edisgo, ctx, **overrides): + """ + Select the time steps the grid is analyzed/optimized for. + + Reduces the time index to a configurable subset. Configuration is + read from the top-level ``timeseries_selection:`` config block (so + eGo can inject it the same way it injects ``overlying_grid``); + inline step params override individual keys of that block. Two + modes: + + ``manual`` + Reduce to an explicit set of time steps. Positioned *before* + the data-import tasks so ``import_heat_pumps`` / ``import_dsm`` + download only the requested steps. The selected index is + stashed in ``ctx.flags['selected_timeindex']`` for those + imports to pick up. + + ``auto`` + Determine the two most critical time intervals and reduce to + them. Must be positioned *after* ``import_overlying_grid_data`` + (needs all active-power time series) and *before* + ``reactive_power``. Two ``method`` options: + + * ``power_flow`` (default) — score intervals via a power flow + (:func:`~.tools.temporal_complexity_reduction.get_most_critical_time_intervals`). + A reactive-power series is set internally to run the scoring + power flow, but ``ctx.flags['reactive_power_set']`` is left + unset so the pipeline's own ``reactive_power`` step still runs + on the reduced index. + * ``residual_load`` — no power flow. The overlying-grid dispatch + is distributed onto the components and the residual load is + ranked over the whole year; intervals are centered on the + highest (load case) and lowest (feed-in case) residual-load + steps. Requires overlying-grid data to be present. + + Both methods delegate to + :func:`~.tools.temporal_complexity_reduction.get_most_critical_time_intervals` + (via its ``by`` parameter) and reduce to a non-overlapping pair + chosen by + :func:`~.tools.temporal_complexity_reduction.select_two_intervals`. + + The auto mode normally yields two disconnected intervals (one for + overloading, one for voltage issues). These are kept separate in the + resulting time index (there is a gap between them). If they overlap, + a non-overlapping pair is chosen if possible, otherwise they are + concatenated into one interval. The intervals themselves are not + stored — a later ``optimize`` step can detect the gap in the time + index and run separate optimizations per interval. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Reads ``ctx.raw_config['timeseries_selection']``. + Sets ``ctx.flags['selected_timeindex']`` (manual) and + ``ctx.flags['timesteps_selected'] = True``. + **overrides + Inline step params overriding keys of the + ``timeseries_selection`` block. Recognized keys: + ``position`` (``"pre_import"`` | ``"post_grid"``, optional) — the + step only acts when the configured ``mode`` matches this + position (``pre_import`` ↔ ``manual``, ``post_grid`` ↔ ``auto``), + otherwise it is a no-op; this lets one pipeline carry both a + pre-import and a post-grid ``select_timesteps`` step and support + either mode via config. When omitted, the step always acts. + ``mode`` (``"manual"`` | ``"auto"``); + for manual: ``timestamps`` (list) or ``start`` / + ``periods`` / ``end`` / ``freq``; + for auto: ``method`` (``"power_flow"`` (default) | + ``"residual_load"``), ``time_steps_per_time_interval``; + for ``method="power_flow"`` additionally ``percentage``, + ``time_step_day_start`` (default 4), ``save_steps`` (default + True; CSV written to ``ctx.results_dir``), + ``use_troubleshooting_mode``, ``overloading_factor``, + ``voltage_deviation_factor``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + Raises + ------ + ValueError + If ``mode`` is missing/unknown, if manual mode has neither + ``timestamps`` nor a range, or if auto mode runs before active + power time series are set. + """ + from edisgo.tools.temporal_complexity_reduction import ( + get_most_critical_time_intervals, + select_two_intervals, + ) + from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex + + cfg = {**ctx.raw_config.get("timeseries_selection", {}), **overrides} + mode = cfg.get("mode") + if mode not in ("manual", "auto"): + raise ValueError( + f"select_timesteps needs mode 'manual' or 'auto', got {mode!r}." + ) + + # A pipeline may include two select_timesteps steps — one before the + # imports (``position: pre_import``, where manual selection belongs) and + # one after import_overlying_grid_data (``position: post_grid``, where auto + # selection belongs) — so the same preset supports both modes. Each step + # only acts when the configured mode matches its position; otherwise it is + # a no-op. When ``position`` is omitted (single-step usage) the step always + # acts. + position = overrides.get("position") + expected_mode = {"pre_import": "manual", "post_grid": "auto"} + if position is not None: + if position not in expected_mode: + raise ValueError( + f"select_timesteps 'position' must be 'pre_import' or " + f"'post_grid', got {position!r}." + ) + if mode != expected_mode[position]: + # This positioned step is not the active selector. If it is the + # pre-import step and no time index has been set yet (i.e. manual + # selection is not driving the index), establish a full-year default + # so the following imports build their time-index-dependent data + # (e.g. EV flexibility bands) on an hourly full-year index. Only a + # label — DB imports fetch scenario-correct data regardless — and it + # is overridden later by oedb_ts / the auto select_timesteps step. + if position == "pre_import" and edisgo.timeseries.timeindex.empty: + _set_default_full_year_timeindex(edisgo, ctx) + ctx.logger.debug( + f"select_timesteps at position {position!r} is a no-op for " + f"mode {mode!r}." + ) + return edisgo + + if mode == "manual": + timestamps = cfg.get("timestamps") + if timestamps is not None: + timeindex = pd.DatetimeIndex(pd.to_datetime(list(timestamps))) + elif cfg.get("end") is not None: + timeindex = pd.date_range( + start=cfg["start"], end=cfg["end"], freq=cfg.get("freq", "h") + ) + elif cfg.get("periods") is not None: + timeindex = pd.date_range( + start=cfg["start"], + periods=cfg["periods"], + freq=cfg.get("freq", "h"), + ) + else: + raise ValueError( + "select_timesteps manual mode needs 'timestamps' or a " + "'start' plus 'periods'/'end' range." + ) + timeindex = timeindex.sort_values().unique() + if not edisgo.timeseries.timeindex.empty: + # A time index is already set (manual selection reducing an existing + # full time series): align the user-supplied timestamps to that + # index's year so date-based slicing matches even if the user wrote + # them in a different (e.g. scenario) year than the internally used + # reference year. + year_diff = edisgo.timeseries.timeindex[0].year - timeindex[0].year + if year_diff != 0: + timeindex = timeindex + pd.DateOffset(years=year_diff) + ctx.flags["selected_timeindex"] = timeindex + if edisgo.timeseries.timeindex.empty: + # positioned before imports: just set the index so HP/DSM + # imports restrict their downloads to it + edisgo.set_timeindex(timeindex) + else: + reduce_timeseries_data_to_given_timeindex(edisgo, timeindex) + ctx.logger.info( + f"select_timesteps (manual): selected {len(timeindex)} time steps." + ) + ctx.flags["timesteps_selected"] = True + return edisgo + + # auto mode + if not ctx.flags.get("timeseries_set"): + raise ValueError( + "select_timesteps mode 'auto' needs active-power time series to " + "be set first (e.g. run oedb_ts before it)." + ) + + method = cfg.get("method", "power_flow") + if method not in ("power_flow", "residual_load"): + raise ValueError( + f"select_timesteps auto 'method' must be 'power_flow' or " + f"'residual_load', got {method!r}." + ) + tsp = cfg.get("time_steps_per_time_interval", 168) + + if method == "residual_load": + # residual-load selection requires overlying-grid data (the dispatch + # distributed onto the components); guard here (mode selection) before + # delegating the computation to the tools function. + og = edisgo.overlying_grid + if all( + s.empty + for s in ( + og.electromobility_active_power, + og.storage_units_active_power, + og.heat_pump_central_active_power, + og.heat_pump_decentral_active_power, + og.dsm_active_power, + og.renewables_curtailment, + ) + ): + raise ValueError( + "select_timesteps method 'residual_load' needs overlying-grid " + "data to be present (run import_overlying_grid_data before it)." + ) + col_a, col_b = "time_steps_load_case", "time_steps_feedin_case" + else: # power_flow + # throwaway reactive power so the scoring power flow yields meaningful + # voltages; do NOT mark reactive_power_set — the pipeline's own + # reactive_power step runs afterwards on the reduced index. + edisgo.set_time_series_reactive_power_control(control="fixed_cosphi") + col_a, col_b = "time_steps_overloading", "time_steps_voltage_issues" + + intervals_df = get_most_critical_time_intervals( + edisgo, + by=method, + percentage=cfg.get("percentage", 1.0), + time_steps_per_time_interval=tsp, + time_step_day_start=cfg.get("time_step_day_start", 4), + save_steps=cfg.get("save_steps", True), + path=str(ctx.results_dir) if ctx.results_dir is not None else "", + use_troubleshooting_mode=cfg.get("use_troubleshooting_mode", True), + overloading_factor=cfg.get("overloading_factor", 0.95), + voltage_deviation_factor=cfg.get("voltage_deviation_factor", 0.95), + ) + intervals = select_two_intervals( + list(intervals_df.get(col_a, [])), + list(intervals_df.get(col_b, [])), + ) + + if not intervals: + raise ValueError( + "select_timesteps mode 'auto' found no critical time intervals; " + "cannot reduce the time index." + ) + + timeindex = intervals[0] + for interval in intervals[1:]: + timeindex = timeindex.union(interval) + timeindex = timeindex.sort_values() + + reduce_timeseries_data_to_given_timeindex(edisgo, timeindex) + ctx.logger.info( + f"select_timesteps (auto): selected {len(intervals)} interval(s), " + f"{len(timeindex)} time steps total." + ) + ctx.flags["timesteps_selected"] = True + return edisgo + + +@register_task("reactive_power") +def task_reactive_power( + edisgo, + ctx, + *, + control="fixed_cosphi", + generators_parametrisation="default", + loads_parametrisation="default", + storage_units_parametrisation="default", +): + """ + Apply reactive-power control on top of the active-power time series. + + This MUST be the last time-series-altering step before + ``analyze`` / ``reinforce`` / ``optimize``. The validator + enforces this ordering rule statically. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['reactive_power_set'] = True``. + control : str, optional + Reactive-power control strategy; typically ``"fixed_cosphi"``. + generators_parametrisation : str or dict, optional + Per-generator parametrisation, ``"default"`` uses the config. + loads_parametrisation : str or dict, optional + Per-load parametrisation. + storage_units_parametrisation : str or dict, optional + Per-storage-unit parametrisation. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.set_time_series_reactive_power_control( + control=control, + generators_parametrisation=generators_parametrisation, + loads_parametrisation=loads_parametrisation, + storage_units_parametrisation=storage_units_parametrisation, + ) + ctx.flags["reactive_power_set"] = True + return edisgo diff --git a/edisgo/run/validator.py b/edisgo/run/validator.py new file mode 100644 index 000000000..5ca548499 --- /dev/null +++ b/edisgo/run/validator.py @@ -0,0 +1,192 @@ +""" +Static validator for pipeline configs. + +The validator enforces structural and ordering rules that the runner +would otherwise hit at execution time — often after 20 minutes of work. +Running these checks up-front turns "cryptic AttributeError after half +the pipeline" into a clear ``ValueError`` at startup. + +Checked rules: + +* every step maps to a known, registered task name; +* ``reactive_power`` comes after every time-series task in a stage, + never before — ``set_time_series_reactive_power_control`` overwrites + reactive power on the currently set active-power time series; +* ``analyze`` and ``reinforce`` require a time-series task earlier in + the stage (or a ``load_from:`` that brings a prepared grid); +* ``optimize`` requires both a time-series task and at least one flex + import earlier in the stage — OPF without flexibility is meaningless; +* flex imports (``import_heat_pumps``, …) require a loaded grid, i.e. + an earlier ``setup_grid`` / ``load_from_base`` / a stage-level + ``load_from:``; +* ``base_reinforce`` likewise requires a loaded grid; +* a stage that declares ``load_from: X`` can only run if stage ``X`` + ran earlier AND contains a ``save`` step. +""" +from __future__ import annotations + +from typing import Any + +from edisgo.run.registry import get_task_meta, known_tasks + +# Human-readable message per required capability. The wording keeps the +# substrings the validator tests assert on ("loaded grid", "time series", +# "flex asset"). +_REQUIREMENT_MESSAGES = { + "grid": "requires a loaded grid (setup_grid or load_from_base) before it", + "timeseries": ( + "requires time series to be set (e.g. worst_case_ts or oedb_ts) " + "before it" + ), + "flex": "requires at least one flex asset to be imported", +} +# Order in which a missing capability is reported when several are missing. +_REQUIREMENT_PRIORITY = ("grid", "timeseries", "flex") + + +def validate(cfg: dict) -> None: + """ + Validate a normalized pipeline config against the ordering rules. + + This function does not return a value. On success it simply + returns; on any rule violation it raises :class:`ValueError` with + a message identifying the offending stage and task. + + Parameters + ---------- + cfg : dict + Normalized config as returned by + :func:`edisgo.run.config.load_config`. Must have a ``stages`` + list at the top level. + + Raises + ------ + ValueError + If the config has no stages, an unknown task name, a + structural problem (reactive before TS, reinforce without TS, + optimize without flex, flex import without grid, …), or a + stage references a ``load_from`` source that doesn't exist or + has no ``save`` step. + + """ + stages = cfg.get("stages") or [] + if not stages: + raise ValueError("Config has no stages to run.") + + known = set(known_tasks()) + available_artifacts: set[str] = set() + + for stage in stages: + name = stage["name"] + pipeline = stage.get("pipeline") or [] + load_from = stage.get("load_from") + + if load_from is not None and load_from not in available_artifacts: + raise ValueError( + f"Stage '{name}' requires 'load_from: {load_from}' but " + f"that stage has not run or did not save. Available: " + f"{sorted(available_artifacts)}" + ) + + # Capabilities established so far in this stage. A stage-level + # load_from reloads the grid topology only — _load_artifact drops + # time series and flex data (import_timeseries=False) — so it + # provides "grid" but NOT "timeseries"/"flex". A task's requirements + # must therefore be satisfied by tasks run in this stage itself. + satisfied: set[str] = {"grid"} if load_from is not None else set() + reactive_set = False + has_save = False + + for step in pipeline: + task_name, _params = _split_step(step) + if task_name not in known: + raise ValueError( + f"Unknown task '{task_name}' in stage '{name}'. " + f"Known: {sorted(known)}" + ) + + meta = get_task_meta(task_name) + + # reactive_power must be the last time-series-altering step. + if meta.ts_altering and reactive_set: + raise ValueError( + f"Stage '{name}': time-series task '{task_name}' comes " + f"after 'reactive_power' — reactive_power must be the " + f"last time-series-altering step." + ) + + # Check declared requirements against what the stage provides. + missing = meta.requires - satisfied + if missing: + cap = next( + (c for c in _REQUIREMENT_PRIORITY if c in missing), + sorted(missing)[0], + ) + detail = _REQUIREMENT_MESSAGES.get( + cap, f"requires '{cap}' to be established before it" + ) + raise ValueError( + f"Stage '{name}': task '{task_name}' {detail}." + ) + + satisfied |= meta.provides + if task_name == "reactive_power": + reactive_set = True + if task_name == "save": + has_save = True + + if has_save: + available_artifacts.add(name) + + +def _split_step(step: Any) -> tuple[str, dict]: + """ + Normalize a pipeline step into ``(task_name, params)``. + + Steps are allowed in two forms in YAML/JSON: + + * bare string — ``worst_case_ts`` → ``("worst_case_ts", {})`` + * single-key mapping — + ``import_electromobility: {charging_strategy: dumb}`` + → ``("import_electromobility", {"charging_strategy": "dumb"})`` + + ``None`` as the parameter value is treated as an empty dict so + that YAML's ``task:`` (with nothing after the colon) works. + + Parameters + ---------- + step : str or dict + Raw step as it appears in the pipeline list. + + Returns + ------- + tuple of (str, dict) + The task name and its keyword arguments. + + Raises + ------ + ValueError + If ``step`` is not a string or a single-key mapping, or if + the parameter value is not a mapping. + + """ + if isinstance(step, str): + return step, {} + if isinstance(step, dict): + if len(step) != 1: + raise ValueError( + f"Task step must be a string or single-key mapping, " + f"got: {step}" + ) + (name, params), = step.items() + if params is None: + params = {} + if not isinstance(params, dict): + raise ValueError( + f"Parameters for task '{name}' must be a mapping, " + f"got: {type(params).__name__}" + ) + return name, params + raise ValueError( + f"Task step must be string or mapping, got: {step!r}" + ) diff --git a/edisgo/tools/config.py b/edisgo/tools/config.py index 6111c1483..2a986e81e 100644 --- a/edisgo/tools/config.py +++ b/edisgo/tools/config.py @@ -312,12 +312,19 @@ def import_tables_from_oep( table_name, metadata, autoload_with=engine, schema=schema_name ) + # The declarative mapper requires a primary key. Some egon-data + # tables/views have none reflected; declare all columns as a + # composite primary key so the ORM class can be built. This + # mirrors what saio does on the OEP path ("assuming primary + # key") and only affects mapping, not the data read back. + class_dict = {"__tablename__": table_name, "__table__": table} + if not list(table.primary_key.columns): + class_dict["__mapper_args__"] = { + "primary_key": list(table.columns) + } + # dynamisch eine ORM-Klasse erzeugen - orm_class = type( - table_name, - (Base,), - {"__tablename__": table_name, "__table__": table}, - ) + orm_class = type(table_name, (Base,), class_dict) orm_classes.append(orm_class) return orm_classes diff --git a/edisgo/tools/spatial_complexity_reduction.py b/edisgo/tools/spatial_complexity_reduction.py index 21e8f8665..ad9a51fad 100644 --- a/edisgo/tools/spatial_complexity_reduction.py +++ b/edisgo/tools/spatial_complexity_reduction.py @@ -1916,6 +1916,237 @@ def spatial_complexity_reduction( return busmap_df, linemap_df +def apply_reduced_results_to_full_grid( + full_grid: EDisGo, + reduced_grid: EDisGo, + *, + flexible_cps: list | None = None, + flexible_hps: list | None = None, + flexible_loads: list | None = None, + flexible_storage_units: list | None = None, +) -> EDisGo: + """ + Write optimized flexible-component dispatch from a spatially-reduced grid + back onto the full grid. + + Counterpart to :func:`spatial_complexity_reduction`: where that function + shrinks a grid for a faster OPF, this function maps the OPF's active-power + results back onto the pre-reduction grid so reinforcement can run on the + full topology. Only components the OPF actually rewrites are touched — + flexible charging points, heat pumps, DSM loads, and storage units. + Inflexible loads/generators are untouched: the OPF never changed their + series, so ``full_grid`` already holds the correct values for them. + + ``full_grid`` and ``reduced_grid`` are matched by name for storage units + (never aggregated by :func:`spatial_complexity_reduction`, so their names + are unchanged) and, for the other three flexibility types, by the + ``old_name`` column that :func:`spatial_complexity_reduction` writes onto + ``reduced_grid.topology.loads_df`` when ``aggregation_mode=True``. A + member listed in ``old_name`` is a load whose active-power series was + merged into one representative row; when ``aggregation_mode=False`` (or a + given member was not merged), ``old_name`` is absent and the member's own + name is used directly — i.e. a plain by-name write-back. + + For a merged representative, the representative's optimized series is + disaggregated onto its ``old_name`` members **per time step**, weighted by + each member's own pre-OPF flexibility envelope (a known input, never the + optimized result): + + * charging points — ``upper_power(t)`` from + ``electromobility.flexibility_bands`` (a charging point with no + connected vehicle has ``upper_power(t) == 0``, so it receives none of + the representative's dispatch that time step); + * heat pumps — ``min(heat_demand(t) / cop(t), p_set)``, i.e. the + electrical-equivalent heat demand capped at the heat pump's own rated + power, mirroring how a charging point's ``upper_power(t)`` is already a + capped bound rather than raw uncapped demand; + * DSM loads — ``p_max(t)`` from :attr:`~.network.dsm.DSM.p_max`. + + Weights always sum back to the representative's value exactly at every + time step; a time step where every member's weight is 0 falls back to an + equal split. + + Reactive power is not read from ``reduced_grid``. After writing active + power, this function calls + :meth:`~.EDisGo.set_time_series_reactive_power_control` on ``full_grid`` + with its defaults, mirroring how the OPF itself derives reactive power + for the components it just optimized (see + :func:`~.io.powermodels_io.from_powermodels`) — reactive power is always + a function of whatever active power is currently set, regardless of + whether that active power came from a default, worst case, or the OPF. + + Parameters + ---------- + full_grid : :class:`~.EDisGo` + The pre-reduction EDisGo instance to write dispatch onto, modified in + place. Must contain every component named in ``flexible_cps`` / + ``flexible_hps`` / ``flexible_loads`` / ``flexible_storage_units`` and + (for merged components) every name listed in ``reduced_grid``'s + ``old_name`` columns. + reduced_grid : :class:`~.EDisGo` + The spatially-reduced EDisGo instance the OPF ran on. Supplies the + optimized active-power series and, if aggregated, the ``old_name`` + provenance. + flexible_cps : list of str, optional + Names of flexible charging points in ``reduced_grid`` to map back. + flexible_hps : list of str, optional + Names of flexible heat-pump loads in ``reduced_grid`` to map back. + flexible_loads : list of str, optional + Names of flexible DSM loads in ``reduced_grid`` to map back. + flexible_storage_units : list of str, optional + Names of flexible storage units in ``reduced_grid`` to map back. + + Returns + ------- + :class:`~.EDisGo` + ``full_grid``, with active power written for the given flexible + components and reactive power recomputed. + + """ + # NOTE: "x or []" is unsafe here - callers may pass a numpy array (e.g. + # task_optimize derives flexible_loads as + # edisgo.dsm.p_min.columns.values), and "array or []" raises + # ValueError ("truth value of an array... is ambiguous") for any array + # with more than one element. "is None" is the correct emptiness check + # for an optional list-like argument. + flexible_cps = list(flexible_cps) if flexible_cps is not None else [] + flexible_hps = list(flexible_hps) if flexible_hps is not None else [] + flexible_loads = list(flexible_loads) if flexible_loads is not None else [] + flexible_storage_units = ( + list(flexible_storage_units) if flexible_storage_units is not None else [] + ) + + def _require_full_timeindex(envelope: DataFrame, envelope_name: str) -> None: + """Raise a clear error if ``envelope`` doesn't cover the full grid's + active time index, instead of a bare ``KeyError`` deep inside a + ``.loc`` lookup. + + This can only happen if ``full_grid``'s flexibility-band/DSM/heat-pump + attributes were never trimmed to the same time index as + ``full_grid.timeseries.timeindex`` - i.e. if the pre-reduction stash + was taken before the run's time index was finalized. + """ + ti = full_grid.timeseries.timeindex + missing = ti.difference(envelope.index) + if len(missing) > 0: + raise ValueError( + f"apply_reduced_results_to_full_grid: full_grid's " + f"{envelope_name} does not cover {len(missing)} of " + f"full_grid.timeseries.timeindex's time steps (e.g. " + f"{missing[0]!r}). This usually means the full-grid stash " + f"was taken before the time index was finalized - run " + f"time-index selection (e.g. select_timesteps) before " + f"spatial_reduce." + ) + + def _old_name_map(loads_df: DataFrame, names: list) -> dict: + """Map each representative name in ``names`` to its member names. + + A name absent from ``old_name`` (not merged, or + ``aggregation_mode=False``) maps to itself. + """ + name_map = {} + for name in names: + old_name = loads_df.at[name, "old_name"] if "old_name" in loads_df else None + name_map[name] = old_name if isinstance(old_name, list) else [name] + return name_map + + def _write_by_name(active_power: DataFrame, names: list, target: DataFrame) -> None: + ti = full_grid.timeseries.timeindex + target.loc[ti, names] = active_power.loc[ti, names].values + + def _disaggregate( + active_power: DataFrame, + name_map: dict, + envelope: DataFrame, + target: DataFrame, + ) -> None: + """Split each representative's series onto its members per time step. + + ``envelope`` holds each member's pre-OPF flexibility envelope + (columns = member names, index = time index); members missing from + ``envelope`` are treated as having an all-zero envelope (equal-split + fallback). + """ + ti = full_grid.timeseries.timeindex + for representative, members in name_map.items(): + if len(members) == 1 and members[0] == representative: + target.loc[ti, representative] = active_power.loc[ti, representative] + continue + weights = pd.DataFrame(index=ti, columns=members, dtype=float) + for member in members: + weights[member] = ( + envelope.loc[ti, member] if member in envelope.columns else 0.0 + ) + weight_sum = weights.sum(axis="columns") + zero_envelope = weight_sum == 0 + shares = weights.div(weight_sum.replace(0, np.nan), axis="index") + shares.loc[zero_envelope, :] = 1.0 / len(members) + representative_power = active_power.loc[ti, representative] + for member in members: + target.loc[ti, member] = shares[member] * representative_power + + reduced_loads_df = reduced_grid.topology.loads_df + full_loads_df = full_grid.topology.loads_df + + # Always routed through _disaggregate (never the by-name fast path): under + # aggregation_mode=True, spatial_complexity_reduction renames EVERY group's + # representative row, including singleton groups (a bus with exactly one + # flexible load of a given type/sector) - so a representative's own name + # can differ from its single old_name member's name. _disaggregate already + # handles that case correctly (a singleton's one weight, whether zero or + # not, always resolves its share to the representative's full value), so + # there is no correct case left for a by-name fast path to shortcut. + if flexible_cps: + name_map = _old_name_map(reduced_loads_df, flexible_cps) + envelope = reduced_grid.electromobility.flexibility_bands["upper_power"] + _require_full_timeindex(envelope, "electromobility.flexibility_bands") + _disaggregate( + reduced_grid.timeseries.loads_active_power, + name_map, + envelope, + full_grid.timeseries._loads_active_power, + ) + + if flexible_hps: + name_map = _old_name_map(reduced_loads_df, flexible_hps) + members_flat = [m for members in name_map.values() for m in members] + heat_demand = full_grid.heat_pump.heat_demand_df[members_flat] + cop = full_grid.heat_pump.cop_df[members_flat] + p_set = full_loads_df.p_set[members_flat] + envelope = (heat_demand / cop).clip(upper=p_set, axis="columns") + _require_full_timeindex(envelope, "heat_pump.heat_demand_df/cop_df") + _disaggregate( + reduced_grid.timeseries.loads_active_power, + name_map, + envelope, + full_grid.timeseries._loads_active_power, + ) + + if flexible_loads: + name_map = _old_name_map(reduced_loads_df, flexible_loads) + _require_full_timeindex(full_grid.dsm.p_max, "dsm.p_max") + _disaggregate( + reduced_grid.timeseries.loads_active_power, + name_map, + full_grid.dsm.p_max, + full_grid.timeseries._loads_active_power, + ) + + if flexible_storage_units: + # Storage units are never aggregated by spatial_complexity_reduction + # (only bus-relabeled), so this is always a plain by-name write-back. + _write_by_name( + reduced_grid.timeseries.storage_units_active_power, + flexible_storage_units, + full_grid.timeseries._storage_units_active_power, + ) + + full_grid.set_time_series_reactive_power_control() + + return full_grid + + def compare_voltage( edisgo_unreduced: EDisGo, edisgo_reduced: EDisGo, diff --git a/edisgo/tools/temporal_complexity_reduction.py b/edisgo/tools/temporal_complexity_reduction.py index 2adc269f9..df4bb0730 100644 --- a/edisgo/tools/temporal_complexity_reduction.py +++ b/edisgo/tools/temporal_complexity_reduction.py @@ -647,6 +647,189 @@ def _troubleshooting_mode( return edisgo_obj +def intervals_overlap(a, b): + """ + Return True if two contiguous time-step intervals overlap. + + Each interval is a :pandas:`pandas.DatetimeIndex` of + contiguous, sorted time steps. Overlap is checked on the closed + ``[min, max]`` ranges, so intervals that merely touch (share an end point) + count as overlapping. + """ + return (a.min() <= b.max()) and (b.min() <= a.max()) + + +def select_two_intervals(load_case, feedin_case): + """ + Pick the time intervals to analyze from two ranked candidate lists. + + Used to reduce the ranked most-critical intervals (e.g. from + :func:`get_most_critical_time_intervals`) to the intervals actually analyzed: + + * Start from the most critical interval of each list (index 0). + * If they do not overlap, both are kept — two disconnected intervals are + returned (a later optimization can detect the gap and optimize each + interval separately). + * If they overlap, walk down the second list to the highest ranked interval + that does not overlap the top interval of the first list, and keep that + pair instead. + * If no non-overlapping pair exists, the two most critical intervals are + concatenated into a single contiguous interval and only that one is + returned. + + Parameters + ---------- + load_case : list of pandas.DatetimeIndex + First ranked list of intervals (most critical first). May be empty. + feedin_case : list of pandas.DatetimeIndex + Second ranked list of intervals (most critical first). May be empty. + + Returns + ------- + list of pandas.DatetimeIndex + One or two contiguous, non-overlapping intervals. Empty if both input + lists are empty. + """ + if not load_case and not feedin_case: + return [] + if not load_case: + return [feedin_case[0]] + if not feedin_case: + return [load_case[0]] + + top = load_case[0] + for cand in feedin_case: + if not intervals_overlap(top, cand): + return [top, cand] + + # no non-overlapping pair -> concatenate the two most critical intervals + merged = top.union(feedin_case[0]).sort_values() + start, end = merged.min(), merged.max() + freq = pd.infer_freq(top) or "H" + return [pd.date_range(start=start, end=end, freq=freq)] + + +def _build_centered_interval( + timestep, timeindex, time_steps_per_time_interval, time_step_day_start +): + """ + Build a contiguous interval centered on a critical time step. + + The interval has ``time_steps_per_time_interval`` steps, is centered on + ``timestep``, and its start is snapped to the ``time_step_day_start`` hour of + day (so intervals begin on that hour). The interval is clipped to lie within + ``timeindex``; if centering would run past either end, it is shifted inward. + Centering guarantees the critical step is not the last step of the interval + (where a storage state-of-charge-end constraint would force zero power). + + Parameters + ---------- + timestep : pandas.Timestamp + Critical time step to center on. + timeindex : pandas.DatetimeIndex + The full (sorted) time index the interval must lie within. + time_steps_per_time_interval : int + Interval length in steps. + time_step_day_start : int + Hour of day the interval should start on. + + Returns + ------- + pandas.DatetimeIndex + The contiguous interval. + """ + n = int(time_steps_per_time_interval) + step = timeindex[1] - timeindex[0] + half = (n // 2) * step + # center on the critical step, then snap the start back to the day-start hour + start = timestep - half + while start.hour != int(time_step_day_start): + start = start - step + end = start + (n - 1) * step + # keep the interval within the available time index; shift inward if needed + if start < timeindex[0]: + start = timeindex[0] + end = start + (n - 1) * step + if end > timeindex[-1]: + end = timeindex[-1] + start = end - (n - 1) * step + if start < timeindex[0]: + start = timeindex[0] + return pd.date_range(start=start, end=end, freq=step) + + +def _most_critical_time_intervals_residual_load( + edisgo_obj, + num_time_intervals=None, + percentage=1.0, + time_steps_per_time_interval=168, + time_step_day_start=0, + save_steps=False, + path="", +): + """ + Determine the most critical time intervals from the residual load. + + Ranks the critical single time steps by residual load (via + :func:`get_most_critical_time_steps` with ``by="residual_load"``) and wraps + each into an interval centered on the step and snapped to the + ``time_step_day_start`` hour (see :func:`_build_centered_interval`). Returns a + DataFrame ranked by residual magnitude with per-case columns + ``time_steps_load_case`` (highest residual) and ``time_steps_feedin_case`` + (lowest residual). Overlaps between intervals are allowed (mirroring the + power-flow interval selection); a downstream :func:`select_two_intervals` + picks a non-overlapping pair. + """ + timeindex = edisgo_obj.timeseries.timeindex + + # number of ranked intervals per case + if num_time_intervals is None: + num_time_intervals = int(np.ceil(len(timeindex) * percentage)) + + from edisgo.network.overlying_grid import ( + distribute_overlying_grid_requirements, + ) + + distributed = distribute_overlying_grid_requirements(edisgo_obj) + residual = distributed.timeseries.residual_load + + load_steps = residual.sort_values(ascending=False).index[:num_time_intervals] + feedin_steps = residual.sort_values(ascending=True).index[:num_time_intervals] + + load_intervals = [ + _build_centered_interval( + t, timeindex, time_steps_per_time_interval, time_step_day_start + ) + for t in load_steps + ] + feedin_intervals = [ + _build_centered_interval( + t, timeindex, time_steps_per_time_interval, time_step_day_start + ) + for t in feedin_steps + ] + + steps = pd.DataFrame( + { + "time_steps_load_case": load_intervals, + "time_steps_feedin_case": feedin_intervals, + } + ) + if len(steps) == 0: + logger.info("No critical steps detected. No network expansion required.") + + if save_steps: + abs_path = os.path.abspath(path) + steps.to_csv( + os.path.join( + abs_path, + f"{edisgo_obj.topology.id}_t_{time_steps_per_time_interval}" + f"_residual_load.csv", + ) + ) + return steps + + def get_most_critical_time_intervals( edisgo_obj, num_time_intervals=None, @@ -659,6 +842,7 @@ def get_most_critical_time_intervals( overloading_factor=0.95, voltage_deviation_factor=0.95, weight_by_costs=True, + by="power_flow", ): """ Get time intervals sorted by severity of overloadings as well as voltage issues. @@ -747,15 +931,32 @@ def get_most_critical_time_intervals( time intervals. Default: True. + by : str + Criticality measure used to determine the intervals. Options: + + * "power_flow" (default): run a power flow and score rolling windows by + overloading and voltage violations. Returns columns + ``time_steps_overloading`` / ``time_steps_voltage_issues``. + * "residual_load": no power flow — rank the critical single steps by + residual load (see :func:`get_most_critical_time_steps` with + ``by="residual_load"``) and center an interval on each, snapped to the + ``time_step_day_start`` hour. Returns columns ``time_steps_load_case`` + (highest residual) / ``time_steps_feedin_case`` (lowest residual). + Overlaps between intervals are allowed. + + Default: "power_flow". Returns -------- :pandas:`pandas.DataFrame` - Contains time intervals in which grid expansion needs due to overloading and - voltage issues are detected. The time intervals are determined independently - for overloading and voltage issues and sorted descending by the expected - cumulated grid expansion costs, so that the time intervals with the highest - expected costs correspond to index 0. + Contains time intervals in which grid expansion needs are detected, + ranked most-critical first. Column names depend on ``by`` (see above): + ``time_steps_overloading``/``time_steps_voltage_issues`` for + ``power_flow``, ``time_steps_load_case``/``time_steps_feedin_case`` for + ``residual_load``. For ``power_flow`` the intervals are determined + independently for overloading and voltage issues and sorted descending by + the expected cumulated grid expansion costs, so that the time intervals + with the highest expected costs correspond to index 0. In case of overloading, the time steps in the respective time interval are given in column "time_steps_overloading" and the share of components for which the maximum overloading is reached during the time interval is given in column @@ -766,6 +967,28 @@ def get_most_critical_time_intervals( "percentage_buses_max_voltage_deviation". """ + if by not in ("power_flow", "residual_load"): + raise ValueError( + f"get_most_critical_time_intervals: 'by' must be 'power_flow' or " + f"'residual_load', got {by!r}." + ) + + if by == "residual_load": + # No power flow: rank the critical single steps by residual load (via + # get_most_critical_time_steps(by="residual_load")) and wrap each into an + # interval centered on the step and snapped to the time_step_day_start + # block. Returns per-case columns time_steps_load_case / + # time_steps_feedin_case (ranked, overlaps allowed). + return _most_critical_time_intervals_residual_load( + edisgo_obj, + num_time_intervals=num_time_intervals, + percentage=percentage, + time_steps_per_time_interval=time_steps_per_time_interval, + time_step_day_start=time_step_day_start, + save_steps=save_steps, + path=path, + ) + # check frequency of time series data timeindex = edisgo_obj.timeseries.timeindex timedelta = timeindex[1] - timeindex[0] @@ -845,6 +1068,64 @@ def get_most_critical_time_intervals( return steps +def _most_critical_time_steps_residual_load( + edisgo_obj, + num_steps_loading=None, + num_steps_voltage=None, + percentage=1.0, +): + """ + Rank time steps by residual load, without running a power flow. + + Distributes the overlying-grid dispatch onto the grid components (via + :func:`~.network.overlying_grid.distribute_overlying_grid_requirements`) and + evaluates the residual load (load minus generation minus storage) over the + whole time index. The load-case steps are those with the highest residual + load, the feed-in-case steps those with the lowest (most negative). + + Parameters + ---------- + edisgo_obj : :class:`~.EDisGo` + num_steps_loading : int or None + Number of highest-residual (load-case) steps to select. If None, + ``percentage`` of all steps is used. + num_steps_voltage : int or None + Number of lowest-residual (feed-in-case) steps to select. If None, + ``percentage`` of all steps is used. + percentage : float + Fraction of all time steps to select per case when the corresponding + ``num_steps_*`` is None. Default: 1.0. + + Returns + ------- + :pandas:`pandas.DatetimeIndex` + Unique union of the selected load-case and feed-in-case time steps. + """ + from edisgo.network.overlying_grid import ( + distribute_overlying_grid_requirements, + ) + + distributed = distribute_overlying_grid_requirements(edisgo_obj) + residual = distributed.timeseries.residual_load + + n = len(residual) + if num_steps_loading is None: + num_steps_loading = int(n * percentage) + if num_steps_voltage is None: + num_steps_voltage = int(n * percentage) + num_steps_loading = min(num_steps_loading, n) + num_steps_voltage = min(num_steps_voltage, n) + + # highest residual = worst load case; lowest residual = worst feed-in case + load_case = residual.sort_values(ascending=False).index[:num_steps_loading] + feedin_case = residual.sort_values(ascending=True).index[:num_steps_voltage] + + steps = load_case.append(feedin_case) + if len(steps) == 0: + logger.warning("No critical steps detected. No network expansion required.") + return pd.DatetimeIndex(steps.unique()) + + def get_most_critical_time_steps( edisgo_obj: EDisGo, mode=None, @@ -857,6 +1138,7 @@ def get_most_critical_time_steps( use_troubleshooting_mode=True, run_initial_analyze=True, weight_by_costs=True, + by="power_flow", ) -> pd.DatetimeIndex: """ Get the time steps with the most critical overloading and voltage issues. @@ -926,14 +1208,51 @@ def get_most_critical_time_steps( If False, only the relative overloading is used. Default: True. + by : str + Criticality measure used to rank time steps. Options: + + * "power_flow" (default): run a power flow and score steps by overloading + and voltage violations (the parameters `mode`, `timesteps`, + `lv_grid_id`, `scale_timeseries`, `use_troubleshooting_mode`, + `run_initial_analyze`, `weight_by_costs` apply to this measure). + * "residual_load": no power flow — rank steps by the residual load + (load minus generation minus storage) after distributing the + overlying-grid dispatch onto the components. The highest residual + steps are the critical load cases, the lowest (most negative) the + critical feed-in cases. `num_steps_loading` / `num_steps_voltage` / + `percentage` control how many of each are selected; the power-flow + parameters are ignored. + + Default: "power_flow". Returns -------- :pandas:`pandas.DatetimeIndex` Time index with unique time steps where maximum overloading or maximum - voltage deviation is reached for at least one component respectively bus. + voltage deviation is reached for at least one component respectively bus + (``by="power_flow"``), or with the highest/lowest residual load + (``by="residual_load"``). """ + if by not in ("power_flow", "residual_load"): + raise ValueError( + f"get_most_critical_time_steps: 'by' must be 'power_flow' or " + f"'residual_load', got {by!r}." + ) + + if by == "residual_load": + # No power flow needed: rank time steps by the residual load (load minus + # generation minus storage) after distributing the overlying-grid + # dispatch onto the components. The most critical load-case steps have + # the highest residual load, the most critical feed-in-case steps the + # lowest (most negative). Returns the union of both, deduplicated. + return _most_critical_time_steps_residual_load( + edisgo_obj, + num_steps_loading=num_steps_loading, + num_steps_voltage=num_steps_voltage, + percentage=percentage, + ) + # Run power flow if run_initial_analyze: if use_troubleshooting_mode: diff --git a/edisgo/tools/tools.py b/edisgo/tools/tools.py index 708811b15..6d3ed1768 100644 --- a/edisgo/tools/tools.py +++ b/edisgo/tools/tools.py @@ -38,6 +38,89 @@ logger = logging.getLogger(__name__) +def align_series_to_timeindex(ts, timeindex, extra_step=False): + """ + Align a time series to a target time index, tolerating a year mismatch. + + Data imported for the overlying grid (from CSV or eTraGo) may be indexed + in a different year than the EDisGo time index. This helper shifts the + series' index by whole years to match ``timeindex`` (using + :class:`pandas.DateOffset`, which — unlike ``Timestamp.replace(year=...)`` + — does not raise on a Feb-29 timestamp when the target year is not a leap + year) and reindexes onto it. Missing steps become ``NaN`` rather than + raising a ``KeyError``. + + Parameters + ---------- + ts : :pandas:`pandas.Series` or \ + :pandas:`pandas.DataFrame` or None + The time series to align. Returned unchanged if ``None``, empty, or + when ``timeindex`` is empty. + timeindex : :pandas:`pandas.DatetimeIndex` + Target time index to align to. + extra_step : bool, optional + If ``True``, append one trailing step to the target index (used for + state-of-charge series that carry an end-of-period value). The step + width is taken from ``timeindex.freq``, falling back to the spacing + of the first two entries; if neither is available (single-entry + index without freq) no extra step is added. + + Returns + ------- + Same type as ``ts`` + ``ts`` reindexed onto the (optionally extended) target index. + + """ + if ts is None or ts.empty or timeindex.empty: + return ts + year_diff = timeindex[0].year - ts.index[0].year + if year_diff != 0: + ts = ts.copy() + ts.index = ts.index + pd.DateOffset(years=year_diff) + target = timeindex + if extra_step: + freq = timeindex.freq or ( + timeindex[1] - timeindex[0] if len(timeindex) > 1 else None + ) + if freq is not None: + target = timeindex.union([timeindex[-1] + freq]) + return ts.reindex(target) + + +def check_timeindex_coverage(timeindex, name, df): + """ + Raises ``ValueError`` if `df` has columns but is missing data for a time + step in `timeindex`. + + Used by :attr:`~.edisgo.EDisGo.set_time_series_manual` and the + self-provided-DataFrame options of + :class:`~.network.timeseries.TimeSeries`'s ``predefined_*`` methods to + enforce that user-provided time series actually cover the active + timeindex, instead of silently writing a partially- or non-overlapping + series. A DataFrame with no columns is exempt - nothing is being + written, so there is nothing to validate coverage for. + + Parameters + ---------- + timeindex : :pandas:`pandas.DatetimeIndex` + Time index to check coverage against. Assumed non-empty by the + caller. + name : str + Parameter name to reference in the raised error message. + df : :pandas:`pandas.DataFrame` or None + DataFrame to check. Skipped if ``None`` or has no columns. + + """ + if df is None or df.shape[1] == 0: + return + missing = timeindex.difference(df.index) + if len(missing) > 0: + raise ValueError( + f"'{name}' does not cover the current timeindex - missing time " + f"steps: {list(missing)}." + ) + + def select_worstcase_snapshots(edisgo_obj): """ Select two worst-case snapshots from time series @@ -1179,6 +1262,25 @@ def reduce_timeseries_data_to_given_timeindex( ) # Battery electric vehicle timeseries if electromobility: + # The EV flexibility bands are built in import_electromobility from the + # raw SimBEV grid (typically 15-min and in the reference year 2011), + # independently of the analysis time index. Before slicing by datetime, + # align them to the target index: first resample to its frequency + # (Electromobility.resample uses the correct per-band aggregation — + # mean for power, max for energy), then shift the year and reindex via + # align_series_to_timeindex so datetime .loc lookups below succeed. + _bands = edisgo_obj.electromobility.flexibility_bands + _band0 = next((b for b in _bands.values() if not b.empty), None) + if _band0 is not None and len(_band0.index) > 1: + band_freq = _band0.index[1] - _band0.index[0] + if band_freq != frequency: + edisgo_obj.electromobility.resample(freq=frequency) + # year-align every (now correctly-sampled) band onto the timeindex + for key, df in edisgo_obj.electromobility.flexibility_bands.items(): + if not df.empty: + edisgo_obj.electromobility.flexibility_bands[key] = ( + align_series_to_timeindex(df, timeindex) + ) if save_ev_soc_initial: # timestep EV SOC from timestep before if possible ts_before = timeindex[0] - frequency @@ -1265,6 +1367,51 @@ def reduce_timeseries_data_to_given_timeindex( ) +def split_into_contiguous_runs(df, freq_orig): + """ + Splits a DataFrame with a (possibly gapped) `DatetimeIndex` into its + maximal contiguous runs. + + A run boundary is any gap between consecutive index entries strictly + larger than `freq_orig`. Used by :func:`resample` so that resampling a + gapped timeindex (e.g. as produced by ``select_timesteps`` in auto mode, + which deliberately keeps two disjoint intervals separate) resamples each + contiguous block independently, rather than silently bridging the gap + with resample artifacts (pandas' own `.resample()` always buckets + contiguously across whatever span the data's index covers, filling any + gap with forward-filled/averaged data rather than leaving it empty). + + Parameters + ---------- + df : :pandas:`pandas.DataFrame` + DataFrame with a :pandas:`pandas.DatetimeIndex`. + Assumed non-empty and sorted. + freq_orig : :pandas:`pandas.Timedelta` + Frequency of the original time series data. Any gap larger than this + is treated as a run boundary. + + Returns + ------- + list(:pandas:`pandas.DataFrame`) + The contiguous runs, in order. A continuous `df` returns a + single-element list containing `df` itself unchanged. + + """ + if len(df.index) < 2: + return [df] + gaps = df.index.to_series().diff().iloc[1:] + run_boundaries = np.flatnonzero((gaps > freq_orig).to_numpy()) + 1 + if len(run_boundaries) == 0: + return [df] + return [ + df.iloc[start:end] + for start, end in zip( + [0, *run_boundaries.tolist()], + [*run_boundaries.tolist(), len(df.index)], + ) + ] + + def resample( object, freq_orig, @@ -1294,58 +1441,46 @@ def resample( List of attributes to resample. Per default, all attributes specified in respective object's `_attributes` are resampled. + Notes + ----- + A gapped index (e.g. as produced by ``select_timesteps`` in auto mode) is + resampled per contiguous run (see :func:`split_into_contiguous_runs`), so + a gap is preserved rather than silently bridged with resample artifacts. + """ if attr_to_resample is None: attr_to_resample = object._attributes + freq_orig = pd.Timedelta(freq_orig) + freq = pd.Timedelta(freq) if not isinstance(freq, pd.Timedelta) else freq + up_sampling = freq < freq_orig + + if method not in ("interpolate", "ffill", "bfill"): + raise NotImplementedError(f"Resampling method {method} is not implemented.") - # add time step at the end of the time series in case of up-sampling so that - # last time interval in the original time series is still included - df_dict = {} for attr in attr_to_resample: - if not getattr(object, attr).empty: - df_dict[attr] = getattr(object, attr) - if pd.Timedelta(freq) < freq_orig: # up-sampling - new_dates = pd.DatetimeIndex([df_dict[attr].index[-1] + freq_orig]) - else: # down-sampling - new_dates = pd.DatetimeIndex([df_dict[attr].index[-1]]) - df_dict[attr] = ( - df_dict[attr] - .reindex(df_dict[attr].index.union(new_dates).unique().sort_values()) - .ffill() - ) + df = getattr(object, attr) + if df.empty: + continue + + resampled_runs = [] + for run in split_into_contiguous_runs(df, freq_orig): + # add time step at the end of the run in case of up-sampling so + # that the last time interval in the run is still included + if up_sampling: + new_dates = pd.DatetimeIndex([run.index[-1] + freq_orig]) + else: + new_dates = pd.DatetimeIndex([run.index[-1]]) + run = run.reindex(run.index.union(new_dates).unique().sort_values()).ffill() - # resample time series - if pd.Timedelta(freq) < freq_orig: # up-sampling - if method == "interpolate": - for attr in df_dict.keys(): - setattr( - object, - attr, - df_dict[attr].resample(freq, closed="left").interpolate().iloc[:-1], - ) - elif method == "ffill": - for attr in df_dict.keys(): - setattr( - object, - attr, - df_dict[attr].resample(freq, closed="left").ffill().iloc[:-1], - ) - elif method == "bfill": - for attr in df_dict.keys(): - setattr( - object, - attr, - df_dict[attr].resample(freq, closed="left").bfill().iloc[:-1], - ) - else: - raise NotImplementedError(f"Resampling method {method} is not implemented.") - else: # down-sampling - for attr in df_dict.keys(): - setattr( - object, - attr, - df_dict[attr].resample(freq).mean(), - ) + if up_sampling: + resampled = getattr(run.resample(freq, closed="left"), method)().iloc[ + :-1 + ] + else: + resampled = run.resample(freq).mean() + resampled_runs.append(resampled) + + setattr(object, attr, pd.concat(resampled_runs)) def reduce_memory_usage(df: pd.DataFrame, show_reduction: bool = False) -> pd.DataFrame: @@ -1374,7 +1509,7 @@ def reduce_memory_usage(df: pd.DataFrame, show_reduction: bool = False) -> pd.Da for col in df.columns: col_type = df[col].dtype - if col_type != object and str(col_type) != "category": + if not pd.api.types.is_object_dtype(col_type) and str(col_type) != "category": c_min = df[col].min() c_max = df[col].max() diff --git a/run_example_05.py b/run_example_05.py new file mode 100644 index 000000000..552082d3d --- /dev/null +++ b/run_example_05.py @@ -0,0 +1,39 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Runner für uc5_select_timesteps.yaml — einfach ``python run_example_05.py``.""" + +import logging + +from edisgo.run.runner import run_edisgo + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(name)s %(levelname)s: %(message)s", +) + +# edisgo = run_edisgo("/storage/JoDa/ego/edisgo_run_edisgo/eDisGo/edisgo/run/presets/uc4_example_MS.yaml") # noqa: E501 +edisgo = run_edisgo( + { + "extends": "uc5_select_timesteps.yaml", + # "grid": {"ding0_path": "/home/gurobi/.ding0/run_hetzner_59763_2023_04_06/ding0_grids/32355"} # noqa: E501 + "grid": { + "ding0_path": "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" # noqa: E501 + }, + # OG path must be the leaf dir for THIS grid (like ding0_path), not the parent. + "overlying_grid": { + "path": "/storage/JoDa/edisgo_playground/overlying_grid_data/32377" + }, + } +) + +print("\n=== Fertig ===") +print("Ausbaukosten:\n", edisgo.results.grid_expansion_costs) +print("\nUngelöste Probleme:\n", edisgo.results.unresolved_issues) diff --git a/run_example_06.py b/run_example_06.py new file mode 100644 index 000000000..fb40baeb0 --- /dev/null +++ b/run_example_06.py @@ -0,0 +1,37 @@ +# This file is part of eDisGo (Electrical Distribution Grid Optimization), +# a Python package for analyzing flexibility options in distribution grids. +# +# Copyright (c) Reiner Lemoine Institut gGmbH +# Contributors are listed in the version control history: +# https://github.com/openego/eDisGo/ +# +# Documentation: https://edisgo.readthedocs.io/ +# +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Runner für uc6_spatial_reduction.yaml — einfach ``python run_example_05.py``.""" + +import logging + +from edisgo.run.runner import run_edisgo + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(name)s %(levelname)s: %(message)s", +) + +# edisgo = run_edisgo("/storage/JoDa/ego/edisgo_run_edisgo/eDisGo/edisgo/run/presets/uc4_example_MS.yaml") # noqa: E501 +edisgo = run_edisgo( + { + "extends": "uc6_spatial_reduction.yaml", + # "grid": {"ding0_path": "/home/gurobi/.ding0/run_hetzner_59763_2023_04_06/ding0_grids/32355"} # noqa: E501 + "grid": { + "ding0_path": "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" # noqa: E501 + }, + # OG path must be the leaf dir for THIS grid (like ding0_path), not the parent. + "overlying_grid": {"path": "/home/gurobi/.edisgo_input/overlying_grid/32377"}, + } +) + +print("\n=== Fertig ===") +print("Ausbaukosten:\n", edisgo.results.grid_expansion_costs) +print("\nUngelöste Probleme:\n", edisgo.results.unresolved_issues) diff --git a/setup.py b/setup.py index 74315be5e..a2a4422b3 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ def read(fname): # sqlalchemy leads to new errors.. should be fixed at some point "numpy ==1.26.4", "pandas >= 1.4.0, < 2.2.0", + "paramiko < 4.0", "plotly < 6.0", "pydot < 4.1.0", "pypower < 5.2.0", @@ -111,6 +112,7 @@ def read(fname): "edisgo": [ os.path.join("config", "*.cfg"), os.path.join("equipment", "*.csv"), + os.path.join("run", "presets", "*.yaml"), ] }, ) diff --git a/tests/flex_opt/test_charging_strategy.py b/tests/flex_opt/test_charging_strategy.py index fe533e4e4..b94c7af3f 100644 --- a/tests/flex_opt/test_charging_strategy.py +++ b/tests/flex_opt/test_charging_strategy.py @@ -97,6 +97,446 @@ def test_charging_strategy(self, caplog): charging_strategy(self.edisgo_obj, strategy="dumb") assert ts._loads_active_power.index.freqstr == "15T" + @pytest.mark.parametrize("strategy", ["dumb", "reduced", "residual"]) + def test_charging_strategy_trims_to_short_timeindex(self, strategy): + """ + Regression test for eDisGo#703: charging_strategy used to write the + full SimBEV-simulation-length series into loads_active_power/ + loads_reactive_power regardless of a shorter active timeindex. When + the edisgo/SimBEV frequencies already match (no internal resample + round-trip), the written series must be trimmed to exactly + edisgo.timeseries.timeindex - no extra rows, no missing rows. + """ + edisgo = EDisGo(ding0_grid=self.ding0_path) + # 15-min frequency matches the SimBEV fixture's stepsize (see + # metadata_simbev_run.json), so no internal resample round-trip is + # triggered - one day instead of the fixture's full simulated week. + short_timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") + edisgo.set_timeindex(short_timeindex) + edisgo.import_electromobility( + data_source="directory", + charging_processes_dir=self.simbev_path, + potential_charging_points_dir=self.tracbev_path, + ) + + charging_strategy(edisgo, strategy=strategy) + + pd.testing.assert_index_equal( + edisgo.timeseries._loads_active_power.index, short_timeindex + ) + pd.testing.assert_index_equal( + edisgo.timeseries._loads_reactive_power.index, short_timeindex + ) + assert not edisgo.timeseries.loads_active_power.isna().any().any() + + def test_charging_strategy_trims_to_gapped_timeindex(self): + """ + Regression test for eDisGo#703: a gapped timeindex (as produced by + select_timesteps in auto mode) must survive charging_strategy + unchanged when no internal frequency resample round-trip is + triggered - the written series must match the gapped index exactly, + not a contiguous range spanning it. + """ + edisgo = EDisGo(ding0_grid=self.ding0_path) + gapped_timeindex = pd.date_range("1/1/2011", periods=24, freq="15min").union( + pd.date_range("1/6/2011 18:00", periods=24, freq="15min") + ) + edisgo.set_timeindex(gapped_timeindex) + edisgo.import_electromobility( + data_source="directory", + charging_processes_dir=self.simbev_path, + potential_charging_points_dir=self.tracbev_path, + ) + + charging_strategy(edisgo, strategy="dumb") + + pd.testing.assert_index_equal( + edisgo.timeseries._loads_active_power.index, gapped_timeindex + ) + + def _setup_edisgo_with_single_synthetic_event( + self, timeindex, park_start_timesteps, park_end_timesteps, chargingdemand_kWh + ): + """ + Helper for the ADR 0001 regression tests below: imports the real + SimBEV/TracBEV fixture (so charging park integration/topology wiring + is realistic), then overwrites charging_processes_df with a single, + fully controlled synthetic event on one of the fixture's own + integrated charging parks, reusing that park's own use_case/capacity + so `harmonize_charging_processes_df` sees realistic values. + """ + edisgo = EDisGo(ding0_grid=self.ding0_path) + edisgo.set_timeindex(timeindex) + edisgo.import_electromobility( + data_source="directory", + charging_processes_dir=self.simbev_path, + potential_charging_points_dir=self.tracbev_path, + ) + integrated = edisgo.electromobility.integrated_charging_parks_df + park_id = integrated.index[0] + template = edisgo.electromobility.charging_processes_df[ + edisgo.electromobility.charging_processes_df.charging_park_id == park_id + ].iloc[0] + + park_time_timesteps = park_end_timesteps - park_start_timesteps + 1 + edisgo.electromobility.charging_processes_df = pd.DataFrame( + { + "ags": [template.ags], + "car_id": [0], + "destination": [template.destination], + # avoid public/hpc (always dumb-charged even under + # "residual") so these events exercise the flex-ranking path + "use_case": ["work"], + "nominal_charging_capacity_kW": [template.nominal_charging_capacity_kW], + "grid_charging_capacity_kW": [template.grid_charging_capacity_kW], + "chargingdemand_kWh": [chargingdemand_kWh], + "park_time_timesteps": [park_time_timesteps], + "park_start_timesteps": [park_start_timesteps], + "park_end_timesteps": [park_end_timesteps], + "charging_park_id": [park_id], + "charging_point_id": [template.charging_point_id], + } + ) + return edisgo, park_id + + def test_residual_drops_fully_out_of_window_events(self): + """ + Regression test for ADR 0001: a charging event whose parking window + has zero overlap with the active timeindex must contribute nothing - + not be tiled/fabricated against repeated residual_load data. + """ + timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") # 1 day + # park window entirely on day 3, well beyond the 1-day active window + edisgo, park_id = self._setup_edisgo_with_single_synthetic_event( + timeindex, + park_start_timesteps=300, + park_end_timesteps=320, + chargingdemand_kWh=10.0, + ) + + charging_strategy(edisgo, strategy="residual") + + edisgo_id = edisgo.electromobility.integrated_charging_parks_df.at[ + park_id, "edisgo_id" + ] + written = edisgo.timeseries.loads_active_power[edisgo_id] + assert (written == 0).all() + + def test_residual_prorates_boundary_straddling_event(self): + """ + Regression test for ADR 0001: a charging event whose parking window + straddles the active timeindex boundary must have its charging + demand prorated by the in-window fraction of its parking time, not + fully charged (which would require tiled/fabricated residual_load + data beyond the active timeindex) and not dropped. + """ + timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") # steps 0-95 + # park window [90, 149] (60 steps), only steps 90-95 (6 steps) are + # in-window -> in_window_fraction = 6/60 = 0.1 + edisgo, park_id = self._setup_edisgo_with_single_synthetic_event( + timeindex, + park_start_timesteps=90, + park_end_timesteps=149, + chargingdemand_kWh=12.0, + ) + edisgo_id = edisgo.electromobility.integrated_charging_parks_df.at[ + park_id, "edisgo_id" + ] + + charging_strategy(edisgo, strategy="residual") + + written = edisgo.timeseries.loads_active_power[edisgo_id] + + # only in-window steps (90-95) may carry any charging + assert (written.iloc[:90] == 0).all() + assert written.iloc[90:].sum() > 0 + + # compare against the same event fully inside a timeindex covering + # its whole parking window - the straddling case must deliver + # strictly less energy than the fully-observable case, since only + # 1/10th of its parking time is actually in-window here + full_timeindex = pd.date_range("1/1/2011", periods=150, freq="15min") + edisgo_full, park_id_full = self._setup_edisgo_with_single_synthetic_event( + full_timeindex, + park_start_timesteps=90, + park_end_timesteps=149, + chargingdemand_kWh=12.0, + ) + charging_strategy(edisgo_full, strategy="residual") + edisgo_id_full = edisgo_full.electromobility.integrated_charging_parks_df.at[ + park_id_full, "edisgo_id" + ] + full_energy = edisgo_full.timeseries.loads_active_power[edisgo_id_full].sum() + + straddling_energy = written.sum() + assert 0 < straddling_energy < full_energy + + def test_residual_fully_inside_event_unaffected(self): + """ + Regression test for ADR 0001: an event whose parking window is + entirely inside the active timeindex must be scheduled exactly the + same regardless of how much longer the active timeindex extends + beyond the event's own window - proration must not affect + fully-observable events at all. + """ + edisgo_short, park_id_short = self._setup_edisgo_with_single_synthetic_event( + pd.date_range("1/1/2011", periods=60, freq="15min"), + park_start_timesteps=10, + park_end_timesteps=50, + chargingdemand_kWh=8.0, + ) + edisgo_long, park_id_long = self._setup_edisgo_with_single_synthetic_event( + pd.date_range("1/1/2011", periods=200, freq="15min"), + park_start_timesteps=10, + park_end_timesteps=50, + chargingdemand_kWh=8.0, + ) + + charging_strategy(edisgo_short, strategy="residual") + charging_strategy(edisgo_long, strategy="residual") + + edisgo_id_short = edisgo_short.electromobility.integrated_charging_parks_df.at[ + park_id_short, "edisgo_id" + ] + edisgo_id_long = edisgo_long.electromobility.integrated_charging_parks_df.at[ + park_id_long, "edisgo_id" + ] + energy_short = edisgo_short.timeseries.loads_active_power[edisgo_id_short].sum() + energy_long = edisgo_long.timeseries.loads_active_power[edisgo_id_long].sum() + + assert energy_short > 0 + assert energy_short == pytest.approx(energy_long) + + def test_residual_no_tiling_across_gapped_timeindex(self): + """ + Regression test for ADR 0001: with a gapped active timeindex, an + event that overlaps both disjoint runs must only ever be scheduled + into steps actually present in the active timeindex - never into the + gap, and never against fabricated/tiled residual_load data. + """ + # two disjoint 15-min runs: steps 0-23 and steps 100-123 (gap of 76 + # steps in between, well beyond any real residual_load coverage) + run_1 = pd.date_range("1/1/2011", periods=24, freq="15min") + run_2 = pd.date_range("1/1/2011", periods=24, freq="15min") + pd.Timedelta( + minutes=15 * 100 + ) + gapped_timeindex = run_1.union(run_2) + + edisgo, park_id = self._setup_edisgo_with_single_synthetic_event( + gapped_timeindex, + park_start_timesteps=10, + park_end_timesteps=110, + chargingdemand_kWh=6.0, + ) + + charging_strategy(edisgo, strategy="residual") + + edisgo_id = edisgo.electromobility.integrated_charging_parks_df.at[ + park_id, "edisgo_id" + ] + written = edisgo.timeseries.loads_active_power[edisgo_id] + + # written series must exactly match the gapped index - nothing + # fabricated to bridge the gap + pd.testing.assert_index_equal(written.index, gapped_timeindex) + assert written.sum() > 0 + + def test_residual_dumb_subbucket_respects_internal_gap(self): + """ + Regression test for ADR 0001: a "dumb-charged" event within the + residual strategy (use_case in {public, hpc} or flex_time == 0) + whose deterministic charging interval spans a gap in the active + timeindex must only ever write to in-window positions - never a + blind contiguous slice bridging the gap. + """ + run_1 = pd.date_range("1/1/2011", periods=10, freq="15min") + run_2 = pd.date_range("1/1/2011", periods=10, freq="15min") + pd.Timedelta( + minutes=15 * 20 + ) + gapped_timeindex = run_1.union(run_2) + + edisgo = EDisGo(ding0_grid=self.ding0_path) + edisgo.set_timeindex(gapped_timeindex) + edisgo.import_electromobility( + data_source="directory", + charging_processes_dir=self.simbev_path, + potential_charging_points_dir=self.tracbev_path, + ) + integrated = edisgo.electromobility.integrated_charging_parks_df + park_id = integrated.index[0] + template = edisgo.electromobility.charging_processes_df[ + edisgo.electromobility.charging_processes_df.charging_park_id == park_id + ].iloc[0] + + # public use_case -> always dumb-charged even under "residual"; + # park window [5, 24] straddles the gap (steps 10-19) + edisgo.electromobility.charging_processes_df = pd.DataFrame( + { + "ags": [template.ags], + "car_id": [0], + "destination": [template.destination], + "use_case": ["public"], + "nominal_charging_capacity_kW": [template.nominal_charging_capacity_kW], + "grid_charging_capacity_kW": [template.grid_charging_capacity_kW], + "chargingdemand_kWh": [2.0], + "park_time_timesteps": [20], + "park_start_timesteps": [5], + "park_end_timesteps": [24], + "charging_park_id": [park_id], + "charging_point_id": [template.charging_point_id], + } + ) + + charging_strategy(edisgo, strategy="residual") + + edisgo_id = integrated.at[park_id, "edisgo_id"] + written = edisgo.timeseries.loads_active_power[edisgo_id] + + pd.testing.assert_index_equal(written.index, gapped_timeindex) + # no NaNs, no crash from writing into a position that doesn't exist + assert not written.isna().any() + + @pytest.mark.parametrize("strategy", ["dumb", "reduced"]) + def test_dumb_reduced_clip_placement_to_active_timeindex(self, strategy): + """ + Regression test for ADR 0002: dumb/reduced must clip their + deterministic charging placement slice to the active timeindex + instead of building the full-SimBEV-length series and relying on a + later crop - a charging interval that extends beyond the active + timeindex must only ever be written for its in-window positions, + at unchanged (unscaled) power. + """ + # active timeindex ends at step 92 (93 steps: 0-92) + timeindex = pd.date_range("1/1/2011", periods=93, freq="15min") + edisgo, park_id = self._setup_edisgo_with_single_synthetic_event( + timeindex, + park_start_timesteps=90, + park_end_timesteps=149, + chargingdemand_kWh=12.0, + ) + edisgo_id = edisgo.electromobility.integrated_charging_parks_df.at[ + park_id, "edisgo_id" + ] + + charging_strategy(edisgo, strategy=strategy) + + written = edisgo.timeseries.loads_active_power[edisgo_id] + nonzero = written[written != 0] + + # written series must match the active timeindex exactly - no extra + # rows for steps beyond it (nothing fabricated/built past the window) + pd.testing.assert_index_equal(written.index, timeindex) + # only in-window steps (<= step 92, i.e. before 1/1/2011 23:15) may + # ever carry a nonzero value + assert (nonzero.index <= timeindex[-1]).all() + assert len(nonzero) > 0 + + # compare against the same event given a timeindex long enough to + # cover its whole charging interval - the clipped case must report + # strictly less energy, since some of its charging steps fall + # outside the shorter active timeindex + long_timeindex = pd.date_range("1/1/2011", periods=150, freq="15min") + edisgo_long, park_id_long = self._setup_edisgo_with_single_synthetic_event( + long_timeindex, + park_start_timesteps=90, + park_end_timesteps=149, + chargingdemand_kWh=12.0, + ) + edisgo_id_long = edisgo_long.electromobility.integrated_charging_parks_df.at[ + park_id_long, "edisgo_id" + ] + charging_strategy(edisgo_long, strategy=strategy) + full_energy = edisgo_long.timeseries.loads_active_power[edisgo_id_long].sum() + + assert 0 < written.sum() < full_energy + + # power at each in-window step must be unchanged (no proration of + # the rate itself) - every nonzero value equals the same per-step + # power the long-timeindex (unclipped) run reports + long_nonzero_values = edisgo_long.timeseries.loads_active_power[edisgo_id_long] + long_nonzero_values = long_nonzero_values[long_nonzero_values != 0] + assert nonzero.iloc[0] == pytest.approx(long_nonzero_values.iloc[0]) + + @pytest.mark.parametrize("strategy", ["dumb", "reduced"]) + def test_dumb_reduced_fully_out_of_window_event_contributes_nothing(self, strategy): + """ + Regression test for ADR 0002: an event whose deterministic charging + interval has zero overlap with the active timeindex must contribute + nothing. + """ + timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") # 1 day + edisgo, park_id = self._setup_edisgo_with_single_synthetic_event( + timeindex, + park_start_timesteps=300, + park_end_timesteps=320, + chargingdemand_kWh=10.0, + ) + edisgo_id = edisgo.electromobility.integrated_charging_parks_df.at[ + park_id, "edisgo_id" + ] + + charging_strategy(edisgo, strategy=strategy) + + written = edisgo.timeseries.loads_active_power[edisgo_id] + assert (written == 0).all() + + @pytest.mark.parametrize("strategy", ["dumb", "reduced"]) + def test_dumb_reduced_respect_internal_gap(self, strategy): + """ + Regression test for ADR 0002: if an event's deterministic charging + interval itself spans a gap in a non-contiguous active timeindex, + every in-window sub-slice of that interval must be written + independently, at full unscaled power - never a blind contiguous + write bridging the gap. + """ + run_1 = pd.date_range("1/1/2011", periods=10, freq="15min") + run_2 = pd.date_range("1/1/2011", periods=10, freq="15min") + pd.Timedelta( + minutes=15 * 20 + ) + gapped_timeindex = run_1.union(run_2) + + edisgo = EDisGo(ding0_grid=self.ding0_path) + edisgo.set_timeindex(gapped_timeindex) + edisgo.import_electromobility( + data_source="directory", + charging_processes_dir=self.simbev_path, + potential_charging_points_dir=self.tracbev_path, + ) + integrated = edisgo.electromobility.integrated_charging_parks_df + park_id = integrated.index[0] + template = edisgo.electromobility.charging_processes_df[ + edisgo.electromobility.charging_processes_df.charging_park_id == park_id + ].iloc[0] + + # work use_case, small enough demand that minimum_charging_time (or + # reduced_charging_time) spans steps 5-24, straddling the gap + # (steps 10-19, absent from the active timeindex) + edisgo.electromobility.charging_processes_df = pd.DataFrame( + { + "ags": [template.ags], + "car_id": [0], + "destination": [template.destination], + "use_case": ["work"], + "nominal_charging_capacity_kW": [template.nominal_charging_capacity_kW], + "grid_charging_capacity_kW": [template.grid_charging_capacity_kW], + "chargingdemand_kWh": [2.0], + "park_time_timesteps": [20], + "park_start_timesteps": [5], + "park_end_timesteps": [24], + "charging_park_id": [park_id], + "charging_point_id": [template.charging_point_id], + } + ) + + charging_strategy(edisgo, strategy=strategy) + + edisgo_id = integrated.at[park_id, "edisgo_id"] + written = edisgo.timeseries.loads_active_power[edisgo_id] + + pd.testing.assert_index_equal(written.index, gapped_timeindex) + # no NaNs, no crash from writing into a position that doesn't exist + assert not written.isna().any() + def test_charging_strategy_with_subset_of_parks(self): """ Charging strategies can be applied to different subsets of charging parks @@ -128,7 +568,6 @@ def test_charging_strategy_with_subset_of_parks(self): # store baseline time series for both parks loads_before = ts._loads_active_power.copy() - ts_a_before = loads_before[edisgo_id_a].copy() ts_b_before = loads_before[edisgo_id_b].copy() # 1) apply a strategy only to park A @@ -152,7 +591,6 @@ def test_charging_strategy_with_subset_of_parks(self): loads_after_second = ts._loads_active_power ts_a_after_second = loads_after_second[edisgo_id_a].copy() - ts_b_after_second = loads_after_second[edisgo_id_b].copy() # park A must not be changed by the second call that targets only park B pd.testing.assert_series_equal( diff --git a/tests/flex_opt/test_heat_pump_operation.py b/tests/flex_opt/test_heat_pump_operation.py index f5b3cad9c..c3fa866f1 100644 --- a/tests/flex_opt/test_heat_pump_operation.py +++ b/tests/flex_opt/test_heat_pump_operation.py @@ -70,3 +70,32 @@ def test_operating_strategy(self): msg = "Heat pump operating strategy dummy is not a valid option." with pytest.raises(ValueError, match=msg): operating_strategy(self.edisgo, strategy="dummy") + + def test_operating_strategy_trims_to_short_timeindex(self): + """ + Regression test for eDisGo#703: operating_strategy used to write + loads_active_power over the full span of heat_demand_df/cop_df + regardless of the active edisgo.timeseries.timeindex. When those are + wider or shifted relative to the active timeindex (e.g. because the + timeindex changed after import_heat_pumps ran), the written series + must be trimmed to exactly edisgo.timeseries.timeindex. + """ + timeindex = pd.date_range("1/1/2011 12:00", periods=2, freq="H") + wide_timeindex = pd.date_range("1/1/2011", periods=24, freq="H") + + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path, timeindex=timeindex) + edisgo.heat_pump.cop_df = pd.DataFrame( + data={"hp1": [5.0] * 24, "hp2": [7.0] * 24}, index=wide_timeindex + ) + edisgo.heat_pump.heat_demand_df = pd.DataFrame( + data={"hp1": [1.0] * 24, "hp2": [3.0] * 24}, index=wide_timeindex + ) + + operating_strategy(edisgo) + + pd.testing.assert_index_equal( + edisgo.timeseries._loads_active_power.index, timeindex + ) + pd.testing.assert_index_equal( + edisgo.timeseries._loads_reactive_power.index, timeindex + ) diff --git a/tests/flex_opt/test_reinforce_measures.py b/tests/flex_opt/test_reinforce_measures.py index 5ea720064..be0aede23 100644 --- a/tests/flex_opt/test_reinforce_measures.py +++ b/tests/flex_opt/test_reinforce_measures.py @@ -300,12 +300,19 @@ def test_reinforce_lines_voltage_issues(self): # LV: # * check where node_2_3 is in_building => problem at - # Bus_BranchTee_LVGrid_5_2, leads to reinforcement of line - # Line_50000003 (which is first line in feeder and not a - # standard line) + # Bus_BranchTee_LVGrid_5_2, node_2_3 is moved back to + # Bus_BranchTee_LVGrid_5_1, which is the feeder representative, so no + # line can be disconnected. All lines on the path from the station to + # the critical node are reinforced, i.e. Line_50000003 (first line in + # feeder, not a standard line) and Line_50000002 (line to the critical + # node, not a standard line). Line_50000003 is only 0.56 m long, while + # Line_50000002 is 30 m of NAYY 4x1x35 and therefore causes most of + # the voltage deviation - reinforcing the first line segment only + # would not resolve the voltage issue. # * check where node_2_3 is not in_building => problem at # Bus_BranchTee_LVGrid_5_5, leads to reinforcement of line - # Line_50000009 (which is first line in feeder and a standard line) + # Line_50000009 (which is the only line in the feeder and a standard + # line) crit_nodes = pd.DataFrame( { @@ -321,8 +328,9 @@ def test_reinforce_lines_voltage_issues(self): ) reinforced_lines = lines_changes.keys() - assert len(lines_changes) == 2 + assert len(lines_changes) == 3 assert "Line_50000003" in reinforced_lines + assert "Line_50000002" in reinforced_lines assert "Line_50000009" in reinforced_lines # check that LV station is one of the buses assert ( @@ -364,6 +372,18 @@ def test_reinforce_lines_voltage_issues(self): line.s_nom, np.sqrt(3) * grid.nominal_voltage * std_line.I_max_th ) assert line.num_parallel == 1 + # second line segment on the path to the critical node is reinforced as + # well + line = self.edisgo.topology.lines_df.loc["Line_50000002"] + assert line.type_info == std_line.name + assert np.isclose(line.r, std_line.R_per_km * line.length) + assert np.isclose( + line.x, std_line.L_per_km * line.length * 2 * np.pi * 50 / 1e3 + ) + assert np.isclose( + line.s_nom, np.sqrt(3) * grid.nominal_voltage * std_line.I_max_th + ) + assert line.num_parallel == 1 line = self.edisgo.topology.lines_df.loc["Line_50000009"] assert line.type_info == std_line.name assert line.num_parallel == 2 diff --git a/tests/io/test_generators_import.py b/tests/io/test_generators_import.py index 47765f703..6e06d2f6e 100644 --- a/tests/io/test_generators_import.py +++ b/tests/io/test_generators_import.py @@ -305,6 +305,66 @@ def test__integrate_pv_rooftop(self, caplog): "matched to an existing PV rooftop plant." in caplog.text ) + def test__integrate_pv_rooftop_two_buildings_on_one_bus(self): + """ + A bus carrying conventional loads of two different buildings must not + break the PV rooftop import. + + This used to raise "cannot reindex on an axis with duplicate labels" + (openego/eGo#213, seen on MV grids 33084 and 33695): the function built + a bus -> building ID map from the conventional loads, and that map was + deduplicated on "building_id" while being indexed by "bus". Two + buildings on one bus therefore left a duplicated bus label, the join + multiplied the generator rows and ``DataFrame.update`` rejected the + non-unique index. + + That map has since been removed entirely -- it fed a building-ID based + matching that was replaced by source-ID matching in d33e8515 -- so the + function no longer reads ``loads_df`` at all and the crash is + structurally impossible. This test is kept as a guard against + reintroducing a per-bus load lookup here. + + Note that the shipped test grid cannot produce the situation on its + own: it has 70 buses with more than one conventional load, but all of + them share a single building ID. The second building is therefore + added explicitly. + """ + edisgo = EDisGo( + ding0_grid=pytest.ding0_test_network_3_path, legacy_ding0_grids=False + ) + loads_df = edisgo.topology.loads_df + gens_df = edisgo.topology.generators_df + # a bus that carries both a PV rooftop generator and a conventional load + pv_buses = set(gens_df[gens_df.subtype == "pv_rooftop"].bus) + conv = loads_df[loads_df.type == "conventional_load"] + bus = conv[conv.bus.isin(pv_buses)].bus.iloc[0] + existing = conv[conv.bus == bus].iloc[0] + + # second building on the same bus + second = existing.copy() + second["building_id"] = int(existing.building_id) + 1_000_000 + edisgo.topology.loads_df.loc["Load_second_building_same_bus"] = second + + pv_df = pd.DataFrame( + data={ + "p_nom": [0.005], + "weather_cell_id": [11051], + "building_id": [430903], + "generator_id": [1], + "type": ["solar"], + "subtype": ["pv_rooftop"], + "source_id": ["SEE970362202254"], + }, + index=[1], + ) + + # used to raise ValueError: cannot reindex on an axis with duplicate labels + generators_import._integrate_pv_rooftop(edisgo, pv_df) + + # no generator was duplicated by the join (generators whose source_id + # is absent from the scenario are legitimately decommissioned here) + assert not edisgo.topology.generators_df.index.has_duplicates + def test__integrate_new_pv_rooftop_to_buildings(self, caplog): pv_df = pd.DataFrame( data={ diff --git a/tests/io/test_powermodels_io.py b/tests/io/test_powermodels_io.py index b3bfab036..556e7bdd0 100644 --- a/tests/io/test_powermodels_io.py +++ b/tests/io/test_powermodels_io.py @@ -304,6 +304,103 @@ def test_to_powermodels(self): ) ) + def test_to_powermodels_flexibility_bands_wider_than_timeindex(self): + """ + Regression test for eDisGo#718: _build_electromobility and + _build_component_timeseries used to read flexibility_bands + positionally (.iloc[0], .values.tolist() on the whole column) + instead of aligning to edisgo.timeseries.timeindex first. When + flexibility_bands spans more rows than the active timeindex (e.g. + stale data from before a later select_timesteps step), this used to + silently take the wrong/misaligned static p_max/e_min/e_max and + write a longer time series than pm["time_series"]["num_steps"] - + both must now be exactly scoped to the active timeindex. + """ + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) + edisgo.set_time_series_worst_case_analysis() + timeindex = edisgo.timeseries.timeindex + + edisgo.add_component( + comp_type="load", + type="charging_point", + ts_active_power=pd.Series(index=timeindex, data=[0.5] * 4), + ts_reactive_power="default", + bus=edisgo.topology.buses_df.index[32], + p_set=3, + ) + + # flexibility_bands spans 8 steps, twice the active 4-step timeindex, + # with distinctive values so misalignment is obvious + wide_index = pd.date_range(timeindex[0], periods=8, freq=timeindex.freq) + edisgo.electromobility.flexibility_bands = { + "lower_energy": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [0.0] * 8}, index=wide_index + ), + "upper_energy": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [9.0, 1, 2, 3, 4, 5, 6, 7]}, + index=wide_index, + ), + "upper_power": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [9.0, 1, 2, 3, 4, 5, 6, 7]}, + index=wide_index, + ), + } + + pm, _ = powermodels_io.to_powermodels( + edisgo, flexible_cps=["Charging_Point_LVGrid_6_1"] + ) + + num_steps = pm["time_series"]["num_steps"] + assert num_steps == len(timeindex) + for key in ("p_max", "e_min", "e_max"): + assert len(pm["time_series"]["electromobility"]["1"][key]) == num_steps + assert pm["time_series"]["electromobility"]["1"]["p_max"] == [ + 9.0, + 1.0, + 2.0, + 3.0, + ] + assert pm["electromobility"]["1"]["p_max"] == pytest.approx(9.0) + + def test_to_powermodels_flexibility_bands_wrong_calendar_raises(self): + """ + Regression test for eDisGo#718: when flexibility_bands doesn't cover + edisgo.timeseries.timeindex at all (genuine staleness, not just a + wider/narrower matching-calendar range), to_powermodels must raise a + clear KeyError rather than silently building wrong OPF input from + mismatched rows. + """ + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) + edisgo.set_time_series_worst_case_analysis() + timeindex = edisgo.timeseries.timeindex + + edisgo.add_component( + comp_type="load", + type="charging_point", + ts_active_power=pd.Series(index=timeindex, data=[0.5] * 4), + ts_reactive_power="default", + bus=edisgo.topology.buses_df.index[32], + p_set=3, + ) + + wrong_index = pd.date_range("2035-01-01", periods=4, freq=timeindex.freq) + edisgo.electromobility.flexibility_bands = { + "lower_energy": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [0.0] * 4}, index=wrong_index + ), + "upper_energy": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [1.0] * 4}, index=wrong_index + ), + "upper_power": pd.DataFrame( + {"Charging_Point_LVGrid_6_1": [1.0] * 4}, index=wrong_index + ), + } + + with pytest.raises(KeyError): + powermodels_io.to_powermodels( + edisgo, flexible_cps=["Charging_Point_LVGrid_6_1"] + ) + def test__get_pf(self): self.edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) self.edisgo.set_time_series_worst_case_analysis() diff --git a/tests/network/test_dsm.py b/tests/network/test_dsm.py index 3e2ea7201..b9981b26c 100644 --- a/tests/network/test_dsm.py +++ b/tests/network/test_dsm.py @@ -66,6 +66,49 @@ def test_reduce_memory(self): self.dsm.e_max = pd.DataFrame() self.dsm.reduce_memory() + def test_resample(self): + """ + Regression test for eDisGo#703: DSM had no resample() method, so + EDisGo.resample_timeseries silently left DSM data at its original + frequency while every sibling container (TimeSeries, Electromobility, + HeatPump, OverlyingGrid) was resampled. + """ + # test up-sampling with default parameters + self.dsm.resample() + assert len(self.dsm.p_max) == 8 + assert (self.dsm.p_max.iloc[0:4, 0] == 5).all() + assert (self.dsm.p_max.iloc[4:8, 1] == 8).all() + assert len(self.dsm.p_min) == 8 + assert len(self.dsm.e_max) == 8 + assert len(self.dsm.e_min) == 8 + + # test down-sampling + self.dsm.resample(freq="1H") + assert len(self.dsm.p_max) == 2 + assert len(self.dsm.p_min) == 2 + assert len(self.dsm.e_max) == 2 + assert len(self.dsm.e_min) == 2 + + # test with empty dataframes - must not raise + self.dsm.e_max = pd.DataFrame() + self.dsm.resample() + + def test_resample_preserves_gapped_index(self): + """ + Regression test: resampling a gapped index must not bridge the gap + with resample artifacts. + """ + gapped_index = pd.date_range("2035-01-08", periods=24, freq="h").union( + pd.date_range("2035-06-10", periods=24, freq="h") + ) + dsm = DSM() + dsm.p_max = pd.DataFrame({"load_1": [5.0] * 48}, index=gapped_index) + + dsm.resample(freq="15min") + gap = dsm.p_max.index.to_series().diff().max() + assert gap > pd.Timedelta("15min") + assert len(dsm.p_max) == 192 # 2 runs * 24h * 4 (15min steps/h) + def test_to_csv(self): # test with default values save_dir = os.path.join(os.getcwd(), "dsm_csv") diff --git a/tests/network/test_electromobility.py b/tests/network/test_electromobility.py index 2afeeb660..db78e79f5 100644 --- a/tests/network/test_electromobility.py +++ b/tests/network/test_electromobility.py @@ -8,7 +8,7 @@ import pandas as pd import pytest -from pandas.testing import assert_frame_equal +from pandas.testing import assert_frame_equal, assert_index_equal from edisgo.edisgo import EDisGo from edisgo.io import electromobility_import @@ -185,6 +185,307 @@ def test_get_flexibility_bands(self): ].index assert (flex_bands_index[1] - flex_bands_index[0]) == pd.Timedelta("1H") + def test_get_flexibility_bands_scopes_to_mismatched_timeindex(self): + """ + Regression test for eDisGo#703: get_flexibility_bands used to build + bands spanning SimBEV's own native calendar/range only, with no + alignment to edisgo.timeseries.timeindex - indexing the bands by a + timeindex in a different year (SimBEV's start_date here is 2011) + and/or a shorter window than SimBEV's simulated range raised + KeyError. The bands must now be year-aligned and trimmed to exactly + that timeindex. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + assert edisgo_obj.electromobility.simbev_config_df.start_date.values[ + 0 + ] == np.datetime64("2011-01-01") + short_timeindex = pd.date_range("2035-01-15", periods=24, freq="h") + edisgo_obj.set_timeindex(short_timeindex) + + bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, ["work", "public"] + ) + + for key in ("upper_power", "lower_energy", "upper_energy"): + assert_index_equal(bands[key].index, short_timeindex) + # must not raise KeyError + edisgo_obj.electromobility.flexibility_bands[key].loc[short_timeindex] + + def test_get_flexibility_bands_carries_forward_true_start_end(self): + """ + Regression test for ADR 0002: upper_energy/lower_energy must reflect + each event's TRUE park_start_timesteps/park_end_timesteps, even when + the active timeindex's window starts after (or ends before) that + true start/end - the bands must not be reset to zero / re-anchored + at the window's own edge. Verified by comparing the same event's + band value at a shared calendar timestamp, once built over the full + SimBEV span and once built over a window starting after the event's + true park_start_timesteps. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + full_timeindex = pd.date_range("1/1/2011", periods=200, freq="15min") + edisgo_obj.set_timeindex(full_timeindex) + full_bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, ["work", "public", "home", "hpc"] + ) + + # pick a charging point/timestamp where upper_energy is already + # elevated (i.e. charging started earlier and hasn't finished) - + # windowing from here on must preserve that value, not reset to 0 + upper_energy_full = full_bands["upper_energy"] + nonzero_mask = upper_energy_full > 0 + elevated_positions = nonzero_mask[nonzero_mask.any(axis=1)] + assert not elevated_positions.empty + window_start = elevated_positions.index[len(elevated_positions.index) // 2] + cp_id = elevated_positions.loc[window_start][ + elevated_positions.loc[window_start] + ].index[0] + expected_value = upper_energy_full.loc[window_start, cp_id] + assert expected_value > 0 + + edisgo_obj_windowed = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj_windowed, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj_windowed) + electromobility_import.integrate_charging_parks(edisgo_obj_windowed) + windowed_timeindex = full_timeindex[full_timeindex >= window_start] + edisgo_obj_windowed.set_timeindex(windowed_timeindex) + + windowed_bands = edisgo_obj_windowed.electromobility.get_flexibility_bands( + edisgo_obj_windowed, ["work", "public", "home", "hpc"] + ) + windowed_value = windowed_bands["upper_energy"].loc[window_start, cp_id] + + assert windowed_value == pytest.approx(expected_value) + + def test_get_flexibility_bands_reflects_partial_progress_for_unfinished_event(self): + """ + Regression test for ADR 0002: even though get_flexibility_bands now + filters out charging processes with zero overlap with the active + timeindex (see test_get_flexibility_bands_excludes_events_and_limits_ + array_size below), a RETAINED event that straddles the window + boundary must still report only the energy that charging-so-far + implies at each in-window timestep - never the event's full + (possibly not-yet-delivered) chargingdemand_kWh and never a naive + proportional share of it. upper_energy/lower_energy are cumulative + running totals of physically possible charging progress, not a + fixed total being allocated, and this must hold regardless of + whether the array is sized to SimBEV's full range or only to what's + needed. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + integrated = edisgo_obj.electromobility.integrated_charging_parks_df + park_id = integrated.index[0] + template = edisgo_obj.electromobility.charging_processes_df[ + edisgo_obj.electromobility.charging_processes_df.charging_park_id == park_id + ].iloc[0] + edisgo_id = integrated.at[park_id, "edisgo_id"] + + # event needs 60 steps of charging at 10 kW (150 kWh) to fulfil its + # demand, parked for 100 steps [50, 149] - charging is NOT finished + # by step 95 (needs steps 50-109) + edisgo_obj.electromobility.charging_processes_df = pd.DataFrame( + { + "ags": [template.ags], + "car_id": [0], + "destination": [template.destination], + "use_case": [template.use_case], + "nominal_charging_capacity_kW": [10.0], + "grid_charging_capacity_kW": [10.0], + "chargingdemand_kWh": [150.0], + "park_time_timesteps": [100], + "park_start_timesteps": [50], + "park_end_timesteps": [149], + "charging_park_id": [park_id], + "charging_point_id": [template.charging_point_id], + } + ) + + # active timeindex ends at step 95 - 46 steps into the 60-step + # charge (steps 50-95 inclusive = 46 steps) + timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") + edisgo_obj.set_timeindex(timeindex) + + bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, [template.use_case] + ) + upper_energy_at_cutoff = bands["upper_energy"][edisgo_id].iloc[-1] + + steps_charged_by_cutoff = 46 + expected_kWh = steps_charged_by_cutoff * 10.0 / 4 # 10 kW, 15-min steps + full_demand_kWh = 150.0 + proportional_share_kWh = full_demand_kWh * steps_charged_by_cutoff / 60 + + assert upper_energy_at_cutoff * 1e3 == pytest.approx(expected_kWh) + # must not be the full (not-yet-delivered) demand ... + assert upper_energy_at_cutoff * 1e3 != pytest.approx(full_demand_kWh) + # ... and not a naive proportional share of it either (they happen + # to coincide here only because chargingdemand_kWh/park_time_timesteps + # is linear - assert the true value directly, not this coincidence) + assert expected_kWh == pytest.approx(proportional_share_kWh) + + def test_get_flexibility_bands_excludes_events_and_limits_array_size(self): + """ + Regression test for ADR 0002: get_flexibility_bands must not build + over SimBEV's entire simulated range (nor over every charging + process) regardless of how much shorter the active timeindex is - + this was the actual "build full, then crop" waste this ADR targets, + distinct from (and originally mistaken for) mere value-correctness + after the final .loc[edisgo_timeindex] clip. A fully out-of-window + event must not extend the internal construction array at all, and + the returned bands must still be correct and exactly the active + timeindex's length. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + integrated = edisgo_obj.electromobility.integrated_charging_parks_df + park_id = integrated.index[0] + template = edisgo_obj.electromobility.charging_processes_df[ + edisgo_obj.electromobility.charging_processes_df.charging_park_id == park_id + ].iloc[0] + edisgo_id = integrated.at[park_id, "edisgo_id"] + + # one event fully inside the active window, one event far outside + # it (near the end of SimBEV's simulated week) that must not affect + # the construction array's size at all + edisgo_obj.electromobility.charging_processes_df = pd.DataFrame( + { + "ags": [template.ags, template.ags], + "car_id": [0, 1], + "destination": [template.destination, template.destination], + "use_case": [template.use_case, template.use_case], + "nominal_charging_capacity_kW": [10.0, 10.0], + "grid_charging_capacity_kW": [10.0, 10.0], + "chargingdemand_kWh": [10.0, 10.0], + "park_time_timesteps": [20, 20], + "park_start_timesteps": [10, 600], + "park_end_timesteps": [29, 619], + "charging_park_id": [park_id, park_id], + "charging_point_id": [ + template.charging_point_id, + template.charging_point_id, + ], + } + ) + + timeindex = pd.date_range("1/1/2011", periods=96, freq="15min") # 1 day + edisgo_obj.set_timeindex(timeindex) + + orig_zeros = np.zeros + shapes = [] + + def spy_zeros(shape, *a, **kw): + shapes.append(shape) + return orig_zeros(shape, *a, **kw) + + np.zeros = spy_zeros + try: + bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, [template.use_case] + ) + finally: + np.zeros = orig_zeros + + # construction arrays must be far smaller than SimBEV's full + # simulated week (672 steps) - sized only around the active window + # and the in-window event, not the far-away out-of-window one + assert all(shape[0] < 200 for shape in shapes) + + # returned bands are still correct: exactly the active timeindex's + # length, and reflect only the in-window event's contribution + assert_index_equal(bands["upper_power"].index, timeindex) + assert bands["upper_power"][edisgo_id].sum() > 0 + + def test_get_flexibility_bands_clips_independently_per_gapped_interval(self): + """ + Regression test for ADR 0002: with a non-contiguous active + timeindex, each disjoint interval's bands must match exactly what a + standalone run scoped to just that interval would produce - i.e. + clipping the true, full band per interval, with no interaction + between intervals. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + run_1 = pd.date_range("1/1/2011", periods=24, freq="15min") + run_2 = pd.date_range("1/1/2011", periods=24, freq="15min") + pd.Timedelta( + days=3 + ) + gapped_timeindex = run_1.union(run_2) + edisgo_obj.set_timeindex(gapped_timeindex) + + gapped_bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, ["work", "public", "home", "hpc"] + ) + + edisgo_obj_run1 = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj_run1, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj_run1) + electromobility_import.integrate_charging_parks(edisgo_obj_run1) + edisgo_obj_run1.set_timeindex(run_1) + run1_bands = edisgo_obj_run1.electromobility.get_flexibility_bands( + edisgo_obj_run1, ["work", "public", "home", "hpc"] + ) + + for key in ("upper_power", "lower_energy", "upper_energy"): + assert_frame_equal( + gapped_bands[key].loc[run_1], + run1_bands[key], + check_freq=False, + ) + + def test_get_flexibility_bands_empty_timeindex_is_a_no_op(self): + """ + With no timeindex set at all, get_flexibility_bands must return the + bands untouched, spanning SimBEV's own native calendar/range - there + is nothing to align/trim against yet. + """ + edisgo_obj = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo_obj, self.simbev_path, self.tracbev_path + ) + electromobility_import.distribute_charging_demand(edisgo_obj) + electromobility_import.integrate_charging_parks(edisgo_obj) + + assert edisgo_obj.timeseries.timeindex.empty + + bands = edisgo_obj.electromobility.get_flexibility_bands( + edisgo_obj, ["work", "public"] + ) + + assert len(bands["upper_power"].index) == 7 * 96 # 7 days, 15-min steps + assert bands["upper_power"].index[0] == pd.Timestamp("2011-01-01") + def test_fix_flexibility_bands_rounding_errors(self, caplog): # set up test data # set charging efficiency to 1 to make things easier diff --git a/tests/network/test_heat.py b/tests/network/test_heat.py index 6ed384bef..aabebdfbd 100644 --- a/tests/network/test_heat.py +++ b/tests/network/test_heat.py @@ -363,6 +363,23 @@ def test_resample_timeseries(self): assert len(heatpump.heat_demand_df) == 2 assert len(heatpump.cop_df) == 2 + def test_resample_timeseries_preserves_gapped_index(self): + """ + Regression test: resampling a gapped index must not bridge the gap + with resample artifacts (pandas' own .resample() would otherwise + fabricate contiguous data across it). + """ + gapped_index = pd.date_range("2035-01-08", periods=24, freq="h").union( + pd.date_range("2035-06-10", periods=24, freq="h") + ) + heatpump = HeatPump() + heatpump.cop_df = pd.DataFrame({"hp1": [5.0] * 48}, index=gapped_index) + + heatpump.resample_timeseries(freq="15min") + gap = heatpump.cop_df.index.to_series().diff().max() + assert gap > pd.Timedelta("15min") + assert len(heatpump.cop_df) == 192 # 2 runs * 24h * 4 (15min steps/h) + def test_check_integrity(self, caplog): # check for empty HeatPump class heatpump = HeatPump() diff --git a/tests/network/test_overlying_grid.py b/tests/network/test_overlying_grid.py index 83ff3564c..fc2c32d2c 100644 --- a/tests/network/test_overlying_grid.py +++ b/tests/network/test_overlying_grid.py @@ -156,6 +156,24 @@ def test_resample(self, caplog): "Data cannot be resampled as it only contains one time step." in caplog.text ) + def test_resample_preserves_gapped_index(self): + """ + Regression test: resampling a gapped index must not bridge the gap + with resample artifacts. + """ + gapped_index = pd.date_range("2035-01-08", periods=24, freq="h").union( + pd.date_range("2035-06-10", periods=24, freq="h") + ) + overlying_grid = OverlyingGrid() + overlying_grid.feedin_district_heating = pd.DataFrame( + {"dh1": [1.4] * 48}, index=gapped_index + ) + + overlying_grid.resample(freq="15min") + gap = overlying_grid.feedin_district_heating.index.to_series().diff().max() + assert gap > pd.Timedelta("15min") + assert len(overlying_grid.feedin_district_heating) == 192 + class TestOverlyingGridFunc: @classmethod @@ -273,28 +291,11 @@ def setup_flexibility_data(self): df, ) - # Resample timeseries and reindex to hourly timedelta + # Resample timeseries and reindex to hourly timedelta. DSM (p_min/p_max) + # is resampled by this call too (eDisGo#703) - no separate manual + # DSM resample needed anymore. self.edisgo.resample_timeseries(freq="1min") - for attr in ["p_min", "p_max"]: - new_dates = pd.DatetimeIndex( - [getattr(self.edisgo.dsm, attr).index[-1] + pd.Timedelta("1h")] - ) - setattr( - self.edisgo.dsm, - attr, - getattr(self.edisgo.dsm, attr) - .reindex( - getattr(self.edisgo.dsm, attr) - .index.union(new_dates) - .unique() - .sort_values() - ) - .ffill() - .resample("1min") - .ffill() - .iloc[:-1], - ) self.timesteps = pd.date_range(start="01/01/2018", periods=240, freq="h") attributes = self.edisgo.timeseries._attributes for attr in attributes: diff --git a/tests/network/test_timeseries.py b/tests/network/test_timeseries.py index 15b9c1071..274ede7c4 100644 --- a/tests/network/test_timeseries.py +++ b/tests/network/test_timeseries.py @@ -1403,6 +1403,27 @@ def test_predefined_fluctuating_generators_by_technology(self): ) # fmt: on + def test_predefined_fluctuating_generators_by_technology_missing_timesteps( + self, + ): + """ + Regression test for eDisGo#703: the self-provided-DataFrame path + used to silently accept a DataFrame missing time steps required by + the active timeindex. + """ + timeindex = pd.date_range("1/1/2011 12:00", periods=2, freq="H") + self.edisgo.timeseries.timeindex = timeindex + + incomplete_ts = pd.DataFrame( + data={"wind": [1], "solar": [3]}, + index=timeindex[:1], + ) + + with pytest.raises(ValueError, match="ts_generators"): + self.edisgo.timeseries.predefined_fluctuating_generators_by_technology( + self.edisgo, incomplete_ts + ) + def test_predefined_fluctuating_generators_by_technology_oedb(self): edisgo_object = EDisGo( ding0_grid=pytest.ding0_test_network_3_path, legacy_ding0_grids=False @@ -1548,6 +1569,27 @@ def test_predefined_dispatchable_generators_by_technology(self): ) # fmt: on + def test_predefined_dispatchable_generators_by_technology_missing_timesteps( + self, + ): + """ + Regression test for eDisGo#703: the self-provided-DataFrame path + used to silently accept a DataFrame missing time steps required by + the active timeindex. + """ + timeindex = pd.date_range("1/1/2011 12:00", periods=2, freq="H") + self.edisgo.timeseries.timeindex = timeindex + + incomplete_ts = pd.DataFrame( + data={"other": [5]}, + index=timeindex[:1], + ) + + with pytest.raises(ValueError, match="ts_generators"): + self.edisgo.timeseries.predefined_dispatchable_generators_by_technology( + self.edisgo, incomplete_ts + ) + def test_predefined_conventional_loads_by_sector(self, caplog): index = pd.date_range("1/1/2018", periods=3, freq="H") self.edisgo.timeseries.timeindex = index @@ -1812,6 +1854,27 @@ def test_predefined_conventional_loads_by_sector(self, caplog): original_annual_consumption ) + def test_predefined_conventional_loads_by_sector_raises_on_missing_timesteps( + self, + ): + """ + Regression test for eDisGo#703: the self-provided-DataFrame path + used to silently accept a DataFrame missing time steps required by + the active timeindex. + """ + timeindex = pd.date_range("1/1/2018", periods=3, freq="H") + self.edisgo.timeseries.timeindex = timeindex + + incomplete_ts = pd.DataFrame( + data={"residential": [1, 2]}, + index=timeindex[:2], + ) + + with pytest.raises(ValueError, match="ts_loads"): + self.edisgo.timeseries.predefined_conventional_loads_by_sector( + self.edisgo, incomplete_ts + ) + def test_predefined_charging_points_by_use_case(self, caplog): index = pd.date_range("1/1/2018", periods=3, freq="H") self.edisgo.timeseries.timeindex = index @@ -1919,6 +1982,27 @@ def test_predefined_charging_points_by_use_case(self, caplog): == (3, 5) # fmt: on + def test_predefined_charging_points_by_use_case_raises_on_missing_timesteps( + self, + ): + """ + Regression test for eDisGo#703: the self-provided-DataFrame path + used to silently accept a DataFrame missing time steps required by + the active timeindex. + """ + timeindex = pd.date_range("1/1/2018", periods=3, freq="H") + self.edisgo.timeseries.timeindex = timeindex + + incomplete_ts = pd.DataFrame( + data={"home": [1, 2]}, + index=timeindex[:2], + ) + + with pytest.raises(ValueError, match="ts_loads"): + self.edisgo.timeseries.predefined_charging_points_by_use_case( + self.edisgo, incomplete_ts + ) + def test_fixed_cosphi(self): # set active power time series for fixed cosphi timeindex = pd.date_range("1/1/1970", periods=3, freq="H") @@ -2339,8 +2423,8 @@ def test_integrity_check(self, caplog): setattr(self.edisgo.timeseries, attr, ts_tmp_duplicated) self.edisgo.timeseries.check_integrity() assert ( - f"{attr} has duplicated columns: {ts_tmp.iloc[:, 0:2].columns.values}" - in caplog.text + f"{attr} has duplicated columns: " + f"{ts_tmp.iloc[:, 0:2].columns.values}" in caplog.text ) caplog.clear() setattr(self.edisgo.timeseries, attr, ts_tmp) @@ -2497,6 +2581,30 @@ def test_resample(self): atol=1e-5, ) + def test_resample_preserves_gapped_timeindex(self): + """ + Regression test: resampling a gapped timeindex (as produced by + select_timesteps in auto mode, which deliberately keeps two disjoint + intervals separate) must not bridge the gap with resample artifacts - + the gap must survive an up-sample/down-sample round-trip, and the + timeindex must be restored exactly. + """ + gapped_index = pd.date_range("2035-01-08", periods=24, freq="h").union( + pd.date_range("2035-06-10", periods=24, freq="h") + ) + self.edisgo.set_timeindex(gapped_index) + gen = self.edisgo.topology.generators_df.index[0] + self.edisgo.set_time_series_manual( + generators_p=pd.DataFrame({gen: [0.1] * 48}, index=gapped_index) + ) + + self.edisgo.timeseries.resample(freq="15min") + gap = self.edisgo.timeseries.timeindex.to_series().diff().max() + assert gap > pd.Timedelta("15min") + + self.edisgo.timeseries.resample(freq="1h") + assert_index_equal(self.edisgo.timeseries.timeindex, gapped_index) + def test_scale_timeseries(self): self.edisgo.set_time_series_worst_case_analysis() edisgo_scaled = copy.deepcopy(self.edisgo) diff --git a/tests/opf/test_powermodels_opf.py b/tests/opf/test_powermodels_opf.py index c79cacb06..9b4300e6d 100644 --- a/tests/opf/test_powermodels_opf.py +++ b/tests/opf/test_powermodels_opf.py @@ -339,3 +339,194 @@ def test_pm_optimize(self): ) ) ) + + +class TestContiguousIntervals: + def test_contiguous_index_is_one_interval(self): + from edisgo.opf.powermodels_opf import _contiguous_intervals + + ti = pd.date_range("2035-01-01", periods=48, freq="h") + result = _contiguous_intervals(ti) + assert len(result) == 1 and result[0].equals(ti) + + def test_two_disconnected_intervals(self): + from edisgo.opf.powermodels_opf import _contiguous_intervals + + a = pd.date_range("2035-01-01", periods=24, freq="h") + b = pd.date_range("2035-07-01", periods=24, freq="h") + result = _contiguous_intervals(a.union(b)) + assert len(result) == 2 + assert result[0].equals(a) and result[1].equals(b) + + def test_freq_restored_on_freqless_index(self): + from edisgo.opf.powermodels_opf import _contiguous_intervals + + idx = pd.DatetimeIndex(pd.date_range("2035-01-01", periods=48, freq="h").values) + assert idx.freq is None + result = _contiguous_intervals(idx) + assert len(result) == 1 and result[0].freq is not None + + def test_single_and_empty(self): + from edisgo.opf.powermodels_opf import _contiguous_intervals + + assert len(_contiguous_intervals(pd.date_range("2035-01-01", periods=1))) == 1 + assert _contiguous_intervals(pd.DatetimeIndex([])) == [] + + +class TestMergeOpfTimeFrames: + @staticmethod + def _empty_snapshot(opf, slack_generator_t): + return { + "slack_generator_t": slack_generator_t, + "hv_requirement_slacks_t": pd.DataFrame(), + "lines_t": {k: pd.DataFrame() for k in opf.lines_t._attributes()}, + "heat_storage_t": { + k: pd.DataFrame() for k in opf.heat_storage_t._attributes() + }, + "grid_slacks_t": { + k: pd.DataFrame() for k in opf.grid_slacks_t._attributes() + }, + "battery_storage_t": { + k: pd.DataFrame() for k in opf.battery_storage_t._attributes() + }, + } + + def test_flat_frame_concatenated_and_sorted(self): + from edisgo.opf.powermodels_opf import _merge_opf_time_frames + from edisgo.opf.results.opf_result_class import OPFResults + + opf = OPFResults() + a = pd.DataFrame({"x": [1.0]}, index=pd.date_range("2035-01-01", periods=1)) + b = pd.DataFrame({"x": [2.0]}, index=pd.date_range("2035-07-01", periods=1)) + _merge_opf_time_frames( + opf, [self._empty_snapshot(opf, a), self._empty_snapshot(opf, b)] + ) + assert len(opf.slack_generator_t) == 2 + assert list(opf.slack_generator_t["x"]) == [1.0, 2.0] + + +class TestPmOptimizeIntervalSplit: + """pm_optimize's multi-interval split, with the single-interval OPF stubbed + (no Julia/DB). Patches powermodels_opf._pm_optimize_single.""" + + @pytest.fixture + def edisgo_obj(self): + e = EDisGo(ding0_grid=pytest.ding0_test_network_path) + e.set_timeindex(pd.date_range("2035-01-01", periods=24, freq="h")) + return e + + def test_single_interval_calls_once_with_freq(self, edisgo_obj, monkeypatch): + import edisgo.opf.powermodels_opf as pmo + + ti = pd.DatetimeIndex(pd.date_range("2035-01-01", periods=24, freq="h").values) + assert ti.freq is None + edisgo_obj.set_timeindex(ti) + seen = [] + monkeypatch.setattr( + pmo, + "_pm_optimize_single", + lambda e, **kw: seen.append(e.timeseries.timeindex.freq), + ) + pmo.pm_optimize(edisgo_obj) + assert seen == [pd.tseries.frequencies.to_offset("h")] + + def test_two_intervals_run_separately_and_restore(self, edisgo_obj, monkeypatch): + import edisgo.opf.powermodels_opf as pmo + + a = pd.date_range("2035-01-01", periods=24, freq="h") + b = pd.date_range("2035-07-01", periods=24, freq="h") + full = a.union(b) + edisgo_obj.set_timeindex(full) + seen = [] + + def fake_single(e, **kw): + seen.append(e.timeseries.timeindex) + e.opf_results.status = "OPTIMAL" + e.opf_results.solver = "Gurobi" + e.opf_results.solution_time = 1.0 + + monkeypatch.setattr(pmo, "_pm_optimize_single", fake_single) + pmo.pm_optimize(edisgo_obj) + assert len(seen) == 2 and seen[0].equals(a) and seen[1].equals(b) + assert edisgo_obj.timeseries.timeindex.equals(full) + assert len(edisgo_obj.opf_results.interval_results) == 2 + assert edisgo_obj.opf_results.solution_time == 2.0 + assert edisgo_obj.opf_results.status == "OPTIMAL" + + def test_overlying_grid_state_restored(self, edisgo_obj, monkeypatch): + import edisgo.opf.powermodels_opf as pmo + + a = pd.date_range("2035-01-01", periods=24, freq="h") + b = pd.date_range("2035-07-01", periods=24, freq="h") + full = a.union(b) + edisgo_obj.set_timeindex(full) + edisgo_obj.overlying_grid.storage_units_soc = pd.Series(1.0, index=full) + seen_types = [] + + def fake_single(e, **kw): + og = e.overlying_grid + seen_types.append(type(og.storage_units_soc).__name__) + og.storage_units_soc = pd.DataFrame( + 0.0, index=e.timeseries.timeindex, columns=["s1"] + ) + e.opf_results.status = "OPTIMAL" + + monkeypatch.setattr(pmo, "_pm_optimize_single", fake_single) + pmo.pm_optimize(edisgo_obj) + assert seen_types == ["Series", "Series"] + assert isinstance(edisgo_obj.overlying_grid.storage_units_soc, pd.Series) + + def test_reactive_power_restored(self, edisgo_obj, monkeypatch): + import edisgo.opf.powermodels_opf as pmo + + a = pd.date_range("2035-01-01", periods=24, freq="h") + b = pd.date_range("2035-07-01", periods=24, freq="h") + full = a.union(b) + edisgo_obj.set_timeindex(full) + gen = edisgo_obj.topology.generators_df.index[0] + edisgo_obj.timeseries._generators_reactive_power = pd.DataFrame( + 0.0, index=full, columns=[gen] + ) + seen_ok = [] + + def fake_single(e, **kw): + ti = e.timeseries.timeindex + q = e.timeseries.generators_reactive_power + seen_ok.append(not q.empty and q.index.equals(ti)) + e.timeseries._generators_reactive_power = pd.DataFrame( + 0.0, index=ti, columns=[gen] + ) + e.opf_results.status = "OPTIMAL" + + monkeypatch.setattr(pmo, "_pm_optimize_single", fake_single) + pmo.pm_optimize(edisgo_obj) + assert seen_ok == [True, True] + assert edisgo_obj.timeseries._generators_reactive_power.index.equals(full) + + def test_infeasible_interval_stores_report_and_raises( + self, edisgo_obj, monkeypatch + ): + import edisgo.opf.powermodels_opf as pmo + + from edisgo.flex_opt.exceptions import InfeasibleModelError + + a = pd.date_range("2035-01-01", periods=24, freq="h") + b = pd.date_range("2035-07-01", periods=24, freq="h") + full = a.union(b) + edisgo_obj.set_timeindex(full) + + def fake_single(e, **kw): + if e.timeseries.timeindex[0] == a[0]: + e.opf_results.status = "OPTIMAL" + e.opf_results.solution_time = 1.0 + else: + raise InfeasibleModelError("stub") + + monkeypatch.setattr(pmo, "_pm_optimize_single", fake_single) + with pytest.raises(InfeasibleModelError): + pmo.pm_optimize(edisgo_obj) + report = edisgo_obj.opf_results.interval_results + assert len(report) == 2 + assert report[0]["status"] == "OPTIMAL" + assert report[1]["status"] == "infeasible" + assert edisgo_obj.timeseries.timeindex.equals(full) diff --git a/tests/run/__init__.py b/tests/run/__init__.py new file mode 100644 index 000000000..baf15e08b --- /dev/null +++ b/tests/run/__init__.py @@ -0,0 +1 @@ +"""Tests for the :mod:`edisgo.run` pipeline runner.""" diff --git a/tests/run/test_config.py b/tests/run/test_config.py new file mode 100644 index 000000000..10b4ec474 --- /dev/null +++ b/tests/run/test_config.py @@ -0,0 +1,154 @@ +""" +Unit tests for :mod:`edisgo.run.config` — loader, merger, adapter. + +Covers YAML/JSON parity, ``extends`` resolution (preset-by-name and +relative paths), deep-merge semantics, stage normalization, and the +eGo-legacy adapter. +""" +import json + +import pytest +import yaml + +from edisgo.run.config import _deep_merge, load_config + + +def _write(tmp_path, name, data): + """ + Helper: write ``data`` to ``tmp_path/name`` as YAML or JSON. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest-provided temporary directory. + name : str + File name with extension (``.yaml``/``.yml``/``.json``). + data : dict + Payload. + + Returns + ------- + pathlib.Path + Path to the written file. + + """ + path = tmp_path / name + if name.endswith(".json"): + path.write_text(json.dumps(data)) + else: + path.write_text(yaml.safe_dump(data)) + return path + + +def test_load_flat_pipeline_normalized_to_stages(tmp_path): + """A flat ``pipeline:`` must normalize to a single 'main' stage.""" + p = _write(tmp_path, "cfg.yaml", { + "scenario": "eGon2035", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + }) + cfg = load_config(str(p)) + assert "pipeline" not in cfg + assert cfg["stages"] == [ + {"name": "main", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"]} + ] + + +def test_yaml_and_json_equivalent(tmp_path): + """YAML and JSON payloads with identical content must load equal.""" + data = { + "scenario": "eGon2035", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + } + yaml_path = _write(tmp_path, "cfg.yaml", data) + json_path = _write(tmp_path, "cfg.json", data) + assert load_config(str(yaml_path)) == load_config(str(json_path)) + + +def test_extends_merges_parent(tmp_path): + """Child config must deep-merge with its ``extends:`` parent.""" + parent = _write(tmp_path, "parent.yaml", { + "scenario": "eGon2035", + "grid": {"legacy_ding0_grids": False}, + "pipeline": ["setup_grid", "reinforce"], + }) + child = _write(tmp_path, "child.yaml", { + "extends": str(parent), + "grid": {"ding0_path": "/tmp/xyz"}, + }) + cfg = load_config(str(child)) + assert cfg["scenario"] == "eGon2035" + assert cfg["grid"] == { + "legacy_ding0_grids": False, "ding0_path": "/tmp/xyz" + } + assert cfg["stages"][0]["pipeline"] == ["setup_grid", "reinforce"] + + +def test_extends_preset_by_name(tmp_path): + """``extends: basic`` must resolve to the bundled basic preset.""" + child = _write(tmp_path, "child.yaml", { + "extends": "basic", + "grid": {"ding0_path": "/tmp/xyz"}, + }) + cfg = load_config(str(child)) + assert "stages" in cfg + assert cfg["grid"]["ding0_path"] == "/tmp/xyz" + + +def test_deep_merge_nested(): + """Nested dicts must be merged key-by-key, child wins on conflict.""" + base = {"a": {"b": 1, "c": 2}, "d": 4} + over = {"a": {"b": 99, "e": 5}} + merged = _deep_merge(base, over) + assert merged == {"a": {"b": 99, "c": 2, "e": 5}, "d": 4} + + +def test_both_pipeline_and_stages_rejected(tmp_path): + """Top-level ``pipeline`` and ``stages`` are mutually exclusive.""" + p = _write(tmp_path, "cfg.yaml", { + "pipeline": ["setup_grid"], + "stages": [{"name": "x", "pipeline": ["setup_grid"]}], + }) + with pytest.raises(ValueError, match="both"): + load_config(str(p)) + + +def test_duplicate_stage_names_rejected(tmp_path): + """Stage names must be unique; duplicates raise ValueError.""" + p = _write(tmp_path, "cfg.yaml", { + "stages": [ + {"name": "x", "pipeline": ["setup_grid"]}, + {"name": "x", "pipeline": ["reinforce"]}, + ], + }) + with pytest.raises(ValueError, match="Duplicate stage"): + load_config(str(p)) + + +def test_ego_legacy_adapter(tmp_path): + """An eGo ``scenario_setting_*.json`` must adapt to the new schema.""" + ego_cfg = { + "eGo": {"eDisGo": True}, + "eTraGo": {"scn_name": "eGon2035"}, + "eDisGo": { + "grid_path": "/some/path", + "results": "/tmp/results", + "tasks": [ + "1_setup_grid", + "base_reinforce", + "import_heat_pumps_from_db", + "worst_case_ts", + "5_grid_reinforcement", + ], + }, + "database": {"host": "localhost"}, + } + p = _write(tmp_path, "legacy.json", ego_cfg) + cfg = load_config(str(p)) + assert cfg["scenario"] == "eGon2035" + assert cfg["grid"]["ding0_path"] == "/some/path" + assert cfg["stages"][0]["pipeline"] == [ + "setup_grid", "base_reinforce", "import_heat_pumps", + "worst_case_ts", "reinforce", + ] + assert cfg["database"]["host"] == "localhost" diff --git a/tests/run/test_registry.py b/tests/run/test_registry.py new file mode 100644 index 000000000..a56070bf6 --- /dev/null +++ b/tests/run/test_registry.py @@ -0,0 +1,35 @@ +""" +Unit tests for :mod:`edisgo.run.registry`. + +Verifies that core tasks are discoverable, that ``get_task`` raises a +useful error on typos, and that duplicate registrations are rejected. +""" +import pytest + +from edisgo.run.registry import get_task, known_tasks, register_task + + +def test_known_tasks_contains_core(): + """All core task names must be registered on import.""" + tasks = known_tasks() + for core in ["setup_grid", "worst_case_ts", "reactive_power", + "reinforce", "analyze", "save"]: + assert core in tasks + + +def test_get_task_unknown_raises(): + """Unknown task names must surface as a descriptive KeyError.""" + with pytest.raises(KeyError, match="Unknown task"): + get_task("does_not_exist") + + +def test_register_task_duplicate_raises(): + """Registering the same task name twice is a bug — must raise.""" + @register_task("_test_task_for_dup_check") + def _a(edisgo, ctx): + """Marker task #1 — test fixture only.""" + + with pytest.raises(ValueError, match="already registered"): + @register_task("_test_task_for_dup_check") + def _b(edisgo, ctx): + """Marker task #2 — test fixture only, must not register.""" diff --git a/tests/run/test_runner.py b/tests/run/test_runner.py new file mode 100644 index 000000000..0ecb6d08e --- /dev/null +++ b/tests/run/test_runner.py @@ -0,0 +1,118 @@ +""" +End-to-end tests for the eDisGo pipeline runner. + +Uses the small test grid under ``tests/data/ding0_test_network_2`` +(exposed by :mod:`tests.conftest` as +``pytest.ding0_test_network_2_path``) to run full pipelines without +touching the database. Covers: + +* the standalone ``run_edisgo`` entry point with a flat pipeline, +* the instance method ``EDisGo.run_pipeline``, +* the stage mechanism with ``save`` + ``load_from``. +""" +import os + +import pytest + +from edisgo.run import run_edisgo + + +@pytest.fixture +def basic_cfg(tmp_path): + """ + Minimal end-to-end config fixture. + + Produces a config that loads the small ding0 test grid, sets + worst-case time series, fixes reactive power, checks integrity, + runs reinforcement, and saves — no database needed. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest-provided temp directory for the run's artifacts. + + Returns + ------- + dict + The config dict. + + """ + return { + "scenario": "eGon2035", + "grid": { + "ding0_path": pytest.ding0_test_network_2_path, + "legacy_ding0_grids": True, + }, + "results": {"directory": str(tmp_path)}, + "pipeline": [ + "setup_grid", + "worst_case_ts", + "reactive_power", + "check_integrity", + "reinforce", + "save", + ], + } + + +def test_runner_basic_end_to_end(basic_cfg): + """A flat-pipeline run must execute and persist the expected artifact.""" + edisgo = run_edisgo(basic_cfg) + assert edisgo is not None + assert edisgo.topology is not None + assert os.path.isdir(os.path.join(basic_cfg["results"]["directory"], + "main")) + + +def test_runner_method_on_edisgo(basic_cfg): + """``EDisGo.run_pipeline`` must operate on the existing instance.""" + from edisgo import EDisGo + + basic_cfg["pipeline"] = basic_cfg["pipeline"][1:] # skip setup_grid + edisgo = EDisGo( + ding0_grid=basic_cfg["grid"]["ding0_path"], + legacy_ding0_grids=True, + ) + edisgo = edisgo.run_pipeline(basic_cfg) + assert edisgo.topology is not None + + +def test_runner_two_stages_with_load_from(tmp_path): + """ + A two-stage run must save the first stage and reload it via + ``load_from`` in the second stage, producing both artifacts. + """ + cfg = { + "scenario": "eGon2035", + "grid": { + "ding0_path": pytest.ding0_test_network_2_path, + "legacy_ding0_grids": True, + }, + "results": {"directory": str(tmp_path)}, + "stages": [ + { + "name": "base", + "pipeline": [ + "setup_grid", + "worst_case_ts", + "reactive_power", + "reinforce", + {"save": {"archive": True}}, + ], + }, + { + "name": "scenario", + "load_from": "base", + "pipeline": [ + "worst_case_ts", + "reactive_power", + "reinforce", + "save", + ], + }, + ], + } + edisgo = run_edisgo(cfg) + assert edisgo.topology is not None + assert os.path.exists(os.path.join(str(tmp_path), "base.zip")) + assert os.path.isdir(os.path.join(str(tmp_path), "scenario")) diff --git a/tests/run/test_tasks.py b/tests/run/test_tasks.py new file mode 100644 index 000000000..46ba7a9cc --- /dev/null +++ b/tests/run/test_tasks.py @@ -0,0 +1,377 @@ +""" +Unit tests for individual pipeline tasks in :mod:`edisgo.run.tasks`. + +These cover the task control flow that unit tests previously missed — the +task modules were the source of every bug found in the review. They run +without a database or SSH tunnel: a small self-constructed ding0 grid is +enough, and the DB-free branches of ``import_overlying_grid_data`` are +exercised directly. +""" + +import glob +import os + +import pandas as pd +import pytest + +import edisgo.run as edisgo_run + +from edisgo.edisgo import EDisGo +from edisgo.io import electromobility_import +from edisgo.run.config import load_config +from edisgo.run.context import RunContext +from edisgo.run.tasks.analysis import task_optimize +from edisgo.run.tasks.flex import task_build_flexibility_bands +from edisgo.run.tasks.io import task_import_overlying_grid_data +from edisgo.run.tasks.timeseries import ( + task_manual_ts, + task_select_timesteps, +) +from edisgo.run.validator import validate +from edisgo.tools.temporal_complexity_reduction import ( + intervals_overlap, + select_two_intervals, +) + + +@pytest.fixture +def edisgo_obj(): + """Small ding0 grid with a 3-step time index, no DB access.""" + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) + edisgo.set_timeindex(pd.date_range("2011-01-01", periods=3, freq="h")) + return edisgo + + +class TestManualTs: + def test_manual_ts_applies_active_power(self, edisgo_obj): + """ + task_manual_ts must forward the eGo-style ``*_active_power`` args to + EDisGo.set_time_series_manual's real parameter names (regression: the + task used to pass unsupported kwargs and always raised TypeError). + """ + ti = edisgo_obj.timeseries.timeindex + gen = edisgo_obj.topology.generators_df.index[0] + df = pd.DataFrame({gen: [0.1, 0.2, 0.3]}, index=ti) + + ctx = RunContext() + result = task_manual_ts(edisgo_obj, ctx, generators_active_power=df) + + assert gen in result.timeseries.generators_active_power.columns + assert ctx.flags["timeseries_set"] is True + + +class TestBuildFlexibilityBands: + def test_build_flexibility_bands_scopes_to_timeindex(self): + """ + Regression test for eDisGo#703: task_build_flexibility_bands used to + need an explicit reduce_timeseries_data_to_given_timeindex call after + get_flexibility_bands to trim/year-align the bands to the active + timeindex; get_flexibility_bands now does this itself, so the task + (which no longer makes that call) must still produce bands matching + edisgo.timeseries.timeindex exactly - including across the year + mismatch between SimBEV's own calendar (2011 in this fixture) and + the scenario timeindex (2035 here). + """ + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_2_path) + electromobility_import.import_electromobility_from_dir( + edisgo, + pytest.simbev_example_scenario_path, + pytest.tracbev_example_scenario_path, + ) + electromobility_import.distribute_charging_demand(edisgo) + electromobility_import.integrate_charging_parks(edisgo) + + short_timeindex = pd.date_range("2035-01-15", periods=24, freq="h") + edisgo.set_timeindex(short_timeindex) + + ctx = RunContext() + result = task_build_flexibility_bands(edisgo, ctx) + + for key in ("upper_power", "lower_energy", "upper_energy"): + pd.testing.assert_index_equal( + result.electromobility.flexibility_bands[key].index, + short_timeindex, + ) + + +class TestImportOverlyingGridData: + def _ctx(self, og_cfg, overlying_grid_data=None): + return RunContext( + raw_config={"overlying_grid": og_cfg}, + overlying_grid_data=overlying_grid_data, + ) + + def test_disabled_returns_unchanged(self): + """enabled: false short-circuits before the grid is even touched.""" + sentinel = object() + ctx = self._ctx({"enabled": False}) + assert task_import_overlying_grid_data(sentinel, ctx) is sentinel + + def test_unknown_source_warns(self, edisgo_obj, caplog): + ctx = self._ctx({"enabled": True, "source": "bogus"}) + result = task_import_overlying_grid_data(edisgo_obj, ctx) + assert result is edisgo_obj + assert "unknown source" in caplog.text + + def test_etrago_without_data_warns(self, edisgo_obj, caplog): + ctx = self._ctx({"enabled": True, "source": "etrago"}, overlying_grid_data=None) + result = task_import_overlying_grid_data(edisgo_obj, ctx) + assert result is edisgo_obj + assert "no" in caplog.text.lower() + + def test_etrago_empty_data_does_not_crash(self, edisgo_obj): + """ + A partial/empty etrago dict must not raise (regression: the task used + to call .empty on dict.get() results that were None). + """ + ctx = self._ctx({"enabled": True, "source": "etrago"}, overlying_grid_data={}) + # must simply return without AttributeError + assert task_import_overlying_grid_data(edisgo_obj, ctx) is edisgo_obj + + def test_csv_without_path_warns(self, edisgo_obj, caplog): + ctx = self._ctx({"enabled": True, "source": "csv"}) + result = task_import_overlying_grid_data(edisgo_obj, ctx) + assert result is edisgo_obj + assert "path" in caplog.text.lower() + + +class TestSelectTimestepsHelpers: + """Pure helpers for auto interval selection — no DB, no power flow.""" + + @staticmethod + def _week(start): + return pd.date_range(start=start, periods=168, freq="h") + + def test_overlap_detection(self): + a = self._week("2035-01-01") + assert intervals_overlap(a, self._week("2035-01-04")) # overlaps + assert not intervals_overlap(a, self._week("2035-06-01")) # disjoint + + def test_disjoint_top_intervals_kept_as_two(self): + load = [self._week("2035-01-01")] + volt = [self._week("2035-06-01")] + result = select_two_intervals(load, volt) + assert len(result) == 2 + assert not intervals_overlap(result[0], result[1]) + + def test_overlap_falls_to_next_ranked_voltage(self): + load = [self._week("2035-01-01")] + # first voltage candidate overlaps the top loading interval, second does not + volt = [self._week("2035-01-03"), self._week("2035-09-01")] + result = select_two_intervals(load, volt) + assert len(result) == 2 + assert result[1].equals(volt[1]) + + def test_all_overlap_concatenates_to_one(self): + load = [self._week("2035-01-01")] + volt = [self._week("2035-01-03")] # only candidate, overlaps + result = select_two_intervals(load, volt) + assert len(result) == 1 + merged = result[0] + # merged interval is contiguous and spans both inputs + assert merged.min() == load[0].min() + assert merged.max() == volt[0].max() + assert (merged[1:] - merged[:-1]).nunique() == 1 # regular spacing + + def test_single_side_only(self): + week = self._week("2035-01-01") + for result in ( + select_two_intervals([week], []), + select_two_intervals([], [week]), + ): + assert len(result) == 1 + assert result[0].equals(week) + assert select_two_intervals([], []) == [] + + +class TestSelectTimestepsManual: + def test_manual_explicit_timestamps_before_imports(self, edisgo_obj): + """ + Manual mode with an empty timeseries (positioned before imports) sets + the index and stashes it for HP/DSM imports. + """ + # start from an empty time index to mimic the pre-import position + edisgo_obj.set_timeindex(pd.DatetimeIndex([])) + ts = ["2011-01-01 00:00", "2011-01-01 02:00"] + ctx = RunContext( + raw_config={"timeseries_selection": {"mode": "manual", "timestamps": ts}} + ) + result = task_select_timesteps(edisgo_obj, ctx) + assert list(result.timeseries.timeindex) == list(pd.to_datetime(ts)) + assert list(ctx.flags["selected_timeindex"]) == list(pd.to_datetime(ts)) + assert ctx.flags["timesteps_selected"] is True + + def test_manual_range_reduces_existing_timeseries(self, edisgo_obj): + """Manual range with an existing 3-step index reduces to the range.""" + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "manual", + "start": "2011-01-01 00:00", + "periods": 2, + "freq": "h", + } + } + ) + result = task_select_timesteps(edisgo_obj, ctx) + assert len(result.timeseries.timeindex) == 2 + + def test_missing_mode_raises(self, edisgo_obj): + with pytest.raises(ValueError, match="mode 'manual' or 'auto'"): + task_select_timesteps(edisgo_obj, RunContext(raw_config={})) + + def test_auto_without_active_power_raises(self, edisgo_obj): + ctx = RunContext(raw_config={"timeseries_selection": {"mode": "auto"}}) + with pytest.raises(ValueError, match="active-power time series"): + task_select_timesteps(edisgo_obj, ctx) + + def test_auto_unknown_method_raises(self, edisgo_obj): + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "auto", + "method": "bogus", + } + } + ) + ctx.flags["timeseries_set"] = True + with pytest.raises(ValueError, match="power_flow.*residual_load"): + task_select_timesteps(edisgo_obj, ctx) + + def test_residual_load_requires_overlying_grid(self, edisgo_obj): + """residual_load method must raise when no overlying-grid data is set.""" + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "auto", + "method": "residual_load", + } + } + ) + ctx.flags["timeseries_set"] = True + with pytest.raises(ValueError, match="overlying-grid data"): + task_select_timesteps(edisgo_obj, ctx) + + +class TestSelectTimestepsPosition: + """The `position` param lets one pipeline carry both a pre-import and a + post-grid select_timesteps step; each no-ops off its mode.""" + + def test_pre_import_noops_in_auto_mode(self, edisgo_obj): + """ + A pre_import step in auto mode must be a no-op — crucially it must NOT + hit the auto guard (no active power set yet), it just returns. + """ + before = edisgo_obj.timeseries.timeindex + ctx = RunContext(raw_config={"timeseries_selection": {"mode": "auto"}}) + result = task_select_timesteps(edisgo_obj, ctx, position="pre_import") + assert result is edisgo_obj + assert result.timeseries.timeindex.equals(before) + assert "timesteps_selected" not in ctx.flags + + def test_post_grid_noops_in_manual_mode(self, edisgo_obj): + """A post_grid step in manual mode must be a no-op (manual already ran + earlier at pre_import).""" + before = edisgo_obj.timeseries.timeindex + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "manual", + "timestamps": ["2011-01-01 00:00"], + } + } + ) + result = task_select_timesteps(edisgo_obj, ctx, position="post_grid") + assert result is edisgo_obj + assert result.timeseries.timeindex.equals(before) + + def test_pre_import_acts_in_manual_mode(self, edisgo_obj): + edisgo_obj.set_timeindex(pd.DatetimeIndex([])) + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "manual", + "timestamps": ["2011-01-01 00:00"], + } + } + ) + task_select_timesteps(edisgo_obj, ctx, position="pre_import") + assert len(edisgo_obj.timeseries.timeindex) == 1 + assert ctx.flags["timesteps_selected"] is True + + def test_bad_position_raises(self, edisgo_obj): + ctx = RunContext(raw_config={"timeseries_selection": {"mode": "manual"}}) + with pytest.raises(ValueError, match="position"): + task_select_timesteps(edisgo_obj, ctx, position="bogus") + + def test_pre_import_sets_default_index_when_empty_in_auto(self, edisgo_obj): + """ + A pre_import step in auto mode with no time index set must establish a + full-year default (so later imports build hourly full-year data), even + though it otherwise no-ops. + """ + edisgo_obj.set_timeindex(pd.DatetimeIndex([])) + ctx = RunContext( + scenario="eGon2035", + raw_config={"timeseries_selection": {"mode": "auto"}}, + ) + task_select_timesteps(edisgo_obj, ctx, position="pre_import") + ti = edisgo_obj.timeseries.timeindex + assert len(ti) == 8760 + assert ti[0].year == 2035 + # still a no-op for actual selection + assert "timesteps_selected" not in ctx.flags + + def test_manual_shifts_user_timestamps_to_timeseries_year(self, edisgo_obj): + """ + Manual mode reducing an existing (differently-yeared) time series shifts + the user timestamps to the time-series year so slicing matches. + """ + # existing time series in 2011 + edisgo_obj.set_timeindex(pd.date_range("2011-06-01", periods=5, freq="h")) + # user selects timestamps written in the scenario year 2035 + ctx = RunContext( + raw_config={ + "timeseries_selection": { + "mode": "manual", + "timestamps": ["2035-06-01 01:00", "2035-06-01 03:00"], + } + } + ) + task_select_timesteps(edisgo_obj, ctx) + ti = edisgo_obj.timeseries.timeindex + assert list(ti) == [ + pd.Timestamp("2011-06-01 01:00"), + pd.Timestamp("2011-06-01 03:00"), + ] + + +class TestOptimizeTaskDelegation: + """task_optimize is thin: expand the `flexible` shortcut and call + edisgo.pm_optimize. The multi-interval split lives in pm_optimize and is + tested in tests/opf/test_powermodels_opf.py.""" + + def test_expands_flexible_shortcut_and_calls_pm_optimize(self, edisgo_obj): + edisgo_obj.set_timeindex(pd.date_range("2035-01-01", periods=24, freq="h")) + captured = {} + + def fake_pm_optimize(**kw): + captured.update(kw) + + edisgo_obj.pm_optimize = fake_pm_optimize + task_optimize(edisgo_obj, RunContext(), flexible=["heat_pumps", "storage"]) + # shortcut expanded to explicit name lists (empty ok if grid lacks type) + assert "flexible_hps" in captured and "flexible_storage_units" in captured + assert isinstance(captured["flexible_hps"], list) + + +def test_all_bundled_presets_validate(): + """ + Every bundled preset must pass the (metadata-driven) validator — this + keeps the task requires/provides declarations in sync with real configs. + """ + presets_dir = os.path.join(os.path.dirname(edisgo_run.__file__), "presets") + presets = sorted(glob.glob(os.path.join(presets_dir, "*.yaml"))) + assert presets, "no bundled presets found" + for path in presets: + validate(load_config(path)) diff --git a/tests/run/test_validator.py b/tests/run/test_validator.py new file mode 100644 index 000000000..9e830e0bc --- /dev/null +++ b/tests/run/test_validator.py @@ -0,0 +1,159 @@ +""" +Unit tests for :mod:`edisgo.run.validator`. + +Each test pins one ordering rule: reactive-before-TS, reinforce +without TS, optimize without flex, flex import without grid, and the +stage-level ``load_from`` constraints. +""" + +import pytest + +from edisgo.run.validator import validate + + +def _wrap(pipeline): + """ + Wrap a flat pipeline into a single-stage config dict. + + Parameters + ---------- + pipeline : list + Ordered list of task names / single-key mappings. + + Returns + ------- + dict + Minimal config in the shape expected by :func:`validate`. + + """ + return {"stages": [{"name": "main", "pipeline": pipeline}]} + + +def test_valid_pipeline(): + """A well-formed pipeline must pass validation without raising.""" + validate( + _wrap(["setup_grid", "worst_case_ts", "reactive_power", "reinforce", "save"]) + ) + + +def test_unknown_task_rejected(): + """Typo'd task names must be rejected.""" + with pytest.raises(ValueError, match="Unknown task"): + validate(_wrap(["setup_grid", "nonexistent_task"])) + + +def test_reactive_before_ts_rejected(): + """reactive_power before a TS task violates the ordering rule.""" + with pytest.raises(ValueError, match="reactive_power"): + validate(_wrap(["setup_grid", "reactive_power", "worst_case_ts"])) + + +def test_select_timesteps_after_reactive_rejected(): + """ + select_timesteps is ts_altering (it reduces the time index), so it must + not appear after reactive_power. + """ + with pytest.raises(ValueError, match="reactive_power"): + validate( + _wrap(["setup_grid", "worst_case_ts", "reactive_power", "select_timesteps"]) + ) + + +def test_select_timesteps_before_reactive_ok(): + """select_timesteps before reactive_power is the intended ordering.""" + validate( + _wrap( + [ + "setup_grid", + "worst_case_ts", + "select_timesteps", + "reactive_power", + "reinforce", + "save", + ] + ) + ) + + +def test_reinforce_without_ts_rejected(): + """reinforce without any prior time-series step must fail.""" + with pytest.raises(ValueError, match="time series"): + validate(_wrap(["setup_grid", "reinforce"])) + + +def test_optimize_without_flex_rejected(): + """optimize requires at least one flex asset to be imported.""" + with pytest.raises(ValueError, match="flex asset"): + validate(_wrap(["setup_grid", "worst_case_ts", "optimize"])) + + +def test_flex_import_before_grid_rejected(): + """Flex imports require a loaded grid — pre-loading is not enough.""" + with pytest.raises(ValueError, match="loaded grid"): + validate(_wrap(["import_heat_pumps", "worst_case_ts", "reinforce"])) + + +def test_stage_load_from_missing_rejected(): + """``load_from: X`` where X has not run must fail.""" + cfg = { + "stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", "reinforce"]}, + {"name": "b", "load_from": "nonexistent", "pipeline": ["reinforce"]}, + ] + } + with pytest.raises(ValueError, match="load_from"): + validate(cfg) + + +def test_stage_load_from_requires_save_in_source(): + """A stage consumed by ``load_from`` must itself end with ``save``.""" + cfg = { + "stages": [ + { + "name": "a", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + }, # no save + {"name": "b", "load_from": "a", "pipeline": ["reinforce"]}, + ] + } + with pytest.raises(ValueError, match="load_from"): + validate(cfg) + + +def test_stage_load_from_with_save_ok(): + """Stage chain with a save in the source must validate successfully.""" + cfg = { + "stages": [ + { + "name": "a", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce", "save"], + }, + # load_from reloads the grid with import_timeseries=False, so the + # consuming stage must set time series itself before reinforce. + { + "name": "b", + "load_from": "a", + "pipeline": ["worst_case_ts", "reinforce", "save"], + }, + ] + } + validate(cfg) + + +def test_stage_load_from_without_ts_rejected(): + """ + load_from does NOT satisfy the time-series prerequisite: the artifact is + reloaded with import_timeseries=False, so reinforce after a bare load_from + (no time-series task in the stage) must be rejected. + """ + cfg = { + "stages": [ + { + "name": "a", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce", "save"], + }, + {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, + ] + } + with pytest.raises(ValueError, match="requires time series"): + validate(cfg) diff --git a/tests/test_edisgo.py b/tests/test_edisgo.py index 9847cc7a1..b5617b7c3 100755 --- a/tests/test_edisgo.py +++ b/tests/test_edisgo.py @@ -18,6 +18,7 @@ from edisgo.edisgo import import_edisgo_from_files from edisgo.flex_opt.reinforce_grid import enhanced_reinforce_grid from edisgo.network.results import Results +from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex class TestEDisGo: @@ -164,7 +165,60 @@ def test_set_time_series_manual(self, caplog): storage_units_ts, self.edisgo.timeseries.storage_units_reactive_power ) - def test_set_time_series_active_power_predefined_demandlib_auto_sets_timeindex(self): + def test_set_time_series_manual_raises_on_missing_timesteps(self): + """ + Regression test for eDisGo#703: set_time_series_manual used to + silently accept a DataFrame missing time steps required by the + active timeindex. It must now raise ValueError instead. + """ + timeindex = pd.date_range("1/1/2018", periods=3, freq="H") + self.edisgo.set_timeindex(timeindex) + + # only 2 of the 3 required time steps + incomplete_ts = pd.DataFrame( + data={"GeneratorFluctuating_15": [2.0, 5.0]}, + index=timeindex[:2], + ) + + with pytest.raises(ValueError, match="generators_p"): + self.edisgo.set_time_series_manual(generators_p=incomplete_ts) + + def test_set_time_series_manual_exempts_zero_column_dataframe(self): + """ + A DataFrame with no columns writes nothing, so it must be exempt + from the timeindex-coverage check even if its (empty) column + selection would otherwise be checked against a mismatched index. + Mirrors a real eGo call site that passes such a DataFrame as a + no-op placeholder. + """ + timeindex = pd.date_range("1/1/2018", periods=3, freq="H") + self.edisgo.set_timeindex(timeindex) + + empty_cols_ts = pd.DataFrame(index=pd.date_range("1/1/1970", periods=1)) + + # must not raise + self.edisgo.set_time_series_manual(generators_q=empty_cols_ts) + + def test_set_time_series_manual_allows_covering_superset(self): + """ + A DataFrame covering the active timeindex (even as a superset with + extra time steps outside it) must still be accepted. + """ + timeindex = pd.date_range("1/1/2018", periods=3, freq="H") + self.edisgo.set_timeindex(timeindex) + + wider_timeindex = pd.date_range("1/1/2018", periods=5, freq="H") + wider_ts = pd.DataFrame( + data={"GeneratorFluctuating_15": [2.0, 5.0, 6.0, 7.0, 8.0]}, + index=wider_timeindex, + ) + + # must not raise + self.edisgo.set_time_series_manual(generators_p=wider_ts) + + def test_set_time_series_active_power_predefined_demandlib_auto_sets_timeindex( + self, + ): edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) # Ensure timeindex is empty initially assert edisgo.timeseries.timeindex.empty @@ -219,7 +273,9 @@ def test_set_time_series_active_power_predefined(self, caplog): # check warning self.edisgo.set_time_series_active_power_predefined() - assert "No timeindex was set. TimeSeries.timeindex is automatically" in caplog.text + assert ( + "No timeindex was set. TimeSeries.timeindex is automatically" in caplog.text + ) # check if right functions are called timeindex = pd.date_range("1/1/2011 12:00", periods=2, freq="H") @@ -422,7 +478,8 @@ def test_generator_import(self): except Exception as e: if "Table does not exist" in str(e) or "HTTP 404" in str(e): pytest.skip( - "Database table not accessible (requires external database connection)" + "Database table not accessible (requires external database " + "connection)" ) else: raise @@ -551,7 +608,7 @@ def test_reinforce_catch_convergence(self): ) assert results.unresolved_issues.empty assert len(results.grid_expansion_costs) == 134 - assert len(results.equipment_changes) == 230 + assert len(results.equipment_changes) == 236 assert results.v_res.shape == (4, 142) # ############### test with catch convergence worst case true ################ @@ -562,7 +619,7 @@ def test_reinforce_catch_convergence(self): results = self.edisgo.reinforce(catch_convergence_problems=True) assert results.unresolved_issues.empty assert len(results.grid_expansion_costs) == 134 - assert len(results.equipment_changes) == 218 + assert len(results.equipment_changes) == 223 assert results.v_res.shape == (4, 142) @pytest.mark.slow @@ -583,8 +640,8 @@ def test_enhanced_reinforce_grid(self): results = edisgo_obj.results - assert len(results.grid_expansion_costs) == 454 - assert len(results.equipment_changes) == 892 + assert len(results.grid_expansion_costs) == 460 + assert len(results.equipment_changes) == 935 assert results.v_res.shape == (4, 148) edisgo_obj = copy.deepcopy(self.edisgo) @@ -1999,9 +2056,43 @@ def test_resample_timeseries(self): }, index=pd.date_range("1/1/2011 12:00", periods=2, freq="H"), ) + # regression test for eDisGo#703: DSM data used to be silently left + # at its original frequency by resample_timeseries + self.edisgo.dsm.p_max = pd.DataFrame( + data={ + "load_1": [5.0, 6.0], + "load_2": [7.0, 8.0], + }, + index=pd.date_range("1/1/2011 12:00", periods=2, freq="H"), + ) self.edisgo.resample_timeseries(freq="30min") assert len(self.edisgo.timeseries.loads_active_power) == 8 assert len(self.edisgo.heat_pump.cop_df) == 4 + assert len(self.edisgo.dsm.p_max) == 4 + + def test_reduce_timeseries_data_to_given_timeindex(self): + """ + EDisGo.reduce_timeseries_data_to_given_timeindex is a thin wrapper + around edisgo.tools.tools.reduce_timeseries_data_to_given_timeindex, + added for discoverability (eDisGo#703 checklist item 7). Must produce + the same result as calling the free function directly. + """ + self.setup_worst_case_time_series() + target_timeindex = self.edisgo.timeseries.timeindex[:2] + + edisgo_via_method = deepcopy(self.edisgo) + edisgo_via_method.reduce_timeseries_data_to_given_timeindex(target_timeindex) + + edisgo_via_function = deepcopy(self.edisgo) + reduce_timeseries_data_to_given_timeindex(edisgo_via_function, target_timeindex) + + assert_frame_equal( + edisgo_via_method.timeseries.loads_active_power, + edisgo_via_function.timeseries.loads_active_power, + ) + pd.testing.assert_index_equal( + edisgo_via_method.timeseries.timeindex, target_timeindex + ) class TestEDisGoFunc: diff --git a/tests/tools/test_spatial_complexity_reduction.py b/tests/tools/test_spatial_complexity_reduction.py index 1747e85ae..425255dee 100644 --- a/tests/tools/test_spatial_complexity_reduction.py +++ b/tests/tools/test_spatial_complexity_reduction.py @@ -3,6 +3,7 @@ from contextlib import nullcontext as does_not_raise import numpy as np +import pandas as pd import pytest from edisgo import EDisGo @@ -396,3 +397,254 @@ def test_remove_short_end_lines(self, test_edisgo_obj): # assert len(edisgo_root.topology.lines_df) - 1 == len( # edisgo_clean.topology.lines_df # ) + + +class TestApplyReducedResultsToFullGrid: + """ + Tests for + :func:`~.tools.spatial_complexity_reduction.apply_reduced_results_to_full_grid`. + + Uses stub OPF results (directly writing to + ``reduced_grid.timeseries._loads_active_power`` / + ``_storage_units_active_power``) rather than running a real OPF, since + what is under test is the map-back/disaggregation logic, not + ``pm_optimize`` itself. + """ + + @pytest.fixture(autouse=True) + def test_edisgo_obj(self): + edisgo_root = EDisGo(ding0_grid=pytest.ding0_test_network_path) + edisgo_root.set_time_series_worst_case_analysis() + make_pseudo_coordinates(edisgo_root) + return edisgo_root + + @pytest.fixture + def full_and_reduced(self, test_edisgo_obj): + full_grid = copy.deepcopy(test_edisgo_obj) + reduced_grid, _, _ = full_grid.spatial_complexity_reduction( + copy_edisgo=True, + mode="kmeansdijkstra", + cluster_area="feeder", + reduction_factor=0.1, + aggregation_mode=True, + load_aggregation_mode="bus", + ) + return full_grid, reduced_grid + + def _first_representative_with(self, reduced_grid, n_members): + loads_df = reduced_grid.topology.loads_df + candidates = loads_df[ + loads_df["old_name"].apply( + lambda v: isinstance(v, list) and len(v) == n_members + ) + ] + assert not candidates.empty, ( + f"fixture grid has no aggregated load representative with " + f"exactly {n_members} old_name member(s); adjust the fixture " + f"or reduction_factor." + ) + return candidates.index[0] + + def test_by_name_write_back_aggregation_mode_false(self, test_edisgo_obj): + # aggregation_mode=False: no merging, so restore is a plain by-name + # write-back for every flexibility type. + full_grid = copy.deepcopy(test_edisgo_obj) + reduced_grid, _, _ = full_grid.spatial_complexity_reduction( + copy_edisgo=True, + mode="kmeansdijkstra", + cluster_area="feeder", + reduction_factor=0.1, + aggregation_mode=False, + ) + ti = full_grid.timeseries.timeindex + load_name = reduced_grid.topology.loads_df.index[0] + storage_name = reduced_grid.topology.storage_units_df.index[0] + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[load_name]) + + reduced_grid.timeseries._loads_active_power.loc[ti, load_name] = [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + reduced_grid.timeseries._storage_units_active_power.loc[ti, storage_name] = [ + 5.0, + 6.0, + 7.0, + 8.0, + ] + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, + reduced_grid=reduced_grid, + flexible_loads=[load_name], + flexible_storage_units=[storage_name], + ) + + assert result.timeseries.loads_active_power.loc[ti, load_name].tolist() == [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + assert result.timeseries.storage_units_active_power.loc[ + ti, storage_name + ].tolist() == [5.0, 6.0, 7.0, 8.0] + + def test_accepts_numpy_array_flexible_component_lists(self, test_edisgo_obj): + # Regression test: task_optimize derives flexible_loads as + # edisgo.dsm.p_min.columns.values (a numpy array), unlike the other + # three flexible_* lists which are built with .tolist(). "x or []" + # raises ValueError ("truth value of an array... is ambiguous") for + # any such array with more than one element - a real crash hit on + # the first end-to-end pipeline run using aggregation_mode=False. + full_grid = copy.deepcopy(test_edisgo_obj) + reduced_grid, _, _ = full_grid.spatial_complexity_reduction( + copy_edisgo=True, + mode="kmeansdijkstra", + cluster_area="feeder", + reduction_factor=0.1, + aggregation_mode=False, + ) + ti = full_grid.timeseries.timeindex + load_name = reduced_grid.topology.loads_df.index[0] + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[load_name]) + reduced_grid.timeseries._loads_active_power.loc[ti, load_name] = [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, + reduced_grid=reduced_grid, + flexible_loads=np.array([load_name]), + ) + + assert result.timeseries.loads_active_power.loc[ti, load_name].tolist() == [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + + def test_disaggregation_multi_member_sums_to_representative(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=2) + members = reduced_grid.topology.loads_df.at[rep, "old_name"] + + p_max = pd.DataFrame(0.0, index=ti, columns=members) + for i, member in enumerate(members): + p_max[member] = [0.1 * (i + 1), 0.0, 0.2 * (i + 1), 0.05 * (i + 1)] + full_grid.dsm.p_max = p_max + + rep_power = pd.Series([1.0, 2.0, 0.0, 3.0], index=ti) + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = rep_power.values + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + total = result.timeseries.loads_active_power.loc[ti, members].sum(axis=1) + assert np.allclose(total.values, rep_power.values) + # Member with zero envelope at t0/t2/t3 gets none of the dispatch; + # both members zero at t1 falls back to an equal split. + assert result.timeseries.loads_active_power.at[ + ti[1], members[0] + ] == pytest.approx(rep_power.iloc[1] / 2) + + def test_disaggregation_singleton_renamed_representative(self, full_and_reduced): + # Regression test: under aggregation_mode=True, spatial_complexity_ + # reduction renames every group's representative, including + # singleton groups (a bus with exactly one flexible load of a given + # type/sector) - so the representative's name can differ from its + # one old_name member's name. A by-name write-back using the + # representative's name would silently miss the real target column + # on full_grid (which only has the original, un-renamed name) and + # create a phantom column instead - this must not happen. + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=1) + member = reduced_grid.topology.loads_df.at[rep, "old_name"][0] + assert rep != member, "fixture assumption: representative was renamed" + + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[member]) + rep_power = pd.Series([7.0, 8.0, 9.0, 10.0], index=ti) + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = rep_power.values + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + assert result.timeseries.loads_active_power.loc[ti, member].tolist() == ( + rep_power.tolist() + ) + assert rep not in result.timeseries.loads_active_power.columns + + def test_raises_clear_error_on_time_index_mismatch(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=1) + member = reduced_grid.topology.loads_df.at[rep, "old_name"][0] + + # dsm.p_max missing the last time step of full_grid's active index. + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti[:-1], columns=[member]) + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + + with pytest.raises(ValueError, match="does not cover"): + spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + def test_storage_units_never_aggregated_always_by_name(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + storage_name = full_grid.topology.storage_units_df.index[0] + assert storage_name in reduced_grid.topology.storage_units_df.index + assert "old_name" not in reduced_grid.topology.storage_units_df.columns + + reduced_grid.timeseries._storage_units_active_power.loc[ti, storage_name] = [ + 1.0, + 1.0, + 1.0, + 1.0, + ] + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, + reduced_grid=reduced_grid, + flexible_storage_units=[storage_name], + ) + assert result.timeseries.storage_units_active_power.loc[ + ti, storage_name + ].tolist() == [1.0, 1.0, 1.0, 1.0] + + def test_reactive_power_recomputed_after_restore(self, full_and_reduced): + full_grid, reduced_grid = full_and_reduced + ti = full_grid.timeseries.timeindex + rep = self._first_representative_with(reduced_grid, n_members=1) + member = reduced_grid.topology.loads_df.at[rep, "old_name"][0] + full_grid.dsm.p_max = pd.DataFrame(1.0, index=ti, columns=[member]) + + reactive_before = full_grid.timeseries.loads_reactive_power.loc[ + ti, member + ].copy() + reduced_grid.timeseries._loads_active_power.loc[ti, rep] = [ + 50.0, + 50.0, + 50.0, + 50.0, + ] + + result = spatial_complexity_reduction.apply_reduced_results_to_full_grid( + full_grid=full_grid, reduced_grid=reduced_grid, flexible_loads=[rep] + ) + + reactive_after = result.timeseries.loads_reactive_power.loc[ti, member] + assert not reactive_after.equals(reactive_before) diff --git a/tests/tools/test_temporal_complexity_reduction.py b/tests/tools/test_temporal_complexity_reduction.py index 1063093f4..4c32daa01 100644 --- a/tests/tools/test_temporal_complexity_reduction.py +++ b/tests/tools/test_temporal_complexity_reduction.py @@ -183,3 +183,92 @@ def test_get_most_critical_time_intervals(self): steps.loc[0, "time_steps_voltage_issues"] == pd.date_range("1/1/2018", periods=24, freq="H") ).all() + + +class TestIntervalHelpers: + """Relocated pure helpers + residual_load selection (no DB / no power flow).""" + + @staticmethod + def _week(start): + return pd.date_range(start=start, periods=168, freq="h") + + def test_intervals_overlap(self): + a = self._week("2035-01-01") + assert temp_red.intervals_overlap(a, self._week("2035-01-04")) + assert not temp_red.intervals_overlap(a, self._week("2035-06-01")) + + def test_select_two_intervals_disjoint(self): + result = temp_red.select_two_intervals( + [self._week("2035-01-01")], [self._week("2035-06-01")] + ) + assert len(result) == 2 + assert not temp_red.intervals_overlap(result[0], result[1]) + + def test_select_two_intervals_next_ranked(self): + load = [self._week("2035-01-01")] + volt = [self._week("2035-01-03"), self._week("2035-09-01")] + result = temp_red.select_two_intervals(load, volt) + assert len(result) == 2 and result[1].equals(volt[1]) + + def test_select_two_intervals_concatenate(self): + result = temp_red.select_two_intervals( + [self._week("2035-01-01")], [self._week("2035-01-03")] + ) + assert len(result) == 1 + assert (result[0][1:] - result[0][:-1]).nunique() == 1 # contiguous + + def test_select_two_intervals_single_and_empty(self): + week = self._week("2035-01-01") + assert temp_red.select_two_intervals([week], [])[0].equals(week) + assert temp_red.select_two_intervals([], [week])[0].equals(week) + assert temp_red.select_two_intervals([], []) == [] + + def test_build_centered_interval(self): + idx = pd.date_range("2035-01-01 00:00", periods=8760, freq="h") + t = pd.Timestamp("2035-02-10 12:00") + iv = temp_red._build_centered_interval(t, idx, 168, 4) + assert len(iv) == 168 + assert iv[0].hour == 4 # starts on the day-start hour + assert t in iv # critical step contained + assert iv[-1] != t # centered -> not the last step + + def test_residual_load_steps_and_intervals(self, monkeypatch): + import types + + idx = pd.date_range("2035-01-01 00:00", periods=8760, freq="h") + residual = pd.Series(range(8760), index=idx, dtype=float) + fake = types.SimpleNamespace( + timeseries=types.SimpleNamespace(residual_load=residual) + ) + monkeypatch.setattr( + "edisgo.network.overlying_grid.distribute_overlying_grid_requirements", + lambda e: fake, + ) + # steps: top-3 highest + bottom-2 lowest residual + steps = temp_red.get_most_critical_time_steps( + object(), by="residual_load", num_steps_loading=3, num_steps_voltage=2 + ) + assert idx[-1] in steps and idx[0] in steps and len(steps) == 5 + + # intervals: per-case columns, centered on the residual max/min steps + e = types.SimpleNamespace( + timeseries=types.SimpleNamespace(timeindex=idx), + topology=types.SimpleNamespace(id="g"), + ) + df = temp_red.get_most_critical_time_intervals( + e, + by="residual_load", + num_time_intervals=2, + time_steps_per_time_interval=168, + time_step_day_start=4, + ) + assert list(df.columns) == ["time_steps_load_case", "time_steps_feedin_case"] + assert len(df) == 2 + # top load-case interval is centered on the global max residual step + assert idx[-1] in df.loc[0, "time_steps_load_case"] + + def test_bad_by_raises(self): + with pytest.raises(ValueError, match="power_flow.*residual_load"): + temp_red.get_most_critical_time_steps(object(), by="bogus") + with pytest.raises(ValueError, match="power_flow.*residual_load"): + temp_red.get_most_critical_time_intervals(object(), by="bogus")