From eb3d7eb1100d65e0c88a4cb5e15087b5424a852e Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Mon, 17 Aug 2026 15:10:13 +1000 Subject: [PATCH 01/12] Apply per-timeslice link limits as per-snapshot p_max_pu/p_min_pu series The new-format translator emits link limits per timeslice, in per-unit form, in a link_timeslice_limits table rather than as per-link series. Expanding them into per-snapshot p_max_pu / p_min_pu here at network build time, via the timeslice_snapshots mapping, keeps the pypsa-friendly directory small and reuses the same mapping that scopes the custom constraints. Each (link, attribute) series is seeded from the timeslice = NaN fallback row and the named timeslices are written over it. Seeding from the fallback rather than the links table's static value matters: since #126 the translator sets p_nom = max(forward, reverse) and ships p_max_pu=1.0 / p_min_pu=0.0 only as placeholders, so a series that fell back to them would over-permit forward flow and disable reverse flow at every snapshot no named timeslice covers. Dark until the orchestrator wiring lands: _add_links_to_network keeps its two-argument form for the current path. Co-Authored-By: Claude Fable 5 --- src/ispypsa/pypsa_build/links.py | 130 ++++++++++++- .../test_add_links_with_timeslice_limits.py | 180 ++++++++++++++++++ 2 files changed, 308 insertions(+), 2 deletions(-) create mode 100644 tests/test_model/test_add_links_with_timeslice_limits.py diff --git a/src/ispypsa/pypsa_build/links.py b/src/ispypsa/pypsa_build/links.py index 36652bde..498d061d 100644 --- a/src/ispypsa/pypsa_build/links.py +++ b/src/ispypsa/pypsa_build/links.py @@ -2,15 +2,141 @@ import pypsa -def _add_links_to_network(network: pypsa.Network, links: pd.DataFrame) -> None: +def _add_links_to_network( + network: pypsa.Network, + links: pd.DataFrame, + link_timeslice_limits: pd.DataFrame | None = None, + timeslice_snapshots: pd.DataFrame | None = None, +) -> None: """Adds the Links defined in a pypsa-friendly input table called `"links"` to the `pypsa.Network` object. + When the new-format per-timeslice limit tables are given, links with + per-timeslice limits get per-snapshot p_max_pu / p_min_pu series in place + of the static values in the links table (see _build_link_pu_overrides). + Args: network: The `pypsa.Network` object links: `pd.DataFrame` with `PyPSA` style `Link` attributes. + link_timeslice_limits: `pd.DataFrame` with per-timeslice per-unit + limits (columns name, attribute, timeslice, value), or None when + all link limits are static. + timeslice_snapshots: `pd.DataFrame` mapping timeslice_ids to the + snapshots they are active at (columns timeslice_id, + investment_periods, snapshots). Required when + link_timeslice_limits is given. Returns: None """ + pu_overrides = _build_link_pu_overrides( + link_timeslice_limits, timeslice_snapshots, links, network.snapshots + ) links["class_name"] = "Link" - links.apply(lambda row: network.add(**row.to_dict()), axis=1) + for _, row in links.iterrows(): + network.add(**(row.to_dict() | pu_overrides.get(row["name"], {}))) + + +def _build_link_pu_overrides( + link_timeslice_limits: pd.DataFrame | None, + timeslice_snapshots: pd.DataFrame | None, + links: pd.DataFrame, + snapshots: pd.MultiIndex, +) -> dict[str, dict[str, pd.Series]]: + """Expands each link's per-timeslice limits into per-snapshot series. + + The translator emits two kinds of limit row per (link, attribute): rows + with a named timeslice, which apply at the snapshots that timeslice is + active, and a row with timeslice = NaN, which is the fallback for every + snapshot no named timeslice covers (the coverage contract is + Open-ISP/ISPyPSA#123). Each series is seeded with the fallback and the + named timeslices are then written over it, so a snapshot's value is its + named timeslice's limit if it has one and the fallback otherwise. + + The links table's static p_max_pu / p_min_pu are placeholders the + translator sets so the columns exist; they only remain in effect at + snapshots the limit rows leave uncovered, which the coverage contract + rules out. A named timeslice with no snapshots leaves the fallback in + place — the translator has already logged it. + + I/O Example: + link_timeslice_limits: + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 + CQ-NQ_existing p_max_pu , 1.0 # fallback + CQ-NQ_existing p_min_pu , -0.714 # fallback only + + timeslice_snapshots: qld_peak_demand active at (2025, 2025-01-13 12:00) + snapshots: (2025, 2025-01-13 12:00), (2025, 2025-01-15 12:00) + + returns: + {"CQ-NQ_existing": {"p_max_pu": series [0.857, 1.0], + "p_min_pu": series [-0.714, -0.714]}} + """ + if link_timeslice_limits is None or link_timeslice_limits.empty: + return {} + timeslice_labels = _timeslice_snapshot_labels(timeslice_snapshots) + static_values = links.set_index("name").loc[:, ["p_max_pu", "p_min_pu"]] + overrides = {} + for (name, attribute), rows in link_timeslice_limits.groupby(["name", "attribute"]): + series = pd.Series(static_values.loc[name, attribute], index=snapshots) + series = _apply_fallback_limit(series, rows) + series = _apply_named_timeslice_limits(series, rows, timeslice_labels) + overrides.setdefault(name, {})[attribute] = series + return overrides + + +def _apply_fallback_limit(series: pd.Series, rows: pd.DataFrame) -> pd.Series: + """Sets every snapshot to the timeslice = NaN fallback row's value, if there is one. + + I/O Example: + series: [1.0, 1.0] + rows: + timeslice value + qld_peak_demand 0.857 + , 0.9 # fallback + -> [0.9, 0.9] + + rows with no fallback row -> series unchanged + """ + fallback = rows.loc[rows["timeslice"].isna(), "value"] + if fallback.empty: + return series + return pd.Series(fallback.iloc[0], index=series.index) + + +def _apply_named_timeslice_limits( + series: pd.Series, rows: pd.DataFrame, timeslice_labels: dict[str, list[tuple]] +) -> pd.Series: + """Writes each named timeslice's value at the snapshots it is active. + + I/O Example: + series: [0.9, 0.9, 0.9] (snapshots s0, s1, s2) + rows: + timeslice value + qld_peak_demand 0.857 + , 0.9 # fallback rows are skipped + timeslice_labels: {"qld_peak_demand": [s1, s2]} + -> [0.9, 0.857, 0.857] + """ + series = series.copy() + for row in rows.loc[rows["timeslice"].notna()].itertuples(): + series.loc[timeslice_labels.get(row.timeslice, [])] = row.value + return series + + +def _timeslice_snapshot_labels( + timeslice_snapshots: pd.DataFrame, +) -> dict[str, list[tuple]]: + """The (investment_period, snapshot) labels each timeslice is active at. + + I/O Example: + timeslice_id=qld_peak_demand, investment_periods=2025, + snapshots=2025-01-13 12:00 + -> {"qld_peak_demand": [(2025, Timestamp("2025-01-13 12:00"))]} + """ + mapping = timeslice_snapshots.copy() + mapping["snapshots"] = pd.to_datetime(mapping["snapshots"]) + return { + timeslice_id: list(zip(rows["investment_periods"], rows["snapshots"])) + for timeslice_id, rows in mapping.groupby("timeslice_id") + } diff --git a/tests/test_model/test_add_links_with_timeslice_limits.py b/tests/test_model/test_add_links_with_timeslice_limits.py new file mode 100644 index 00000000..2261e052 --- /dev/null +++ b/tests/test_model/test_add_links_with_timeslice_limits.py @@ -0,0 +1,180 @@ +import pandas as pd +import pypsa + +from ispypsa.pypsa_build.links import _add_links_to_network + + +def _network() -> pypsa.Network: + snapshots = pd.date_range("2025-01-01", periods=4, freq="h") + index = pd.MultiIndex.from_arrays([[2025] * 4, list(snapshots)]) + network = pypsa.Network(snapshots=index, investment_periods=[2025]) + network.add("Bus", "bus1") + network.add("Bus", "bus2") + return network + + +def _links(csv_str_to_df) -> pd.DataFrame: + # p_max_pu / p_min_pu are the translator's placeholders; the real limits + # come from link_timeslice_limits. + return csv_str_to_df(""" + name, bus0, bus1, carrier, p_nom, p_max_pu, p_min_pu, p_nom_extendable + CQ-NQ_existing, bus1, bus2, AC, 1400, 1.0, 0.0, False + """) + + +def _link_pu_limits(network: pypsa.Network, name: str) -> pd.DataFrame: + """The per-snapshot p_max_pu / p_min_pu of one link, one row per snapshot.""" + limits = pd.DataFrame( + { + "p_max_pu": network.links_t.p_max_pu[name], + "p_min_pu": network.links_t.p_min_pu[name], + } + ) + limits.index = limits.index.set_names(["investment_periods", "snapshots"]) + return limits.reset_index() + + +def test_named_timeslices_overlay_the_fallback(csv_str_to_df): + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857 + CQ-NQ_existing, p_max_pu, , 1.0 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -0.9 + CQ-NQ_existing, p_min_pu, , -0.714 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + qld_peak_demand, 2025, 2025-01-01 01:00:00 + qld_peak_demand, 2025, 2025-01-01 02:00:00 + """) + + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + expected = csv_str_to_df(""" + investment_periods, snapshots, p_max_pu, p_min_pu + 2025, 2025-01-01 00:00:00, 1.0, -0.714 + 2025, 2025-01-01 01:00:00, 0.857, -0.9 + 2025, 2025-01-01 02:00:00, 0.857, -0.9 + 2025, 2025-01-01 03:00:00, 1.0, -0.714 + """) + expected["snapshots"] = pd.to_datetime(expected["snapshots"]) + pd.testing.assert_frame_equal(_link_pu_limits(network, "CQ-NQ_existing"), expected) + + +def test_fallback_only_attribute_gets_the_fallback_at_every_snapshot(csv_str_to_df): + # The static p_min_pu placeholder (0.0) must not survive at uncovered + # snapshots — that would silently disable reverse flow. + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857 + CQ-NQ_existing, p_max_pu, , 1.0 + CQ-NQ_existing, p_min_pu, , -0.714 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + qld_peak_demand, 2025, 2025-01-01 01:00:00 + """) + + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + expected = csv_str_to_df(""" + investment_periods, snapshots, p_max_pu, p_min_pu + 2025, 2025-01-01 00:00:00, 1.0, -0.714 + 2025, 2025-01-01 01:00:00, 0.857, -0.714 + 2025, 2025-01-01 02:00:00, 1.0, -0.714 + 2025, 2025-01-01 03:00:00, 1.0, -0.714 + """) + expected["snapshots"] = pd.to_datetime(expected["snapshots"]) + pd.testing.assert_frame_equal(_link_pu_limits(network, "CQ-NQ_existing"), expected) + + +def test_named_timeslices_that_tile_the_snapshots_need_no_fallback(csv_str_to_df): + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857 + CQ-NQ_existing, p_max_pu, qld_winter_reference, 1.0 + CQ-NQ_existing, p_min_pu, , -0.714 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + qld_winter_reference, 2025, 2025-01-01 00:00:00 + qld_peak_demand, 2025, 2025-01-01 01:00:00 + qld_peak_demand, 2025, 2025-01-01 02:00:00 + qld_winter_reference, 2025, 2025-01-01 03:00:00 + """) + + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + expected = csv_str_to_df(""" + investment_periods, snapshots, p_max_pu, p_min_pu + 2025, 2025-01-01 00:00:00, 1.0, -0.714 + 2025, 2025-01-01 01:00:00, 0.857, -0.714 + 2025, 2025-01-01 02:00:00, 0.857, -0.714 + 2025, 2025-01-01 03:00:00, 1.0, -0.714 + """) + expected["snapshots"] = pd.to_datetime(expected["snapshots"]) + pd.testing.assert_frame_equal(_link_pu_limits(network, "CQ-NQ_existing"), expected) + + +def test_named_timeslice_with_no_snapshots_leaves_the_fallback(csv_str_to_df): + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857 + CQ-NQ_existing, p_max_pu, , 1.0 + CQ-NQ_existing, p_min_pu, , -0.714 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + """) + + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + expected = csv_str_to_df(""" + investment_periods, snapshots, p_max_pu, p_min_pu + 2025, 2025-01-01 00:00:00, 1.0, -0.714 + 2025, 2025-01-01 01:00:00, 1.0, -0.714 + 2025, 2025-01-01 02:00:00, 1.0, -0.714 + 2025, 2025-01-01 03:00:00, 1.0, -0.714 + """) + expected["snapshots"] = pd.to_datetime(expected["snapshots"]) + pd.testing.assert_frame_equal(_link_pu_limits(network, "CQ-NQ_existing"), expected) + + +def test_links_without_timeslice_limits_keep_their_static_values(csv_str_to_df): + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + """) + + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + assert "CQ-NQ_existing" not in network.links_t.p_max_pu.columns + assert "CQ-NQ_existing" not in network.links_t.p_min_pu.columns + assert network.links.loc["CQ-NQ_existing", "p_max_pu"] == 1.0 + assert network.links.loc["CQ-NQ_existing", "p_min_pu"] == 0.0 + + +def test_old_format_call_without_limit_tables(csv_str_to_df): + network = _network() + + _add_links_to_network(network, _links(csv_str_to_df)) + + assert network.links.loc["CQ-NQ_existing", "p_nom"] == 1400 + assert "CQ-NQ_existing" not in network.links_t.p_max_pu.columns From b4c6d8174970c93c6c90a85e51687f754563063a Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Mon, 17 Aug 2026 16:51:50 +1000 Subject: [PATCH 02/12] State when the link limit tables are required, and flag the None handling for flag retirement "None when all link limits are static" read as a per-run choice, but on the new-format path the translator always emits limit rows and the links table's p_max_pu / p_min_pu are placeholders, so omitting the table there would silently mis-model every link. Spell out the two paths and mark the None handling with the existing FEATURE_FLAG_CLEANUP convention. Co-Authored-By: Claude Fable 5 --- src/ispypsa/pypsa_build/links.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/ispypsa/pypsa_build/links.py b/src/ispypsa/pypsa_build/links.py index 498d061d..a7a3cca9 100644 --- a/src/ispypsa/pypsa_build/links.py +++ b/src/ispypsa/pypsa_build/links.py @@ -11,19 +11,27 @@ def _add_links_to_network( """Adds the Links defined in a pypsa-friendly input table called `"links"` to the `pypsa.Network` object. - When the new-format per-timeslice limit tables are given, links with - per-timeslice limits get per-snapshot p_max_pu / p_min_pu series in place - of the static values in the links table (see _build_link_pu_overrides). + On the new-format path the two limit tables are required: each link's + per-timeslice limits are expanded into per-snapshot p_max_pu / p_min_pu + series that replace the links table's placeholder values (see + _build_link_pu_overrides). On the old-format path both are omitted and + the links table's p_max_pu / p_min_pu are the limits. + + FEATURE_FLAG_CLEANUP[use_new_table_format]: once the flag is retired, + make link_timeslice_limits and timeslice_snapshots required and drop the + None handling in _build_link_pu_overrides. Args: network: The `pypsa.Network` object links: `pd.DataFrame` with `PyPSA` style `Link` attributes. link_timeslice_limits: `pd.DataFrame` with per-timeslice per-unit - limits (columns name, attribute, timeslice, value), or None when - all link limits are static. + limits (columns name, attribute, timeslice, value). Required on + the new-format path, where the links table's p_max_pu / p_min_pu + are placeholders it overrides at every snapshot; omitted on the + old-format path. timeslice_snapshots: `pd.DataFrame` mapping timeslice_ids to the snapshots they are active at (columns timeslice_id, - investment_periods, snapshots). Required when + investment_periods, snapshots). Required whenever link_timeslice_limits is given. Returns: None From 0d1cf7905b539ec1ed6c74d668c8a6cf8318159f Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 09:46:06 +1000 Subject: [PATCH 03/12] Show _add_links_to_network as an I/O example instead of an Args list The Args prose restated what the example can show directly: what each table looks like, and that the function returns None because the network is modified in place. Co-Authored-By: Claude Fable 5 --- src/ispypsa/pypsa_build/links.py | 40 +++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/ispypsa/pypsa_build/links.py b/src/ispypsa/pypsa_build/links.py index a7a3cca9..965bcaad 100644 --- a/src/ispypsa/pypsa_build/links.py +++ b/src/ispypsa/pypsa_build/links.py @@ -21,20 +21,32 @@ def _add_links_to_network( make link_timeslice_limits and timeslice_snapshots required and drop the None handling in _build_link_pu_overrides. - Args: - network: The `pypsa.Network` object - links: `pd.DataFrame` with `PyPSA` style `Link` attributes. - link_timeslice_limits: `pd.DataFrame` with per-timeslice per-unit - limits (columns name, attribute, timeslice, value). Required on - the new-format path, where the links table's p_max_pu / p_min_pu - are placeholders it overrides at every snapshot; omitted on the - old-format path. - timeslice_snapshots: `pd.DataFrame` mapping timeslice_ids to the - snapshots they are active at (columns timeslice_id, - investment_periods, snapshots). Required whenever - link_timeslice_limits is given. - - Returns: None + I/O Example (new-format path): + links (the p_max_pu / p_min_pu here are placeholders): + name bus0 bus1 carrier p_nom p_max_pu p_min_pu p_nom_extendable + CQ-NQ_existing CQ NQ AC 1400 1.0 0.0 False + + link_timeslice_limits: + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 + CQ-NQ_existing p_max_pu , 1.0 # fallback + CQ-NQ_existing p_min_pu , -0.714 # fallback only + + timeslice_snapshots: + timeslice_id investment_periods snapshots + qld_peak_demand 2025 2025-01-13 12:00 + + network.snapshots: (2025, 2025-01-13 12:00), (2025, 2025-01-15 12:00) + + returns None; network is modified in place — it gains the Link + CQ-NQ_existing with p_nom = 1400 and the per-snapshot series + network.links_t.p_max_pu["CQ-NQ_existing"] = [0.857, 1.0] + network.links_t.p_min_pu["CQ-NQ_existing"] = [-0.714, -0.714] + + I/O Example (old-format path): + _add_links_to_network(network, links) + returns None; network gains the Link with the links table's static + p_max_pu / p_min_pu and no per-snapshot series. """ pu_overrides = _build_link_pu_overrides( link_timeslice_limits, timeslice_snapshots, links, network.snapshots From 83b7c1a8895e5ea3cdf9570efacae37bc4749847 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 09:51:48 +1000 Subject: [PATCH 04/12] Lay out the _build_link_pu_overrides example in the same table form as its caller Co-Authored-By: Claude Fable 5 --- src/ispypsa/pypsa_build/links.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ispypsa/pypsa_build/links.py b/src/ispypsa/pypsa_build/links.py index 965bcaad..b7cbdb28 100644 --- a/src/ispypsa/pypsa_build/links.py +++ b/src/ispypsa/pypsa_build/links.py @@ -85,12 +85,20 @@ def _build_link_pu_overrides( CQ-NQ_existing p_max_pu , 1.0 # fallback CQ-NQ_existing p_min_pu , -0.714 # fallback only - timeslice_snapshots: qld_peak_demand active at (2025, 2025-01-13 12:00) + timeslice_snapshots: + timeslice_id investment_periods snapshots + qld_peak_demand 2025 2025-01-13 12:00 + + links (the p_max_pu / p_min_pu here are placeholders): + name p_max_pu p_min_pu + CQ-NQ_existing 1.0 0.0 + snapshots: (2025, 2025-01-13 12:00), (2025, 2025-01-15 12:00) returns: {"CQ-NQ_existing": {"p_max_pu": series [0.857, 1.0], "p_min_pu": series [-0.714, -0.714]}} + (each series indexed by snapshots) """ if link_timeslice_limits is None or link_timeslice_limits.empty: return {} From 3a50e0fac43233cf570fa4c83db3d3d1f3f97a27 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 11:20:56 +1000 Subject: [PATCH 05/12] Give zero-capacity corridors per-unit limits of 0 instead of skipping them Filtering zero-p_nom links out of link_timeslice_limits made them the one class of link pypsa_build had to know about: their placeholder p_max_pu / p_min_pu did real work, and every consumer needed a caveat for links with no limit rows. Defining per-unit-of-zero as 0 at the point of division lets them flow through pypsa_build like any other link. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/network.py | 15 ++++++++------- tests/test_translator/test_network.py | 8 +++++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/ispypsa/translator/network.py b/src/ispypsa/translator/network.py index 66233841..454adfdb 100644 --- a/src/ispypsa/translator/network.py +++ b/src/ispypsa/translator/network.py @@ -350,9 +350,7 @@ def _translate_timeslice_limits_to_pu( Named-timeslice rows carry their timeslice; a timeslice = NaN row is the fallback applied to snapshots no named timeslice covers (the coverage contract is Open-ISP/ISPyPSA#123). pypsa_build expands these into per-snapshot - series via the timeslice_snapshots mapping. Zero-p_nom links (new parallel - corridors) are skipped — all their limits are zero and the per-unit form is - undefined. + series via the timeslice_snapshots mapping. I/O Example: limits: @@ -370,21 +368,24 @@ def _translate_timeslice_limits_to_pu( CQ-NQ_existing p_max_pu qld_peak_demand 0.857 # 1200/1400 CQ-NQ_existing p_max_pu qld_winter_reference 1.0 # 1400/1400 CQ-NQ_existing p_min_pu , -0.714 # fallback, -1000/1400 + + A zero-p_nom link with capacity 0 in a row -> value 0.0 for that row. """ rows = limits.merge( existing_links.loc[:, ["isp_name", "name", "p_nom"]], left_on="path_id", right_on="isp_name", ) - # Zero-p_nom links (new parallel corridors) have no per-unit form, so they - # contribute no timeslice rows. - rows = rows[rows["p_nom"] > 0] rows["attribute"] = rows["direction"].map( {"forward": "p_max_pu", "reverse": "p_min_pu"} ) # Reverse flow is negative, so reverse limits become negative p_min_pu values. sign = rows["direction"].map({"forward": 1.0, "reverse": -1.0}) - rows["value"] = sign * rows["capacity"] / rows["p_nom"] + # A zero-p_nom link (new parallel corridor) has only zero limits; 0/0 is + # undefined, so its per-unit value is defined as 0. + rows["value"] = np.where( + rows["p_nom"] > 0, sign * rows["capacity"] / rows["p_nom"], 0.0 + ) return rows.loc[:, _LINK_TIMESLICE_LIMIT_COLUMNS].reset_index(drop=True) diff --git a/tests/test_translator/test_network.py b/tests/test_translator/test_network.py index 90652be7..ce4eb3ad 100644 --- a/tests/test_translator/test_network.py +++ b/tests/test_translator/test_network.py @@ -493,8 +493,8 @@ def test_translate_network_to_links_zero_capacity_parallel_path( """A new parallel corridor has zero existing capacity, given as a timeslice = NaN fallback of 0 in both directions. It becomes an inert existing link at p_nom 0 (its buildable capacity is modelled by expansion - links), and the zero-p_nom link is skipped when translating per-timeslice - limits, so it yields no link_timeslice_limits rows and no 0/0 division.""" + links) whose per-timeslice limits are per-unit 0 rather than an undefined + 0/0, so pypsa_build treats it like any other link.""" ispypsa_tables = _network_tables(csv_str_to_df) ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" path_id, geo_from, geo_to, carrier @@ -521,7 +521,9 @@ def test_translate_network_to_links_zero_capacity_parallel_path( pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) expected_limits = csv_str_to_df(""" - name, attribute, timeslice, value + name, attribute, timeslice, value + CNSW-SNW_existing, p_max_pu, , 0.0 + CNSW-SNW_existing, p_min_pu, , 0.0 """) pd.testing.assert_frame_equal( link_timeslice_limits, expected_limits, check_dtype=False From 1303050d105e4a2e259c7bc14601279ba3e6bbb2 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 11:20:59 +1000 Subject: [PATCH 06/12] Build link limit series by merging onto a snapshot grid, and raise on gaps Seeding each series from the links table's placeholder p_max_pu / p_min_pu gave an uncovered snapshot a plausible value and hid the gap. The series are now built as one long table - every (link, attribute) pair at every snapshot, named-timeslice values merged in where active, the blank- timeslice fallback filled elsewhere - so a snapshot with neither is a NaN that raises with the affected links named, and the intermediate table can be asserted directly in tests. The merge form also replaces the per-pair label lookups and overwrite loop with plain pandas joins, so a named timeslice with no snapshots or a pair with no fallback row falls out of merge semantics rather than needing its own branch. Co-Authored-By: Claude Fable 5 --- src/ispypsa/pypsa_build/links.py | 278 +++++++++++++----- .../test_add_links_with_timeslice_limits.py | 68 ++++- 2 files changed, 263 insertions(+), 83 deletions(-) diff --git a/src/ispypsa/pypsa_build/links.py b/src/ispypsa/pypsa_build/links.py index b7cbdb28..9fffd382 100644 --- a/src/ispypsa/pypsa_build/links.py +++ b/src/ispypsa/pypsa_build/links.py @@ -1,6 +1,14 @@ import pandas as pd import pypsa +_LIMIT_PER_SNAPSHOT_COLUMNS = [ + "name", + "attribute", + "investment_periods", + "snapshots", + "value", +] + def _add_links_to_network( network: pypsa.Network, @@ -11,7 +19,7 @@ def _add_links_to_network( """Adds the Links defined in a pypsa-friendly input table called `"links"` to the `pypsa.Network` object. - On the new-format path the two limit tables are required: each link's + On the new-format path the two timeslice tables are required: each link's per-timeslice limits are expanded into per-snapshot p_max_pu / p_min_pu series that replace the links table's placeholder values (see _build_link_pu_overrides). On the old-format path both are omitted and @@ -36,20 +44,40 @@ def _add_links_to_network( timeslice_id investment_periods snapshots qld_peak_demand 2025 2025-01-13 12:00 - network.snapshots: (2025, 2025-01-13 12:00), (2025, 2025-01-15 12:00) + network.snapshots: + investment_periods snapshots + 2025 2025-01-13 12:00 + 2025 2025-01-15 12:00 - returns None; network is modified in place — it gains the Link - CQ-NQ_existing with p_nom = 1400 and the per-snapshot series - network.links_t.p_max_pu["CQ-NQ_existing"] = [0.857, 1.0] - network.links_t.p_min_pu["CQ-NQ_existing"] = [-0.714, -0.714] + returns None; network is modified in place. It gains the Link: + network.links: + name bus0 bus1 p_nom p_nom_extendable + CQ-NQ_existing CQ NQ 1400 False + with per-snapshot limits in place of the placeholders: + network.links_t.p_max_pu: + investment_periods snapshots CQ-NQ_existing + 2025 2025-01-13 12:00 0.857 + 2025 2025-01-15 12:00 1.0 + network.links_t.p_min_pu: + investment_periods snapshots CQ-NQ_existing + 2025 2025-01-13 12:00 -0.714 + 2025 2025-01-15 12:00 -0.714 I/O Example (old-format path): - _add_links_to_network(network, links) - returns None; network gains the Link with the links table's static - p_max_pu / p_min_pu and no per-snapshot series. + links: + name bus0 bus1 carrier p_nom p_max_pu p_min_pu p_nom_extendable + CQ-NQ_existing CQ NQ AC 1400 1.0 -0.5 False + + returns None; network is modified in place. It gains the Link with + the links table's p_max_pu / p_min_pu as static values: + network.links: + name bus0 bus1 p_nom p_max_pu p_min_pu p_nom_extendable + CQ-NQ_existing CQ NQ 1400 1.0 -0.5 False + and no per-snapshot series (network.links_t.p_max_pu / p_min_pu have + no CQ-NQ_existing column). """ pu_overrides = _build_link_pu_overrides( - link_timeslice_limits, timeslice_snapshots, links, network.snapshots + link_timeslice_limits, timeslice_snapshots, network.snapshots ) links["class_name"] = "Link" for _, row in links.iterrows(): @@ -59,7 +87,6 @@ def _add_links_to_network( def _build_link_pu_overrides( link_timeslice_limits: pd.DataFrame | None, timeslice_snapshots: pd.DataFrame | None, - links: pd.DataFrame, snapshots: pd.MultiIndex, ) -> dict[str, dict[str, pd.Series]]: """Expands each link's per-timeslice limits into per-snapshot series. @@ -67,16 +94,9 @@ def _build_link_pu_overrides( The translator emits two kinds of limit row per (link, attribute): rows with a named timeslice, which apply at the snapshots that timeslice is active, and a row with timeslice = NaN, which is the fallback for every - snapshot no named timeslice covers (the coverage contract is - Open-ISP/ISPyPSA#123). Each series is seeded with the fallback and the - named timeslices are then written over it, so a snapshot's value is its - named timeslice's limit if it has one and the fallback otherwise. - - The links table's static p_max_pu / p_min_pu are placeholders the - translator sets so the columns exist; they only remain in effect at - snapshots the limit rows leave uncovered, which the coverage contract - rules out. A named timeslice with no snapshots leaves the fallback in - place — the translator has already logged it. + snapshot no named timeslice covers. Every snapshot must end up with a + value from one or the other; a snapshot left without one raises rather + than silently keeping the links table's placeholder p_max_pu / p_min_pu. I/O Example: link_timeslice_limits: @@ -89,82 +109,180 @@ def _build_link_pu_overrides( timeslice_id investment_periods snapshots qld_peak_demand 2025 2025-01-13 12:00 - links (the p_max_pu / p_min_pu here are placeholders): - name p_max_pu p_min_pu - CQ-NQ_existing 1.0 0.0 + snapshots: + investment_periods snapshots + 2025 2025-01-13 12:00 + 2025 2025-01-15 12:00 - snapshots: (2025, 2025-01-13 12:00), (2025, 2025-01-15 12:00) + returns (each series indexed by snapshots): + {"CQ-NQ_existing": {"p_max_pu": [0.857, 1.0], + "p_min_pu": [-0.714, -0.714]}} - returns: - {"CQ-NQ_existing": {"p_max_pu": series [0.857, 1.0], - "p_min_pu": series [-0.714, -0.714]}} - (each series indexed by snapshots) + Without the p_max_pu fallback row, p_max_pu would be undefined at the + second snapshot -> ValueError. """ if link_timeslice_limits is None or link_timeslice_limits.empty: return {} - timeslice_labels = _timeslice_snapshot_labels(timeslice_snapshots) - static_values = links.set_index("name").loc[:, ["p_max_pu", "p_min_pu"]] - overrides = {} - for (name, attribute), rows in link_timeslice_limits.groupby(["name", "attribute"]): - series = pd.Series(static_values.loc[name, attribute], index=snapshots) - series = _apply_fallback_limit(series, rows) - series = _apply_named_timeslice_limits(series, rows, timeslice_labels) - overrides.setdefault(name, {})[attribute] = series - return overrides + limits_per_snapshot = _expand_limits_to_snapshots( + link_timeslice_limits, timeslice_snapshots, snapshots + ) + _raise_if_snapshots_uncovered(limits_per_snapshot) + return _series_by_link_and_attribute(limits_per_snapshot, snapshots) + + +def _expand_limits_to_snapshots( + link_timeslice_limits: pd.DataFrame, + timeslice_snapshots: pd.DataFrame, + snapshots: pd.MultiIndex, +) -> pd.DataFrame: + """One row per (link, attribute, snapshot): the named timeslice's value where + one is active there, else the fallback, else NaN. + + I/O Example: + link_timeslice_limits: + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 + CQ-NQ_existing p_max_pu , 1.0 + CQ-NQ_existing p_min_pu qld_peak_demand -0.9 # no fallback + + timeslice_snapshots: + timeslice_id investment_periods snapshots + qld_peak_demand 2025 2025-01-13 12:00 + + snapshots: + investment_periods snapshots + 2025 2025-01-13 12:00 + 2025 2025-01-15 12:00 + + returns: + name attribute investment_periods snapshots value + CQ-NQ_existing p_max_pu 2025 2025-01-13 12:00 0.857 + CQ-NQ_existing p_max_pu 2025 2025-01-15 12:00 1.0 # fallback + CQ-NQ_existing p_min_pu 2025 2025-01-13 12:00 -0.9 + CQ-NQ_existing p_min_pu 2025 2025-01-15 12:00 # uncovered + """ + is_fallback = link_timeslice_limits["timeslice"].isna() + named = _place_named_limits_at_snapshots( + link_timeslice_limits[~is_fallback], timeslice_snapshots + ) + fallback = link_timeslice_limits.loc[is_fallback, ["name", "attribute", "value"]] + grid = _link_attribute_snapshot_grid(link_timeslice_limits, snapshots) + grid = grid.merge(named, how="left") + grid = grid.merge(fallback.rename(columns={"value": "fallback"}), how="left") + grid["value"] = grid["value"].fillna(grid["fallback"]) + return grid.loc[:, _LIMIT_PER_SNAPSHOT_COLUMNS] -def _apply_fallback_limit(series: pd.Series, rows: pd.DataFrame) -> pd.Series: - """Sets every snapshot to the timeslice = NaN fallback row's value, if there is one. +def _link_attribute_snapshot_grid( + link_timeslice_limits: pd.DataFrame, snapshots: pd.MultiIndex +) -> pd.DataFrame: + """Every (link, attribute) pair in the limits table at every snapshot. I/O Example: - series: [1.0, 1.0] - rows: - timeslice value - qld_peak_demand 0.857 - , 0.9 # fallback - -> [0.9, 0.9] - - rows with no fallback row -> series unchanged + link_timeslice_limits (only name and attribute are read): + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 + CQ-NQ_existing p_max_pu , 1.0 # same pair, once in the grid + + snapshots: + investment_periods snapshots + 2025 2025-01-13 12:00 + 2025 2025-01-15 12:00 + + returns: + name attribute investment_periods snapshots + CQ-NQ_existing p_max_pu 2025 2025-01-13 12:00 + CQ-NQ_existing p_max_pu 2025 2025-01-15 12:00 + """ + pairs = link_timeslice_limits.loc[:, ["name", "attribute"]].drop_duplicates() + snapshot_rows = pd.DataFrame( + snapshots.tolist(), columns=["investment_periods", "snapshots"] + ) + return pairs.merge(snapshot_rows, how="cross") + + +def _place_named_limits_at_snapshots( + named: pd.DataFrame, timeslice_snapshots: pd.DataFrame +) -> pd.DataFrame: + """Places each named-timeslice limit at the snapshots its timeslice is active. + + A timeslice with no snapshots contributes no rows (the translator has + already logged it). + + I/O Example: + named: + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 + + timeslice_snapshots: + timeslice_id investment_periods snapshots + qld_peak_demand 2025 2025-01-13 12:00 + + returns: + name attribute investment_periods snapshots value + CQ-NQ_existing p_max_pu 2025 2025-01-13 12:00 0.857 """ - fallback = rows.loc[rows["timeslice"].isna(), "value"] - if fallback.empty: - return series - return pd.Series(fallback.iloc[0], index=series.index) + active_at = timeslice_snapshots.rename(columns={"timeslice_id": "timeslice"}) + active_at["snapshots"] = pd.to_datetime(active_at["snapshots"]) + placed = named.merge(active_at, on="timeslice") + return placed.loc[:, _LIMIT_PER_SNAPSHOT_COLUMNS] -def _apply_named_timeslice_limits( - series: pd.Series, rows: pd.DataFrame, timeslice_labels: dict[str, list[tuple]] -) -> pd.Series: - """Writes each named timeslice's value at the snapshots it is active. +def _raise_if_snapshots_uncovered(limits_per_snapshot: pd.DataFrame) -> None: + """Raises if any (link, attribute) has a snapshot with neither a named-timeslice + nor a fallback value. I/O Example: - series: [0.9, 0.9, 0.9] (snapshots s0, s1, s2) - rows: - timeslice value - qld_peak_demand 0.857 - , 0.9 # fallback rows are skipped - timeslice_labels: {"qld_peak_demand": [s1, s2]} - -> [0.9, 0.857, 0.857] + limits_per_snapshot: + name attribute investment_periods snapshots value + CQ-NQ_existing p_max_pu 2025 2025-01-13 12:00 0.857 + CQ-NQ_existing p_max_pu 2025 2025-01-15 12:00 1.0 + returns None + + limits_per_snapshot: + name attribute investment_periods snapshots value + CQ-NQ_existing p_min_pu 2025 2025-01-13 12:00 -0.9 + CQ-NQ_existing p_min_pu 2025 2025-01-15 12:00 # uncovered + raises ValueError naming (CQ-NQ_existing, p_min_pu) and the + uncovered snapshot (2025, 2025-01-15 12:00). """ - series = series.copy() - for row in rows.loc[rows["timeslice"].notna()].itertuples(): - series.loc[timeslice_labels.get(row.timeslice, [])] = row.value - return series + uncovered = limits_per_snapshot[limits_per_snapshot["value"].isna()] + if uncovered.empty: + return + pairs = sorted(set(zip(uncovered["name"], uncovered["attribute"]))) + first = uncovered.loc[:, ["investment_periods", "snapshots"]].head(5) + raise ValueError( + f"link_timeslice_limits leaves {len(uncovered)} (link, attribute, snapshot) " + f"combination(s) undefined: no fallback (blank-timeslice) row and no named " + f"timeslice active there. Affected (link, attribute): {pairs}. " + f"First uncovered snapshots:\n{first.to_string(index=False)}" + ) -def _timeslice_snapshot_labels( - timeslice_snapshots: pd.DataFrame, -) -> dict[str, list[tuple]]: - """The (investment_period, snapshot) labels each timeslice is active at. +def _series_by_link_and_attribute( + limits_per_snapshot: pd.DataFrame, snapshots: pd.MultiIndex +) -> dict[str, dict[str, pd.Series]]: + """Reshapes the long table into {link: {attribute: series indexed by snapshots}}. I/O Example: - timeslice_id=qld_peak_demand, investment_periods=2025, - snapshots=2025-01-13 12:00 - -> {"qld_peak_demand": [(2025, Timestamp("2025-01-13 12:00"))]} + limits_per_snapshot: + name attribute investment_periods snapshots value + CQ-NQ_existing p_max_pu 2025 2025-01-13 12:00 0.857 + CQ-NQ_existing p_max_pu 2025 2025-01-15 12:00 1.0 + CQ-NQ_existing p_min_pu 2025 2025-01-13 12:00 -0.714 + CQ-NQ_existing p_min_pu 2025 2025-01-15 12:00 -0.714 + + snapshots: + investment_periods snapshots + 2025 2025-01-13 12:00 + 2025 2025-01-15 12:00 + + returns (each series indexed by snapshots): + {"CQ-NQ_existing": {"p_max_pu": [0.857, 1.0], + "p_min_pu": [-0.714, -0.714]}} """ - mapping = timeslice_snapshots.copy() - mapping["snapshots"] = pd.to_datetime(mapping["snapshots"]) - return { - timeslice_id: list(zip(rows["investment_periods"], rows["snapshots"])) - for timeslice_id, rows in mapping.groupby("timeslice_id") - } + overrides = {} + for (name, attribute), rows in limits_per_snapshot.groupby(["name", "attribute"]): + series = rows.set_index(["investment_periods", "snapshots"])["value"] + overrides.setdefault(name, {})[attribute] = series.reindex(snapshots) + return overrides diff --git a/tests/test_model/test_add_links_with_timeslice_limits.py b/tests/test_model/test_add_links_with_timeslice_limits.py index 2261e052..dd20df0c 100644 --- a/tests/test_model/test_add_links_with_timeslice_limits.py +++ b/tests/test_model/test_add_links_with_timeslice_limits.py @@ -1,7 +1,11 @@ import pandas as pd import pypsa +import pytest -from ispypsa.pypsa_build.links import _add_links_to_network +from ispypsa.pypsa_build.links import ( + _add_links_to_network, + _expand_limits_to_snapshots, +) def _network() -> pypsa.Network: @@ -14,8 +18,8 @@ def _network() -> pypsa.Network: def _links(csv_str_to_df) -> pd.DataFrame: - # p_max_pu / p_min_pu are the translator's placeholders; the real limits - # come from link_timeslice_limits. + # p_max_pu / p_min_pu are the translator's placeholders, never read on the + # new-format path; the real limits come from link_timeslice_limits. return csv_str_to_df(""" name, bus0, bus1, carrier, p_nom, p_max_pu, p_min_pu, p_nom_extendable CQ-NQ_existing, bus1, bus2, AC, 1400, 1.0, 0.0, False @@ -152,6 +156,27 @@ def test_named_timeslice_with_no_snapshots_leaves_the_fallback(csv_str_to_df): pd.testing.assert_frame_equal(_link_pu_limits(network, "CQ-NQ_existing"), expected) +def test_snapshot_covered_by_neither_named_timeslice_nor_fallback_raises(csv_str_to_df): + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857 + CQ-NQ_existing, p_min_pu, , -0.714 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + qld_peak_demand, 2025, 2025-01-01 01:00:00 + """) + + with pytest.raises( + ValueError, + match=r"leaves 3 \(link, attribute, snapshot\) combination\(s\) undefined", + ): + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + def test_links_without_timeslice_limits_keep_their_static_values(csv_str_to_df): network = _network() link_timeslice_limits = csv_str_to_df(""" @@ -178,3 +203,40 @@ def test_old_format_call_without_limit_tables(csv_str_to_df): assert network.links.loc["CQ-NQ_existing", "p_nom"] == 1400 assert "CQ-NQ_existing" not in network.links_t.p_max_pu.columns + + +def test_expand_limits_to_snapshots(csv_str_to_df): + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857 + CQ-NQ_existing, p_max_pu, , 1.0 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -0.9 + NQ-CQ_other, p_max_pu, no_snapshots, 0.5 + NQ-CQ_other, p_max_pu, , 0.8 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + qld_peak_demand, 2025, 2025-01-01 01:00:00 + """) + snapshots = pd.MultiIndex.from_arrays( + [[2025, 2025], pd.to_datetime(["2025-01-01 00:00", "2025-01-01 01:00"])] + ) + + result = _expand_limits_to_snapshots( + link_timeslice_limits, timeslice_snapshots, snapshots + ) + + expected = csv_str_to_df(""" + name, attribute, investment_periods, snapshots, value + CQ-NQ_existing, p_max_pu, 2025, 2025-01-01 00:00:00, 1.0 + CQ-NQ_existing, p_max_pu, 2025, 2025-01-01 01:00:00, 0.857 + CQ-NQ_existing, p_min_pu, 2025, 2025-01-01 00:00:00, + CQ-NQ_existing, p_min_pu, 2025, 2025-01-01 01:00:00, -0.9 + NQ-CQ_other, p_max_pu, 2025, 2025-01-01 00:00:00, 0.8 + NQ-CQ_other, p_max_pu, 2025, 2025-01-01 01:00:00, 0.8 + """) + # Rows: named value where active, fallback elsewhere, NaN where neither + # (CQ-NQ_existing p_min_pu at 00:00); a named timeslice with no snapshots + # (no_snapshots) contributes nothing so NQ-CQ_other takes its fallback. + expected["snapshots"] = pd.to_datetime(expected["snapshots"]) + pd.testing.assert_frame_equal(result, expected) From 4d25323f977b0314c40487439ccd0a3df4e207be Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 11:30:21 +1000 Subject: [PATCH 07/12] Flatten the uncovered-snapshot error so the test can assert it in full The to_string table made the message whitespace-sensitive; a plain comma-separated (investment_period, snapshot) list reads the same and lets the test pin the whole message. Co-Authored-By: Claude Fable 5 --- src/ispypsa/pypsa_build/links.py | 8 ++++++-- .../test_add_links_with_timeslice_limits.py | 13 +++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/ispypsa/pypsa_build/links.py b/src/ispypsa/pypsa_build/links.py index 9fffd382..9c957082 100644 --- a/src/ispypsa/pypsa_build/links.py +++ b/src/ispypsa/pypsa_build/links.py @@ -250,12 +250,16 @@ def _raise_if_snapshots_uncovered(limits_per_snapshot: pd.DataFrame) -> None: if uncovered.empty: return pairs = sorted(set(zip(uncovered["name"], uncovered["attribute"]))) - first = uncovered.loc[:, ["investment_periods", "snapshots"]].head(5) + first = uncovered.head(5) + first_snapshots = [ + f"({period}, {snapshot})" + for period, snapshot in zip(first["investment_periods"], first["snapshots"]) + ] raise ValueError( f"link_timeslice_limits leaves {len(uncovered)} (link, attribute, snapshot) " f"combination(s) undefined: no fallback (blank-timeslice) row and no named " f"timeslice active there. Affected (link, attribute): {pairs}. " - f"First uncovered snapshots:\n{first.to_string(index=False)}" + f"First uncovered (investment_period, snapshot): {', '.join(first_snapshots)}" ) diff --git a/tests/test_model/test_add_links_with_timeslice_limits.py b/tests/test_model/test_add_links_with_timeslice_limits.py index dd20df0c..135e6362 100644 --- a/tests/test_model/test_add_links_with_timeslice_limits.py +++ b/tests/test_model/test_add_links_with_timeslice_limits.py @@ -168,14 +168,19 @@ def test_snapshot_covered_by_neither_named_timeslice_nor_fallback_raises(csv_str qld_peak_demand, 2025, 2025-01-01 01:00:00 """) - with pytest.raises( - ValueError, - match=r"leaves 3 \(link, attribute, snapshot\) combination\(s\) undefined", - ): + with pytest.raises(ValueError) as excinfo: _add_links_to_network( network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots ) + assert str(excinfo.value) == ( + "link_timeslice_limits leaves 3 (link, attribute, snapshot) combination(s) " + "undefined: no fallback (blank-timeslice) row and no named timeslice active " + "there. Affected (link, attribute): [('CQ-NQ_existing', 'p_max_pu')]. " + "First uncovered (investment_period, snapshot): (2025, 2025-01-01 00:00:00), " + "(2025, 2025-01-01 02:00:00), (2025, 2025-01-01 03:00:00)" + ) + def test_links_without_timeslice_limits_keep_their_static_values(csv_str_to_df): network = _network() From 0e11b34e2c8a73fb7f092337b3e63f92831da82b Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 12:39:26 +1000 Subject: [PATCH 08/12] Pin the fallback-only + no-timeslices dtype failure as an xfail An all-blank timeslice column read from CSV comes back float64 and can't be merged onto the object-typed empty timeslice_snapshots. The proper fix is schema-typed reading of the pypsa-friendly tables (Open-ISP/ISPyPSA#138), so this records the case rather than casting at the merge site. Co-Authored-By: Claude Fable 5 --- .../test_add_links_with_timeslice_limits.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_model/test_add_links_with_timeslice_limits.py b/tests/test_model/test_add_links_with_timeslice_limits.py index 135e6362..3d0e3026 100644 --- a/tests/test_model/test_add_links_with_timeslice_limits.py +++ b/tests/test_model/test_add_links_with_timeslice_limits.py @@ -98,6 +98,40 @@ def test_fallback_only_attribute_gets_the_fallback_at_every_snapshot(csv_str_to_ pd.testing.assert_frame_equal(_link_pu_limits(network, "CQ-NQ_existing"), expected) +@pytest.mark.xfail( + reason="Open-ISP/ISPyPSA#138: an all-blank timeslice column is read from CSV as " + "float64 and cannot be merged onto the object-typed (empty) timeslice_snapshots", + raises=ValueError, + strict=True, +) +def test_fallback_only_limits_with_no_timeslices_apply_at_every_snapshot(csv_str_to_df): + # The translator emits this shape for zero-capacity corridors and all-default + # paths in a run with no timeslices configured. + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, , 0.0 + CQ-NQ_existing, p_min_pu, , 0.0 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + """) + + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + expected = csv_str_to_df(""" + investment_periods, snapshots, p_max_pu, p_min_pu + 2025, 2025-01-01 00:00:00, 0.0, 0.0 + 2025, 2025-01-01 01:00:00, 0.0, 0.0 + 2025, 2025-01-01 02:00:00, 0.0, 0.0 + 2025, 2025-01-01 03:00:00, 0.0, 0.0 + """) + expected["snapshots"] = pd.to_datetime(expected["snapshots"]) + pd.testing.assert_frame_equal(_link_pu_limits(network, "CQ-NQ_existing"), expected) + + def test_named_timeslices_that_tile_the_snapshots_need_no_fallback(csv_str_to_df): network = _network() link_timeslice_limits = csv_str_to_df(""" From 3b9d0e9c6732c37853ef203f57b53d56c6d20ae8 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 13:11:26 +1000 Subject: [PATCH 09/12] Require limits for every existing link, and split the coverage error by extent The coverage check drew its universe from the limits table itself, so a (link, attribute) with no rows at all was never checked and the links table's placeholder p_max_pu / p_min_pu survived silently. The grid is now built from the non-extendable links x both attributes, so an existing link the limits never mention raises like a partially covered one does; expansion links keep their static values and are not checked. The error message now separates pairs undefined at every snapshot (typically no limit rows at all) from those undefined at only some, and samples uncovered snapshots only for the latter. Co-Authored-By: Claude Fable 5 --- src/ispypsa/pypsa_build/links.py | 142 +++++++++++++----- .../test_add_links_with_timeslice_limits.py | 107 +++++++++++-- 2 files changed, 200 insertions(+), 49 deletions(-) diff --git a/src/ispypsa/pypsa_build/links.py b/src/ispypsa/pypsa_build/links.py index 9c957082..7d24dab9 100644 --- a/src/ispypsa/pypsa_build/links.py +++ b/src/ispypsa/pypsa_build/links.py @@ -19,11 +19,12 @@ def _add_links_to_network( """Adds the Links defined in a pypsa-friendly input table called `"links"` to the `pypsa.Network` object. - On the new-format path the two timeslice tables are required: each link's - per-timeslice limits are expanded into per-snapshot p_max_pu / p_min_pu - series that replace the links table's placeholder values (see - _build_link_pu_overrides). On the old-format path both are omitted and - the links table's p_max_pu / p_min_pu are the limits. + On the new-format path the two timeslice tables are required: every + existing (non-extendable) link's per-timeslice limits are expanded into + per-snapshot p_max_pu / p_min_pu series that replace the links table's + placeholder values (see _build_link_pu_overrides). Expansion links keep + the links table's static p_max_pu / p_min_pu. On the old-format path both + tables are omitted and every link's p_max_pu / p_min_pu are the limits. FEATURE_FLAG_CLEANUP[use_new_table_format]: once the flag is retired, make link_timeslice_limits and timeslice_snapshots required and drop the @@ -77,7 +78,7 @@ def _add_links_to_network( no CQ-NQ_existing column). """ pu_overrides = _build_link_pu_overrides( - link_timeslice_limits, timeslice_snapshots, network.snapshots + links, link_timeslice_limits, timeslice_snapshots, network.snapshots ) links["class_name"] = "Link" for _, row in links.iterrows(): @@ -85,20 +86,28 @@ def _add_links_to_network( def _build_link_pu_overrides( + links: pd.DataFrame, link_timeslice_limits: pd.DataFrame | None, timeslice_snapshots: pd.DataFrame | None, snapshots: pd.MultiIndex, ) -> dict[str, dict[str, pd.Series]]: - """Expands each link's per-timeslice limits into per-snapshot series. + """Expands each existing link's per-timeslice limits into per-snapshot series. The translator emits two kinds of limit row per (link, attribute): rows with a named timeslice, which apply at the snapshots that timeslice is active, and a row with timeslice = NaN, which is the fallback for every - snapshot no named timeslice covers. Every snapshot must end up with a - value from one or the other; a snapshot left without one raises rather + snapshot no named timeslice covers. Every existing (non-extendable) link + must end up with a value for both p_max_pu and p_min_pu at every snapshot + from one or the other; a link or snapshot left without one raises rather than silently keeping the links table's placeholder p_max_pu / p_min_pu. + Expansion links are not checked: their p_max_pu / p_min_pu are static. I/O Example: + links (only name and p_nom_extendable are read): + name p_nom_extendable + CQ-NQ_existing False + CQ-NQ_option_1 True # expansion link: not checked + link_timeslice_limits: name attribute timeslice value CQ-NQ_existing p_max_pu qld_peak_demand 0.857 @@ -119,26 +128,32 @@ def _build_link_pu_overrides( "p_min_pu": [-0.714, -0.714]}} Without the p_max_pu fallback row, p_max_pu would be undefined at the - second snapshot -> ValueError. + second snapshot -> ValueError. Without any p_min_pu row, p_min_pu would + be undefined at every snapshot -> ValueError. """ - if link_timeslice_limits is None or link_timeslice_limits.empty: + if link_timeslice_limits is None: return {} limits_per_snapshot = _expand_limits_to_snapshots( - link_timeslice_limits, timeslice_snapshots, snapshots + links, link_timeslice_limits, timeslice_snapshots, snapshots ) _raise_if_snapshots_uncovered(limits_per_snapshot) return _series_by_link_and_attribute(limits_per_snapshot, snapshots) def _expand_limits_to_snapshots( + links: pd.DataFrame, link_timeslice_limits: pd.DataFrame, timeslice_snapshots: pd.DataFrame, snapshots: pd.MultiIndex, ) -> pd.DataFrame: - """One row per (link, attribute, snapshot): the named timeslice's value where - one is active there, else the fallback, else NaN. + """One row per (existing link, attribute, snapshot): the named timeslice's + value where one is active there, else the fallback, else NaN. I/O Example: + links (only name and p_nom_extendable are read): + name p_nom_extendable + CQ-NQ_existing False + link_timeslice_limits: name attribute timeslice value CQ-NQ_existing p_max_pu qld_peak_demand 0.857 @@ -166,7 +181,7 @@ def _expand_limits_to_snapshots( link_timeslice_limits[~is_fallback], timeslice_snapshots ) fallback = link_timeslice_limits.loc[is_fallback, ["name", "attribute", "value"]] - grid = _link_attribute_snapshot_grid(link_timeslice_limits, snapshots) + grid = _link_attribute_snapshot_grid(links, snapshots) grid = grid.merge(named, how="left") grid = grid.merge(fallback.rename(columns={"value": "fallback"}), how="left") grid["value"] = grid["value"].fillna(grid["fallback"]) @@ -174,15 +189,16 @@ def _expand_limits_to_snapshots( def _link_attribute_snapshot_grid( - link_timeslice_limits: pd.DataFrame, snapshots: pd.MultiIndex + links: pd.DataFrame, snapshots: pd.MultiIndex ) -> pd.DataFrame: - """Every (link, attribute) pair in the limits table at every snapshot. + """Every existing (non-extendable) link, for both p_max_pu and p_min_pu, at + every snapshot. Expansion links are left out: their limits are static. I/O Example: - link_timeslice_limits (only name and attribute are read): - name attribute timeslice value - CQ-NQ_existing p_max_pu qld_peak_demand 0.857 - CQ-NQ_existing p_max_pu , 1.0 # same pair, once in the grid + links (only name and p_nom_extendable are read): + name p_nom_extendable + CQ-NQ_existing False + CQ-NQ_option_1 True # left out snapshots: investment_periods snapshots @@ -193,12 +209,15 @@ def _link_attribute_snapshot_grid( name attribute investment_periods snapshots CQ-NQ_existing p_max_pu 2025 2025-01-13 12:00 CQ-NQ_existing p_max_pu 2025 2025-01-15 12:00 + CQ-NQ_existing p_min_pu 2025 2025-01-13 12:00 + CQ-NQ_existing p_min_pu 2025 2025-01-15 12:00 """ - pairs = link_timeslice_limits.loc[:, ["name", "attribute"]].drop_duplicates() + existing = links.loc[~links["p_nom_extendable"], ["name"]] + attributes = pd.DataFrame({"attribute": ["p_max_pu", "p_min_pu"]}) snapshot_rows = pd.DataFrame( snapshots.tolist(), columns=["investment_periods", "snapshots"] ) - return pairs.merge(snapshot_rows, how="cross") + return existing.merge(attributes, how="cross").merge(snapshot_rows, how="cross") def _place_named_limits_at_snapshots( @@ -232,6 +251,11 @@ def _raise_if_snapshots_uncovered(limits_per_snapshot: pd.DataFrame) -> None: """Raises if any (link, attribute) has a snapshot with neither a named-timeslice nor a fallback value. + The message separates pairs undefined at every snapshot (typically a link + with no limit rows at all) from pairs undefined at only some snapshots + (named-timeslice rows with no fallback), and samples the uncovered + snapshots of the latter. + I/O Example: limits_per_snapshot: name attribute investment_periods snapshots value @@ -241,26 +265,72 @@ def _raise_if_snapshots_uncovered(limits_per_snapshot: pd.DataFrame) -> None: limits_per_snapshot: name attribute investment_periods snapshots value + CQ-NQ_existing p_max_pu 2025 2025-01-13 12:00 # no rows at all + CQ-NQ_existing p_max_pu 2025 2025-01-15 12:00 CQ-NQ_existing p_min_pu 2025 2025-01-13 12:00 -0.9 - CQ-NQ_existing p_min_pu 2025 2025-01-15 12:00 # uncovered - raises ValueError naming (CQ-NQ_existing, p_min_pu) and the - uncovered snapshot (2025, 2025-01-15 12:00). + CQ-NQ_existing p_min_pu 2025 2025-01-15 12:00 # no fallback + raises ValueError: "... Undefined at every snapshot: [('CQ-NQ_existing', + 'p_max_pu')]. Undefined at some snapshots: [('CQ-NQ_existing', 'p_min_pu')], + first uncovered (investment_period, snapshot): (2025, 2025-01-15 12:00:00)" """ uncovered = limits_per_snapshot[limits_per_snapshot["value"].isna()] if uncovered.empty: return - pairs = sorted(set(zip(uncovered["name"], uncovered["attribute"]))) - first = uncovered.head(5) - first_snapshots = [ - f"({period}, {snapshot})" - for period, snapshot in zip(first["investment_periods"], first["snapshots"]) - ] + at_every, at_some = _split_pairs_by_uncovered_extent(limits_per_snapshot, uncovered) + sample = _first_uncovered_snapshots(uncovered, at_some) raise ValueError( - f"link_timeslice_limits leaves {len(uncovered)} (link, attribute, snapshot) " - f"combination(s) undefined: no fallback (blank-timeslice) row and no named " - f"timeslice active there. Affected (link, attribute): {pairs}. " - f"First uncovered (investment_period, snapshot): {', '.join(first_snapshots)}" + f"link_timeslice_limits leaves p_max_pu / p_min_pu undefined for existing " + f"links: no fallback (blank-timeslice) row and no named timeslice active " + f"there. Undefined at every snapshot: {at_every}. " + f"Undefined at some snapshots: {at_some}{sample}" + ) + + +def _split_pairs_by_uncovered_extent( + limits_per_snapshot: pd.DataFrame, uncovered: pd.DataFrame +) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]: + """Splits the (link, attribute) pairs with uncovered snapshots into those + uncovered at every snapshot and those uncovered at only some. + + I/O Example: + limits_per_snapshot: A p_max_pu at 2 snapshots (both NaN), + A p_min_pu at 2 snapshots (one NaN) + uncovered: the 3 NaN rows + + returns ([("A", "p_max_pu")], [("A", "p_min_pu")]) + """ + keys = ["name", "attribute"] + n_snapshots = limits_per_snapshot.groupby(keys).size() + n_uncovered = uncovered.groupby(keys).size() + at_every = n_uncovered == n_snapshots.loc[n_uncovered.index] + return sorted(at_every[at_every].index), sorted(at_every[~at_every].index) + + +def _first_uncovered_snapshots( + uncovered: pd.DataFrame, pairs: list[tuple[str, str]] +) -> str: + """An error-message clause sampling the first five uncovered snapshots of the + given (link, attribute) pairs; empty when there are no pairs to sample. + + I/O Example: + uncovered: + name attribute investment_periods snapshots + A p_max_pu 2025 2025-01-13 12:00 + A p_min_pu 2025 2025-01-15 12:00 + pairs: [("A", "p_min_pu")] + returns ", first uncovered (investment_period, snapshot): (2025, 2025-01-15 12:00:00)" + + pairs: [] + returns "" + """ + if not pairs: + return "" + rows = uncovered.set_index(["name", "attribute"]).loc[pairs].head(5) + sample = ", ".join( + f"({period}, {snapshot})" + for period, snapshot in zip(rows["investment_periods"], rows["snapshots"]) ) + return f", first uncovered (investment_period, snapshot): {sample}" def _series_by_link_and_attribute( diff --git a/tests/test_model/test_add_links_with_timeslice_limits.py b/tests/test_model/test_add_links_with_timeslice_limits.py index 3d0e3026..a642fae6 100644 --- a/tests/test_model/test_add_links_with_timeslice_limits.py +++ b/tests/test_model/test_add_links_with_timeslice_limits.py @@ -208,15 +208,19 @@ def test_snapshot_covered_by_neither_named_timeslice_nor_fallback_raises(csv_str ) assert str(excinfo.value) == ( - "link_timeslice_limits leaves 3 (link, attribute, snapshot) combination(s) " - "undefined: no fallback (blank-timeslice) row and no named timeslice active " - "there. Affected (link, attribute): [('CQ-NQ_existing', 'p_max_pu')]. " - "First uncovered (investment_period, snapshot): (2025, 2025-01-01 00:00:00), " + "link_timeslice_limits leaves p_max_pu / p_min_pu undefined for existing " + "links: no fallback (blank-timeslice) row and no named timeslice active " + "there. Undefined at every snapshot: []. " + "Undefined at some snapshots: [('CQ-NQ_existing', 'p_max_pu')], " + "first uncovered (investment_period, snapshot): (2025, 2025-01-01 00:00:00), " "(2025, 2025-01-01 02:00:00), (2025, 2025-01-01 03:00:00)" ) -def test_links_without_timeslice_limits_keep_their_static_values(csv_str_to_df): +def test_existing_link_with_no_timeslice_limits_raises(csv_str_to_df): + # The links table's p_max_pu / p_min_pu are placeholders on the new-format + # path, so an existing link the limits table never mentions must not + # silently keep them. network = _network() link_timeslice_limits = csv_str_to_df(""" name, attribute, timeslice, value @@ -225,14 +229,81 @@ def test_links_without_timeslice_limits_keep_their_static_values(csv_str_to_df): timeslice_id, investment_periods, snapshots """) - _add_links_to_network( - network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + with pytest.raises(ValueError) as excinfo: + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + assert str(excinfo.value) == ( + "link_timeslice_limits leaves p_max_pu / p_min_pu undefined for existing " + "links: no fallback (blank-timeslice) row and no named timeslice active " + "there. Undefined at every snapshot: [('CQ-NQ_existing', 'p_max_pu'), " + "('CQ-NQ_existing', 'p_min_pu')]. Undefined at some snapshots: []" ) - assert "CQ-NQ_existing" not in network.links_t.p_max_pu.columns - assert "CQ-NQ_existing" not in network.links_t.p_min_pu.columns - assert network.links.loc["CQ-NQ_existing", "p_max_pu"] == 1.0 - assert network.links.loc["CQ-NQ_existing", "p_min_pu"] == 0.0 + +def test_existing_link_missing_one_attribute_raises(csv_str_to_df): + # p_max_pu is fully covered but there is no p_min_pu row at all: the + # placeholder p_min_pu = 0.0 would silently disable reverse flow. + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857 + CQ-NQ_existing, p_max_pu, , 1.0 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + qld_peak_demand, 2025, 2025-01-01 01:00:00 + """) + + with pytest.raises(ValueError) as excinfo: + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + assert str(excinfo.value) == ( + "link_timeslice_limits leaves p_max_pu / p_min_pu undefined for existing " + "links: no fallback (blank-timeslice) row and no named timeslice active " + "there. Undefined at every snapshot: [('CQ-NQ_existing', 'p_min_pu')]. " + "Undefined at some snapshots: []" + ) + + +def test_expansion_links_keep_their_static_values(csv_str_to_df): + # Expansion links (p_nom_extendable) carry real static p_max_pu / p_min_pu + # and are not in link_timeslice_limits, so they are neither overridden nor + # required to be covered. + network = _network() + links = csv_str_to_df(""" + name, bus0, bus1, carrier, p_nom, p_max_pu, p_min_pu, p_nom_extendable + CQ-NQ_existing, bus1, bus2, AC, 1400, 1.0, 0.0, False + CQ-NQ_option_1, bus1, bus2, AC, 0, 0.9, -0.6, True + """) + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857 + CQ-NQ_existing, p_max_pu, , 1.0 + CQ-NQ_existing, p_min_pu, , -0.714 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + qld_peak_demand, 2025, 2025-01-01 01:00:00 + """) + + _add_links_to_network(network, links, link_timeslice_limits, timeslice_snapshots) + + expected = csv_str_to_df(""" + Link, p_max_pu, p_min_pu + CQ-NQ_existing, 1.0, 0.0 + CQ-NQ_option_1, 0.9, -0.6 + """).set_index("Link") + # The existing link's static values are the untouched placeholders; its + # real limits live in links_t (covered by the tests above). + pd.testing.assert_frame_equal( + network.links.loc[:, ["p_max_pu", "p_min_pu"]], expected, check_names=False + ) + assert list(network.links_t.p_max_pu.columns) == ["CQ-NQ_existing"] + assert list(network.links_t.p_min_pu.columns) == ["CQ-NQ_existing"] def test_old_format_call_without_limit_tables(csv_str_to_df): @@ -260,9 +331,15 @@ def test_expand_limits_to_snapshots(csv_str_to_df): snapshots = pd.MultiIndex.from_arrays( [[2025, 2025], pd.to_datetime(["2025-01-01 00:00", "2025-01-01 01:00"])] ) + links = csv_str_to_df(""" + name, p_nom_extendable + CQ-NQ_existing, False + NQ-CQ_other, False + CQ-NQ_option_1, True + """) result = _expand_limits_to_snapshots( - link_timeslice_limits, timeslice_snapshots, snapshots + links, link_timeslice_limits, timeslice_snapshots, snapshots ) expected = csv_str_to_df(""" @@ -273,9 +350,13 @@ def test_expand_limits_to_snapshots(csv_str_to_df): CQ-NQ_existing, p_min_pu, 2025, 2025-01-01 01:00:00, -0.9 NQ-CQ_other, p_max_pu, 2025, 2025-01-01 00:00:00, 0.8 NQ-CQ_other, p_max_pu, 2025, 2025-01-01 01:00:00, 0.8 + NQ-CQ_other, p_min_pu, 2025, 2025-01-01 00:00:00, + NQ-CQ_other, p_min_pu, 2025, 2025-01-01 01:00:00, """) # Rows: named value where active, fallback elsewhere, NaN where neither # (CQ-NQ_existing p_min_pu at 00:00); a named timeslice with no snapshots - # (no_snapshots) contributes nothing so NQ-CQ_other takes its fallback. + # (no_snapshots) contributes nothing so NQ-CQ_other takes its fallback; + # NQ-CQ_other has no p_min_pu rows at all so that attribute is all NaN; + # the expansion link CQ-NQ_option_1 is not in the grid. expected["snapshots"] = pd.to_datetime(expected["snapshots"]) pd.testing.assert_frame_equal(result, expected) From c6d33aef422f4b9905a5bc53e03f80ec0b710ea2 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 13:20:06 +1000 Subject: [PATCH 10/12] Assert the old-format link path against a full frame Per-cell lookups can miss stray columns or a placeholder leaking through; a side-by-side frame with a non-placeholder p_min_pu pins that the links table's static values really are the limits when no limit tables are passed. Co-Authored-By: Claude Fable 5 --- .../test_add_links_with_timeslice_limits.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/test_model/test_add_links_with_timeslice_limits.py b/tests/test_model/test_add_links_with_timeslice_limits.py index a642fae6..8de262cf 100644 --- a/tests/test_model/test_add_links_with_timeslice_limits.py +++ b/tests/test_model/test_add_links_with_timeslice_limits.py @@ -307,12 +307,28 @@ def test_expansion_links_keep_their_static_values(csv_str_to_df): def test_old_format_call_without_limit_tables(csv_str_to_df): + # With no limit tables the links table's p_max_pu / p_min_pu are the real + # limits and stay static; nothing is checked or overridden. network = _network() + links = csv_str_to_df(""" + name, bus0, bus1, carrier, p_nom, p_max_pu, p_min_pu, p_nom_extendable + CQ-NQ_existing, bus1, bus2, AC, 1400, 1.0, -0.5, False + """) - _add_links_to_network(network, _links(csv_str_to_df)) + _add_links_to_network(network, links) - assert network.links.loc["CQ-NQ_existing", "p_nom"] == 1400 - assert "CQ-NQ_existing" not in network.links_t.p_max_pu.columns + expected = csv_str_to_df(""" + Link, p_nom, p_max_pu, p_min_pu + CQ-NQ_existing, 1400, 1.0, -0.5 + """).set_index("Link") + pd.testing.assert_frame_equal( + network.links.loc[:, ["p_nom", "p_max_pu", "p_min_pu"]], + expected, + check_names=False, + check_dtype=False, + ) + assert list(network.links_t.p_max_pu.columns) == [] + assert list(network.links_t.p_min_pu.columns) == [] def test_expand_limits_to_snapshots(csv_str_to_df): From 5a3ec3494aa648ebce01147cbf94a3da4007fa6d Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 13:24:36 +1000 Subject: [PATCH 11/12] Pin the templater's named-timeslice shape for zero-capacity corridors The test described a blank-timeslice fallback of 0, but _new_parallel_path_rows emits explicit zeros per direction and named timeslice with no fallback. Now that zero rows reach pypsa_build's coverage check the two shapes behave differently, so the test exercises the real one and keeps the fallback form as a separate case. Co-Authored-By: Claude Fable 5 --- tests/test_translator/test_network.py | 61 ++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/tests/test_translator/test_network.py b/tests/test_translator/test_network.py index ce4eb3ad..3c3e80b0 100644 --- a/tests/test_translator/test_network.py +++ b/tests/test_translator/test_network.py @@ -490,17 +490,66 @@ def test_translate_network_to_links_rezs_attached_to_parent_node( def test_translate_network_to_links_zero_capacity_parallel_path( csv_str_to_df, sample_model_config ): - """A new parallel corridor has zero existing capacity, given as a - timeslice = NaN fallback of 0 in both directions. It becomes an inert - existing link at p_nom 0 (its buildable capacity is modelled by expansion - links) whose per-timeslice limits are per-unit 0 rather than an undefined - 0/0, so pypsa_build treats it like any other link.""" + """A new parallel corridor has zero existing capacity, which the templater + (_new_parallel_path_rows) gives as explicit 0 rows for each direction and + each region-prefixed canonical timeslice, with no fallback row. It becomes + an inert existing link at p_nom 0 (its buildable capacity is modelled by + expansion links) whose per-timeslice limits are per-unit 0 rather than an + undefined 0/0, so pypsa_build treats it like any other link.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CNSW-SNW, CNSW, SNW, AC + """) + # The templater's shape: 2 directions x 3 named timeslices, all 0, no fallback. + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CNSW-SNW, forward, nsw_peak_demand, 0 + CNSW-SNW, forward, nsw_summer_typical, 0 + CNSW-SNW, forward, nsw_winter_reference, 0 + CNSW-SNW, reverse, nsw_peak_demand, 0 + CNSW-SNW, reverse, nsw_summer_typical, 0 + CNSW-SNW, reverse, nsw_winter_reference, 0 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CNSW-SNW, CNSW-SNW_existing, AC, CNSW, SNW, 0, 0.0, 1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CNSW-SNW_existing, p_max_pu, nsw_peak_demand, 0.0 + CNSW-SNW_existing, p_max_pu, nsw_summer_typical, 0.0 + CNSW-SNW_existing, p_max_pu, nsw_winter_reference, 0.0 + CNSW-SNW_existing, p_min_pu, nsw_peak_demand, 0.0 + CNSW-SNW_existing, p_min_pu, nsw_summer_typical, 0.0 + CNSW-SNW_existing, p_min_pu, nsw_winter_reference, 0.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits, expected_limits, check_dtype=False + ) + + +def test_translate_network_to_links_zero_capacity_fallback_row( + csv_str_to_df, sample_model_config +): + """A zero-capacity corridor given as a timeslice = NaN fallback of 0 in + both directions (a hand-authored alternative to the templater's named + rows) also becomes an inert p_nom 0 link with per-unit 0 fallbacks.""" ispypsa_tables = _network_tables(csv_str_to_df) ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" path_id, geo_from, geo_to, carrier CNSW-SNW, CNSW, SNW, AC """) - # Zero existing capacity as a timeslice = NaN fallback (covers the year). ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" path_id, direction, timeslice, capacity CNSW-SNW, forward, , 0 From 4faa45f2399536a0b5374fe0c0649e6da5d41544 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Mon, 24 Aug 2026 09:52:22 +1000 Subject: [PATCH 12/12] Name the snapshot-grid helper with a verb phrase _link_attribute_snapshot_grid read as a noun, so the orchestrator line didn't say what the helper does. _create_* matches the verb-phrase convention the other helpers follow. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011fNbcEWRpvRjndGUXvt4FV --- src/ispypsa/pypsa_build/links.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ispypsa/pypsa_build/links.py b/src/ispypsa/pypsa_build/links.py index 7d24dab9..e8ec3759 100644 --- a/src/ispypsa/pypsa_build/links.py +++ b/src/ispypsa/pypsa_build/links.py @@ -181,14 +181,14 @@ def _expand_limits_to_snapshots( link_timeslice_limits[~is_fallback], timeslice_snapshots ) fallback = link_timeslice_limits.loc[is_fallback, ["name", "attribute", "value"]] - grid = _link_attribute_snapshot_grid(links, snapshots) + grid = _create_link_attribute_snapshot_grid(links, snapshots) grid = grid.merge(named, how="left") grid = grid.merge(fallback.rename(columns={"value": "fallback"}), how="left") grid["value"] = grid["value"].fillna(grid["fallback"]) return grid.loc[:, _LIMIT_PER_SNAPSHOT_COLUMNS] -def _link_attribute_snapshot_grid( +def _create_link_attribute_snapshot_grid( links: pd.DataFrame, snapshots: pd.MultiIndex ) -> pd.DataFrame: """Every existing (non-extendable) link, for both p_max_pu and p_min_pu, at