diff --git a/SWEET_python/advanced_dst_city.py b/SWEET_python/advanced_dst_city.py index 1c70b47..b0dfc50 100644 --- a/SWEET_python/advanced_dst_city.py +++ b/SWEET_python/advanced_dst_city.py @@ -48,6 +48,7 @@ from SWEET_python.city_params import City, CityParameters, CustomError from SWEET_python.class_defs import DivsDF, LandfillType, LandfillWasteMassDF, Variant from SWEET_python import dst_common as common +from SWEET_python import site_inflow from SWEET_python.dst_common import YearlyFloat, YearlyFractions __all__ = ["AdvancedDSTCityRequest", "CityLandfillSpec", "run_advanced_dst_city"] @@ -106,6 +107,28 @@ class CityLandfillSpec(BaseModel): "decays." ), ) + accepted_waste_mass: Optional[Variant[YearlyFloat]] = Field( + None, + description=( + "Total waste crossing this site's weighbridge each year, in tons, " + "from every source -- a fact measured at the gate rather than a " + "share of anything. Omit (the default) and the site receives " + "exactly the city's allocation, which is the existing behaviour.\n\n" + "This does not replace ``landfill_split_timeline``, which still " + "divides the city's own landfilled waste across the city's sites " + "and must still sum to ~1 per year. Whatever this figure exceeds " + "the city's allocation is modeled as a second stream arriving from " + "outside the baseline: it carries the city's post-diversion " + "residual composition, and is combusted and windowed on exactly the " + "same terms as the city's stream, because it is the same hole in " + "the ground.\n\n" + "It never changes the city's own emissions -- a city answers for " + "the waste it generates, wherever that waste goes. A figure *below* " + "the city's allocation is read as 'no waste from outside', not as a " + "reduction of the city's; see ``site_inflow.surplus_intake``. Ask " + "for the split with ``with_site_emissions=True``." + ), + ) class AdvancedDSTCityRequest(BaseModel): @@ -852,7 +875,10 @@ def run_advanced_dst_city_limits( # Public entry point # --------------------------------------------------------------------------- # def run_advanced_dst_city( - request: AdvancedDSTCityRequest, *, with_mass_flow: bool = False + request: AdvancedDSTCityRequest, + *, + with_mass_flow: bool = False, + with_site_emissions: bool = False, ) -> dict: """Run the city-level advanced DST. @@ -872,6 +898,17 @@ def run_advanced_dst_city( *show* the mass flow should read it from here rather than reimplementing the diversion split, which is per waste type and does not survive being approximated as a scalar on the total. + + With ``with_site_emissions=True`` the result carries a ``"site_emissions"`` + key: per variant, one entry per landfill in request order, each + ``{"city", "other", "total"}`` in tons of methane per year. ``city`` is the + methane this city's waste produced at that site and is what the city's own + total is built from; ``other`` is what waste from outside the baseline + produced there (see ``CityLandfillSpec.accepted_waste_mass``), and is zero + for a site that only takes the city's waste. They are two independent passes + through the decay kernel, summed -- never a ratio applied to one of them, + which is wrong by tens of percent wherever the city's share of a site moves + over time. Asking for them changes no emissions. """ implement_year = int(request.implement_year) @@ -965,6 +1002,19 @@ def run_advanced_dst_city( ) net_masses[label] = wgen.sub(divs.sum(), fill_value=0.0) + # The mix the city is left to bury, as per-year component shares. Any waste + # arriving at a site from outside the baseline is given this same mix, so + # that both streams at a site decay at one `k` and therefore superpose -- + # see `site_inflow` for why that is load-bearing rather than tidy. + # The fallback is the composition as *shares* rather than `wgen_*`, the + # composition as masses: a year the city generates nothing has an all-zero + # mass frame and so no fallback at all, which left a site's outside waste + # with a zero mix and deposited none of it. + residual_mix = { + "baseline": site_inflow.residual_composition(net_masses["baseline"], baseline_fractions), + "scenario": site_inflow.residual_composition(net_masses["scenario"], scenario_fractions), + } + # --- City-wide decomposition rates + compost emission factors --- ref_year = min(max(implement_year, int(years.min())), int(years.max())) ks_baseline, ks_scenario = common.decomposition_rates( @@ -1024,6 +1074,15 @@ def run_advanced_dst_city( scenario_masses: List[pd.DataFrame] = [] baseline_ox: List[pd.Series] = [] scenario_ox: List[pd.Series] = [] + # One entry per site, `None` where the site takes only the city's waste. + # Deliberately kept out of `*_parameters.landfills`: that list is what + # `City.sum_landfill_emissions` sums, and a city does not answer for a + # neighbour's waste. Keeping these out of it is the whole reason the city's + # total is bit-identical rather than merely close. + baseline_surplus: List[Optional[object]] = [] + scenario_surplus: List[Optional[object]] = [] + baseline_surplus_masses: List[Optional[pd.DataFrame]] = [] + scenario_surplus_masses: List[Optional[pd.DataFrame]] = [] for index, spec in enumerate(request.landfills): base_type = int(spec.landfill_type["baseline"]) @@ -1064,6 +1123,24 @@ def run_advanced_dst_city( # Net-of-diversion city waste, scaled to this landfill's per-year share. mass_base = LandfillWasteMassDF.create_advanced(wgen_baseline, divs_baseline, share_base.copy()).df mass_scen = LandfillWasteMassDF.create_advanced(wgen_scenario, divs_scenario, share_scen.copy()).df + + # What the city sends here, at the gate -- read before the combustion + # and window lines below, because a stated gate total is measured at the + # weighbridge and a site that burns its intake still accepted all of it. + surplus_base_masses = surplus_scen_masses = None + if spec.accepted_waste_mass is not None: + accepted_base, accepted_scen = common.variant_series( + spec.accepted_waste_mass, years, implement_year, default=0.0 + ) + surplus_base_masses = site_inflow.surplus_masses( + site_inflow.surplus_intake(accepted_base, mass_base.sum(axis=1)), + residual_mix["baseline"], + ) + surplus_scen_masses = site_inflow.surplus_masses( + site_inflow.surplus_intake(accepted_scen, mass_scen.sum(axis=1)), + residual_mix["scenario"], + ) + # A combusting facility burns its intake and deposits only the reject. # Scaled per variant here, ahead of both the window and the pre-implement # splice below, so a site that starts combusting at implement_year keeps @@ -1082,13 +1159,41 @@ def run_advanced_dst_city( baseline_masses.append(mass_base) scenario_masses.append(mass_scen) - baseline_landfills.append(common.build_landfill( + # The surplus is the same waste in the same hole, so it takes the same + # three transformations in the same order. Anything else and the two + # streams disagree at a boundary -- a closure year, an implement year, + # the year a site starts burning -- which is exactly where a reader + # would notice and could not explain it. + if surplus_base_masses is not None: + surplus_base_masses = surplus_base_masses * _deposited_share(base_combusts, city) + surplus_scen_masses = surplus_scen_masses * _deposited_share(scen_combusts, city) + surplus_base_masses = common.apply_window(surplus_base_masses, b_open, b_close) + surplus_scen_masses = common.apply_window(surplus_scen_masses, s_open, s_close) + surplus_scen_masses.loc[: implement_year - 1, :] = surplus_base_masses.loc[ + : implement_year - 1, : + ] + # A stated total at or below the city's allocation in every year + # leaves nothing to model, and running the kernel on an all-zero + # frame would only produce an all-zero frame. + if not ( + surplus_base_masses.to_numpy().any() or surplus_scen_masses.to_numpy().any() + ): + surplus_base_masses = surplus_scen_masses = None + baseline_surplus_masses.append(surplus_base_masses) + scenario_surplus_masses.append(surplus_scen_masses) + + # Lifted into a dict so the surplus stream below is built from the very + # same arguments rather than a copy of them that has to be kept in step: + # two streams at one site must agree on every parameter but their mass, + # or they decay differently and stop adding up to the site. + base_landfill_kwargs = dict( open_year=b_open, close_year=b_close, site_type_idx=base_type, mcf=mcf_base, gas_capture_efficiency=gas_base, flaring=flare_base, oxidation_factor=ox_base, ks=ks_baseline, city_params_dict=baseline_params_dict, city_instance_attrs=city_instance_attrs, implement_year=implement_year, scenario=0, landfill_index=index, - )) + ) + baseline_landfills.append(common.build_landfill(**base_landfill_kwargs)) # The model evaluates from `open_date` onward (`model_v2.estimate_emissions2` # builds its year range from it), so the scenario has to start wherever # its mass frame can first be nonzero -- and the splice above puts @@ -1098,13 +1203,29 @@ def run_advanced_dst_city( # the scenario's emissions frame entirely, leaving the two halves a # different shape for a caller trying to subtract them. scenario_open_for_model = min(b_open, s_open) - scenario_landfills.append(common.build_landfill( + scen_landfill_kwargs = dict( open_year=scenario_open_for_model, close_year=s_close, site_type_idx=scen_type, mcf=mcf_scen, gas_capture_efficiency=gas_scen, flaring=flare_scen, oxidation_factor=ox_scen, ks=ks_scenario, city_params_dict=scenario_params_dict, city_instance_attrs=city_instance_attrs, implement_year=implement_year, scenario=1, landfill_index=index, - )) + ) + scenario_landfills.append(common.build_landfill(**scen_landfill_kwargs)) + + # The surplus twin. Same open year above all else: `model_v2` builds its + # year range from `open_date`, so a twin opened anywhere else returns a + # differently indexed frame and the site total silently gains NaNs where + # the two failed to line up. + baseline_surplus.append( + common.build_landfill(**base_landfill_kwargs) + if surplus_base_masses is not None + else None + ) + scenario_surplus.append( + common.build_landfill(**scen_landfill_kwargs) + if surplus_scen_masses is not None + else None + ) # --- Wire up and run the engine --- baseline_parameters.landfills = baseline_landfills @@ -1121,6 +1242,21 @@ def run_advanced_dst_city( landfill.oxidation_factor = ox landfill.estimate_emissions(skip_ox=True) + # The surplus streams, run the same way -- but only after + # `repopulate_attr_dicts` above has settled the city's parameter dict, which + # these never receive because they are not in `parameters.landfills`. + for streams, landfills, masses, oxidations in ( + (baseline_surplus, baseline_landfills, baseline_surplus_masses, baseline_ox), + (scenario_surplus, scenario_landfills, scenario_surplus_masses, scenario_ox), + ): + for surplus, city_stream, mass, ox in zip(streams, landfills, masses, oxidations): + if surplus is None: + continue + site_inflow.adopt_city_params(city_stream, surplus) + surplus.waste_mass_df = mass + surplus.oxidation_factor = ox + surplus.estimate_emissions(skip_ox=True) + city.baseline_parameters = baseline_parameters city.scenario_parameters[0] = scenario_parameters @@ -1144,6 +1280,17 @@ def run_advanced_dst_city( "baseline": baseline_parameters.total_emissions, "scenario": scenario_parameters.total_emissions, } + if with_site_emissions: + result["site_emissions"] = { + "baseline": [ + site_inflow.emission_frames(city_stream, surplus) + for city_stream, surplus in zip(baseline_landfills, baseline_surplus) + ], + "scenario": [ + site_inflow.emission_frames(city_stream, surplus) + for city_stream, surplus in zip(scenario_landfills, scenario_surplus) + ], + } if with_mass_flow: result["mass_flow"] = { "baseline": _mass_flow( diff --git a/SWEET_python/site_inflow.py b/SWEET_python/site_inflow.py new file mode 100644 index 0000000..d810086 --- /dev/null +++ b/SWEET_python/site_inflow.py @@ -0,0 +1,250 @@ +"""Waste arriving at a disposal site that the city did not send it. + +A city-level ADST run shares the city's landfilled residual out across the +city's own sites: ``landfill_split_timeline`` is *fractions that must sum to +1.0* (``advanced_dst_city._validate_shares``), so every ton at every site came +from the city by construction. Real sites are not like that. A regional +landfill takes waste from neighbouring municipalities, from private haulers, +from a catchment the baseline never describes -- and the operator knows the +gate total, not this city's share of it. + +So a site may state ``accepted_waste_mass``: everything crossing its weighbridge +in a year, from every source. Whatever that exceeds the city's own allocation is +modeled as a second stream at the same site. + +Two things then have to be true, and this module is arranged so that both are +structural facts about the code rather than arithmetic that happens to come out +right. + +**The city's emissions do not move.** A city is answerable for the waste it +generates, wherever that waste ends up; a neighbour's waste is the neighbour's +inventory. ``City.sum_landfill_emissions`` sums ``.emissions`` over every entry +of ``parameters.landfills`` -- so the surplus landfills built here are never put +in that list. Nothing about the city's total is recomputed, re-derived or +carefully cancelled: it is the same expression over the same objects, and the +test asserts frame equality rather than a tolerance. + +**A site's total is its streams, summed exactly.** Deposited mass enters the +first-order-decay kernel once and multiplicatively (``model_v2``: +``ch4_produce = ks_values * L_0[waste] * waste_masses * exp_term * mcf_values``) +and every step after it -- capture, flare, oxidation, the unit conversion -- is +a mass-independent factor. ``E(0)`` is exactly ``0.0``. So emissions superpose, +measured at ~1e-16 relative, and "the city's share of this site's methane" is a +physical quantity rather than an allocation convention somebody had to invent +and defend. + +That invariant needs every stream at a site to agree on every parameter but its +mass, which is why the surplus twin is built from the city stream's own kwargs +dict rather than a copy of it. Note what is *not* on that list: composition. +``k`` is computed once per variant from the city's generated mix +(``advanced_dst_city`` around the ``decomposition_rates`` call) and handed +identically to every landfill, so a stream's own mix moves only its +per-component deposited masses -- which the kernel is linear in. Two streams at +one site with entirely different mixes still superpose exactly. + +So the surplus carrying the city's post-diversion residual composition is a +**modelling default, not a numerical requirement**. It is the default because it +is the defensible reading of a gate observation -- a landfill's intake is +residual in character whoever sent it, since somebody's diversion happened +upstream of it -- and because the user has no second composition to hand. A +later PR can let a site state its own mix for the surplus, and +``residual_composition`` is the one function it has to replace. + +(The 27.2% figure quoted in PR #785 is about blending two mixes into a *single* +kernel run, where one ``k`` has to stand for both. That is not what happens +here: each stream gets its own ``Landfill`` and its own pass.) + +The other way to get this wrong is to skip the second kernel run and attribute +by tonnage: ``city tons / site tons * site total``. That is wrong whenever the +city's share moves over time, because this year's emissions come from decades of +deposit cohorts each with their own split -- measured at up to 28% error for a +site with a growing neighbour, and in a direction that looks entirely plausible. +Each stream gets its own ``Landfill`` and its own pass through the kernel. + +Why the functions here are free functions rather than ``Landfill`` methods: the +package already separates a landfill's construction from its definition -- +``dst_common.build_landfill`` builds a :class:`~SWEET_python.landfill.Landfill` +from a different module and leaves the caller to assign ``waste_mass_df``. This +follows that convention rather than growing either ``landfill.py`` or the +9,000-line ``city_params.py``. +""" + +from typing import Optional + +import pandas as pd +from pydantic import BaseModel, ConfigDict + +from SWEET_python.city_params import City, CustomError +from SWEET_python.landfill import Landfill + +__all__ = [ + "SiteEmissions", + "residual_composition", + "surplus_intake", + "surplus_masses", + "adopt_city_params", + "emission_frames", +] + + +class SiteEmissions(BaseModel): + """One site's methane, split by whose waste produced it. + + Each frame is years x the model's degradable components plus ``total``, in + **tons of methane per year** -- the same units and the same conversion + ``City.sum_landfill_emissions`` applies to the city's own figure, so the two + outputs can be read side by side. + + ``city`` is the share the city's own headline total is built from; ``other`` + is waste that reached the gate from outside the baseline, and is an all-zero + frame rather than ``None`` for the ordinary site, so no caller has to + special-case it. + """ + + city: pd.DataFrame + other: pd.DataFrame + total: pd.DataFrame + + model_config = ConfigDict(arbitrary_types_allowed=True) + + +def residual_composition( + residual: pd.DataFrame, fractions: pd.DataFrame +) -> pd.DataFrame: + """Per-year component shares of what the city is left to bury; rows sum to 1. + + The mix the surplus is given. This is the single place that decision is + made, and the one function a later "a site states its own mix" PR replaces + -- nothing downstream of it assumes the two streams at a site agree, because + ``k`` is city-wide and they superpose whether they agree or not. + + Not the *generated* mix. Composting and anaerobic digestion draw only on + organics and recycling only on recyclables, so the residual differs from + what the city generates in shape and not merely in scale; splitting a + gate-observed tonnage by the generated mix overstates its degradable half + and with it the methane. + + ``fractions`` is the fallback for a year the city buries nothing, which + would otherwise be 0/0. It must be the city's composition **as shares**, not + as masses: a year in which the city generates nothing has an all-zero mass + frame, so a mass fallback is no fallback at all -- it leaves every share at + zero and the surplus is then deposited as nothing. That is not a hypothetical + tidy-up. A site taking regional waste while the city it belongs to collects + none yet is an ordinary shape, and it silently lost every ton of it. + """ + totals = residual.sum(axis=1) + shares = residual.div(totals, axis=0) + + empty = totals <= 0 + if empty.any(): + fallback_totals = fractions.sum(axis=1) + fallback = fractions.div(fallback_totals.where(fallback_totals > 0), axis=0) + shares.loc[empty, :] = fallback.loc[empty, :] + + return shares.fillna(0.0) + + +def surplus_intake( + accepted: pd.Series, city_intake: pd.Series +) -> pd.Series: + """Tons per year reaching a site's gate from outside this baseline. + + Compared at the *gate*, before combustion and before the open/close window, + because that is where the stated total was measured: a site that burns its + intake still accepted all of it. + + Floored at zero, which is the answer to a stated total *below* what the city + sends. That is a real condition -- a gate figure and an allocation are + measured by different people in different years -- and it is not a licence + to shrink the city's stream. The city's waste has to go somewhere, this + feature does not add anywhere else for it to go, and a city's emissions are + not reduced by a neighbour's paperwork. So the site simply has no outside + waste, and the caller is free to show the city's figure as the floor it is. + """ + return (accepted - city_intake).clip(lower=0.0) + + +def surplus_masses( + surplus: pd.Series, composition: pd.DataFrame +) -> pd.DataFrame: + """The surplus tonnage split by component, at the city's residual mix.""" + return composition.mul(surplus, axis=0) + + +def adopt_city_params(source: Landfill, *surplus: Landfill) -> None: + """Give the surplus landfills their site's own ``city_params_dict``. + + ``CityParameters.repopulate_attr_dicts`` pushes a freshly dumped parameter + dict onto every landfill in ``parameters.landfills`` just before the engine + runs. The surplus landfills are deliberately not in that list -- that list is + what the city's total is summed over -- so they miss that pass, and would + otherwise decay against whatever dict they were constructed with. + + Copying the object off the site's city-stream landfill, rather than + rebuilding it, is what makes them exactly equal: two streams at one site + must agree on every non-mass parameter or they do not superpose, and an + equality that has to be maintained by hand is one that will eventually drift. + + Mirrors all three attributes ``repopulate_attr_dicts`` sets, not just the + first. The other two only exist once a landfill has run, which a surplus + stream has not yet -- but copying one of three is the kind of near-miss that + survives a refactor and then stops being harmless. + """ + for landfill in surplus: + landfill.city_params_dict = source.city_params_dict + if hasattr(landfill, "model"): + landfill.model.city_params_dict = source.city_params_dict + landfill.model.landfill_instance_attrs = landfill.model_dump() + + +def require_linear(landfill: Landfill) -> None: + """Refuse to split a site whose emissions are not linear in deposited mass. + + ``Landfill.doing_fancy_ox`` is hardcoded ``False`` and its body derives an + oxidation factor from one year's available methane and then clips it three + times -- genuinely nonlinear in mass. Everything here assumes it stays off: + with it on, two streams at a site stop superposing and "the city's share" + silently becomes an allocation convention rather than a measurement, with no + error and no visible symptom. + + So this raises rather than warns, and it is a runtime check rather than a + test: the flag is an attribute anybody can set, and a test only fails for + whoever runs the suite. + """ + if getattr(landfill, "doing_fancy_ox", False): + raise CustomError( + "nonlinear_oxidation", + "This site uses CALMIM oxidation, whose emissions are not linear in " + "deposited mass, so its waste cannot be split by source.", + ) + + +def emission_frames( + city_stream: Landfill, surplus_stream: Optional[Landfill] +) -> SiteEmissions: + """One site's emissions, split by whose waste produced them.""" + require_linear(city_stream) + city = _to_tons_ch4(city_stream.emissions) + + if surplus_stream is None or surplus_stream.emissions is None: + other = city * 0.0 + else: + # Both streams were built on the same open year, so the kernel gave them + # the same index -- but reindex rather than trust it, because a silent + # NaN here would read as a plausible number downstream. + other = _to_tons_ch4(surplus_stream.emissions).reindex( + index=city.index, columns=city.columns, fill_value=0.0 + ) + + return SiteEmissions(city=city, other=other, total=city + other) + + +def _to_tons_ch4(emissions: pd.DataFrame) -> pd.DataFrame: + """m^3 of methane to tons of it, the way the city's own total is converted. + + ``City.sum_landfill_emissions`` converts to tons of CO2e and then divides by + 28, which lands back on tons of methane. Reproduced rather than simplified + so that a per-site figure and the city figure it has to reconcile with pass + through identical arithmetic -- including the same rounding. + """ + return emissions.map(City.convert_methane_m3_to_ton_co2e) / 28 diff --git a/changelog/2026-09.md b/changelog/2026-09.md index d171107..4b52fe6 100644 --- a/changelog/2026-09.md +++ b/changelog/2026-09.md @@ -47,6 +47,61 @@ landfill received — instead of re-deriving it. ## Added +- **A disposal site can accept waste the city did not send it.** `CityLandfillSpec` + takes an optional `accepted_waste_mass`: everything crossing that site's + weighbridge in a year, from every source. `landfill_split_timeline` is + fractions that must sum to 1, so until now every ton at every site came from + the city by construction — a regional landfill taking waste from a + neighbouring municipality was structurally unsayable, and site operators know + their gate total rather than one city's share of it. + + Whatever the stated total exceeds the city's own allocation is modelled as a + second stream at the same site: its own `Landfill`, its own pass through the + decay kernel, deposited and windowed on exactly the same terms as the city's + waste because it is the same hole in the ground. A figure at or below the + city's allocation is read as "no waste from outside" rather than as a + reduction of the city's — the city's waste still has to go somewhere, and this + does not add anywhere else for it to go. + + **The city's own emissions do not move**, and the reason is structural rather + than arithmetic: `City.sum_landfill_emissions` sums `parameters.landfills`, + and the surplus landfills are never put in that list. The test asserts frame + equality, not a tolerance. Every request without the field takes the path it + took before, with `city_params.py` untouched. + + `run_advanced_dst_city(request, with_site_emissions=True)` returns the split: + per landfill in request order, `city`, `other` and `total` in tons of methane + per year. The two are separate kernel runs summed, never a ratio applied to + one of them — this year's emissions come from decades of deposit cohorts each + with their own split, and attributing by this year's tonnage was measured at + up to 28% error where a site's mix of sources moves over time. A test pins + that case so the shortcut cannot be reintroduced quietly. + + Superposition is exact because the kernel is linear in deposited mass + (measured at ~1e-16 relative; `E(0)` is exactly `0.0`), so "the city's share + of this site's methane" is a measured quantity rather than an allocation + convention. `Landfill.doing_fancy_ox` is the one thing that would break it — + its body is genuinely nonlinear in mass — so the split now refuses to run + rather than silently returning an attribution nobody could defend. + + The surplus carries the city's post-diversion residual composition. That is a + modelling default, not a numerical constraint: `k` is city-wide, so a stream's + own mix moves only its per-component masses. It is the default because a gate + observation is downstream of whatever diversion happened upstream of it. + `site_inflow.residual_composition` is the single function a later "a site + states its own mix" change replaces. In a year the city buries nothing that + mix is 0/0, and the fallback is the composition **as shares** — the + composition as masses is all-zero in exactly the years it would be needed, + which left the surplus with a zero mix and deposited none of it. A site taking + regional waste while its city has not started collecting yet is an ordinary + shape, and it silently lost every ton: 200,000 t/yr over thirty years came + back as 0 t CH4. Pinned by test. + + New behaviour lives in `SWEET_python/site_inflow.py`, following the convention + `dst_common.build_landfill` already sets — a landfill's construction lives + outside `landfill.py` — rather than growing the 9,000-line `city_params.py`. + ([#65](https://github.com/RMI/SWEET_python/pull/65)) + - **`run_advanced_dst_city_limits` — what a city's composition allows.** Takes the same request as `run_advanced_dst_city` and returns, per year: the fraction of each waste component's own mass the diversion pathways claim, the diff --git a/changelog/README.md b/changelog/README.md index 0393293..992f2b9 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -15,7 +15,7 @@ The project does not publish semantic version tags, so releases are tracked by Newest first: -- [2026-09](2026-09.md) — Model output frames get one fixed waste-type column order instead of a per-process one (a `set` fed the ordering, and Python randomizes set iteration per process), moving values in the last bit only, where a sum runs across the reordered columns; City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart +- [2026-09](2026-09.md) — A disposal site can accept waste the city did not send it (`accepted_waste_mass`), modelled as a second stream at the same site, with per-site emissions returned split into the city's share and the rest; the city's own total is untouched by construction. Model output frames get one fixed waste-type column order instead of a per-process one (a `set` fed the ordering, and Python randomizes set iteration per process), moving values in the last bit only, where a sum runs across the reordered columns; City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart - [2026-08](2026-08.md) — Single-site adst gains optional `depth` (deep-dump MCF bump) and `k_override` (caller-supplied decomposition rate) inputs, restoring the last two site-DST levers; `/sdst` flaring efficiency reaches the model again after a variable-name bug silently forced flare destruction to 0.98; annual model applies cover oxidation by emission year not deposit year, fixing biocover having no effect on closed landfills (WasteMAP #719); `City.sdst_v1_5` custom-site path holds the scenario equal to the baseline before the implementation year even when composition changes (was back-dating the new composition onto pre-implementation deposits) (model-output change); MCF consolidated into a new `SWEET_python.mcf` module and both dump types moved to the IPCC uncategorised-SWDS 0.6 (open dumps up from 0.4, controlled dumps down from 0.7), with a supplied depth now selecting the deep/shallow category (model-output change) - [2026-07](2026-07.md) — All ten waste types eligible for combustion (metal/glass/other added); methane-only model treats combustion as landfill diversion (model-output change) - [2026-06](2026-06.md) — New single-site and city-level ADST modeling modules, min-cost max-flow rewrite of the city DST diversion allocator, physical-k fix for cold/dry sites, no more spurious negative food-waste mass diff --git a/tests/test_site_other_source.py b/tests/test_site_other_source.py new file mode 100644 index 0000000..e91ae32 --- /dev/null +++ b/tests/test_site_other_source.py @@ -0,0 +1,469 @@ +"""A named site can receive waste the city did not send. + +Until now a city's disposal sites shared out exactly the city's landfilled +residual: ``landfill_split_timeline`` is *fractions that must sum to 1.0* +(``_validate_shares``), so a site receiving mass from anywhere else was +structurally unsayable. Real sites take waste from neighbouring municipalities, +from private haulers, from a regional catchment the baseline never describes -- +and the site's own operator knows its gate total, not the city's share of it. + +So a spec may now carry ``accepted_waste_mass``: everything crossing that site's +weighbridge in a year, **from every source**, city waste included. It is a gate +total, not the outside share -- the outside share is what it exceeds the city's +own allocation by, which is the model's subtraction to do rather than the user's. +A figure at or below that allocation therefore means "no outside waste", not a +smaller city stream. Two consequences, and this module exists to pin both: + +**The city's own emissions must not move.** A city is responsible for the waste +it generates, wherever that waste goes. Other-source waste is somebody else's +inventory. The guarantee is structural rather than arithmetic: the other-source +landfills are never added to ``parameters.landfills``, which is the list +``City.sum_landfill_emissions`` iterates -- so the city total is bit-identical, +not merely close. + +**A site's total is the sum of its streams, exactly.** SWEET's landfill +emissions are exactly linear in deposited mass -- deposited mass enters the +first-order-decay kernel once, multiplicatively (``model_v2`` ``ch4_produce = +ks_values * L_0[waste] * waste_masses * exp_term * mcf_values``), and every step +after it is a mass-independent factor. Measured here at ~1e-16 relative. That is +what makes "the city's share of this site's emissions" a physical quantity +rather than an allocation convention someone had to invent. + +Linearity needs the two streams to agree on every parameter but mass, which the +implementation gets by building the surplus twin from the city stream's own +kwargs. Composition is not one of those parameters: ``k`` is computed once from +the city's generated mix and handed to every landfill, so a stream's own mix +moves only its per-component masses. The surplus carries the city's +post-diversion residual mix because that is the defensible reading of a gate +observation, not because superposition would otherwise fail. +""" + +import numpy as np +import pytest + +from SWEET_python.advanced_dst_city import ( + AdvancedDSTCityRequest, + run_advanced_dst_city, +) + +YEARS = list(range(1990, 2051)) +# Food-heavy, so diversion has something to bite on and the residual +# composition is visibly different from the generated one. +FRACTIONS = [0.5, 0.1, 0.05, 0.1, 0.05, 0.1, 0.02, 0.03, 0.0, 0.05] +IMPLEMENT_YEAR = 2025 +GENERATED = 100_000.0 + + +def _spec( + *, + open_close=(1990, 2051), + accepted=None, + accepted_scenario=None, + combusts=None, + combusts_scenario=None, +): + spec = dict( + landfill_type={"baseline": 2, "scenario": 2}, + landfill_open_close={"baseline": list(open_close), "scenario": list(open_close)}, + gas_capture_efficiency={ + "baseline": {y: 0.0 for y in YEARS}, + "scenario": {y: 0.0 for y in YEARS}, + }, + ) + if accepted is not None: + spec["accepted_waste_mass"] = { + "baseline": dict(accepted), + "scenario": dict(accepted_scenario if accepted_scenario is not None else accepted), + } + if combusts is not None or combusts_scenario is not None: + spec["combusts"] = { + "baseline": bool(combusts), + "scenario": bool(combusts if combusts_scenario is None else combusts_scenario), + } + return spec + + +def _request(specs, shares, *, diversion=None, generated=None): + extra = {} + if diversion is not None: + extra["diversion_fractions"] = { + "baseline": {"compost": {y: diversion for y in YEARS}}, + "scenario": {"compost": {y: diversion for y in YEARS}}, + } + return AdvancedDSTCityRequest( + city_name="Overflow", + precipitation=1200.0, + temperature=20.0, + implement_year=IMPLEMENT_YEAR, + waste_mass={ + "baseline": dict(generated) if generated else {y: GENERATED for y in YEARS}, + "scenario": dict(generated) if generated else {y: GENERATED for y in YEARS}, + }, + waste_fractions={ + "baseline": {y: FRACTIONS for y in YEARS}, + "scenario": {y: FRACTIONS for y in YEARS}, + }, + landfills=specs, + landfill_split_timeline={ + "baseline": {y: list(shares) for y in YEARS}, + "scenario": {y: list(shares) for y in YEARS}, + }, + country="USA", + **extra, + ) + + +def _total(frame): + return np.asarray(frame["total"], dtype=float) + + +# `accepted_waste_mass` is the whole gate total, not the neighbour's share of +# it, so every fixture here has to clear whatever the city sends. With 25% +# composted the city buries ~78,154 t/yr of its 100,000 t; a site taking the +# city's entire residual and 100,000 t is therefore comfortably over. +GATE_WITH_NEIGHBOUR = {y: 100_000.0 for y in YEARS} + +# A gate total that GROWS away from the city's flat allocation, so the city's +# share of the site falls year on year. Any attribution built on a same-year +# tonnage ratio gets this case badly wrong; see the last test. +GATE_GROWING = {y: 100_000.0 + 3_000.0 * (1.05 ** (y - 1990)) for y in YEARS} + + +# --------------------------------------------------------------------------- # +# The city's emissions are the city's waste, wherever it goes +# --------------------------------------------------------------------------- # + +def test_other_source_waste_does_not_change_the_city_total(): + """The whole feature, in one assertion. Bit-identical, not merely close.""" + without = run_advanced_dst_city(_request([_spec()], (1.0,), diversion=0.25)) + with_other = run_advanced_dst_city( + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,), diversion=0.25) + ) + + for variant in ("baseline", "scenario"): + assert without[variant].equals(with_other[variant]), ( + f"{variant}: other-source waste moved the city's own emissions" + ) + + +def test_an_absent_field_and_a_zero_series_are_the_same_request(): + """No silent second code path for the city that never uses this.""" + absent = run_advanced_dst_city(_request([_spec()], (1.0,), diversion=0.25)) + zeros = run_advanced_dst_city( + _request([_spec(accepted={y: 0.0 for y in YEARS})], (1.0,), diversion=0.25) + ) + + for variant in ("baseline", "scenario"): + assert absent[variant].equals(zeros[variant]) + + +def test_site_shares_still_have_to_sum_to_one(): + """Other-source mass is additive, not an escape hatch from the split. + + The city's landfilled waste still goes entirely to the city's own sites -- + this feature opens the *site* side, not the city side. + """ + from SWEET_python.city_params import CustomError + + with pytest.raises(CustomError): + run_advanced_dst_city( + _request([_spec(accepted=GATE_WITH_NEIGHBOUR), _spec()], (0.5, 0.2)) + ) + + +# --------------------------------------------------------------------------- # +# A site's total is the sum of its streams +# --------------------------------------------------------------------------- # + +def test_a_sites_emissions_split_into_the_city_and_the_rest(): + result = run_advanced_dst_city( + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,), diversion=0.25), + with_site_emissions=True, + ) + site = result["site_emissions"]["baseline"][0] + + city, other, total = _total(site.city), _total(site.other), _total(site.total) + residual = np.abs(total - (city + other)) + scale = np.where(total == 0, 1.0, total) + + assert np.max(residual / scale) < 1e-12, "a site's streams do not add to its total" + assert other.sum() > 0, "the neighbour's waste produced no emissions" + + +def test_the_city_contribution_is_what_the_city_alone_would_have_produced(): + """Superposition, stated as the thing a reader actually wants to trust. + + The city's slice of a shared site emits exactly what it would emit at a site + nobody else used. If this fails, the attribution is a convention rather than + a measurement. + """ + alone = run_advanced_dst_city( + _request([_spec()], (1.0,), diversion=0.25), with_site_emissions=True + ) + shared = run_advanced_dst_city( + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,), diversion=0.25), + with_site_emissions=True, + ) + + a = _total(alone["site_emissions"]["baseline"][0].city) + b = _total(shared["site_emissions"]["baseline"][0].city) + scale = np.where(a == 0, 1.0, a) + + assert np.max(np.abs(a - b) / scale) < 1e-12 + + +def test_a_site_with_no_other_source_reports_a_total_equal_to_its_city_share(): + result = run_advanced_dst_city( + _request([_spec()], (1.0,), diversion=0.25), with_site_emissions=True + ) + site = result["site_emissions"]["baseline"][0] + + assert np.allclose(_total(site.other), 0.0) + assert np.allclose(_total(site.total), _total(site.city), rtol=0, atol=1e-12) + + +def test_the_city_shares_of_every_site_add_up_to_the_city_total(): + """The two outputs reconcile: city emissions are the per-site city shares + plus whatever the diversion pathways themselves emit.""" + request = _request( + [_spec(accepted=GATE_WITH_NEIGHBOUR), _spec(accepted=GATE_GROWING)], + (0.6, 0.4), + diversion=0.25, + ) + result = run_advanced_dst_city(request, with_site_emissions=True) + + per_site = sum( + _total(site.city) for site in result["site_emissions"]["baseline"] + ) + city_total = _total(result["baseline"]) + + # The gap is the compost/anaerobic emissions, which are the city's but are + # not at any landfill. It must be non-negative and smooth, not noise. + diversion_share = city_total - per_site + assert np.all(diversion_share > -1e-9) + assert diversion_share.sum() > 0 + + +# --------------------------------------------------------------------------- # +# The bug this feature is most likely to be "simplified" into +# --------------------------------------------------------------------------- # + +def test_the_city_share_is_not_this_years_tonnage_ratio(): + """Do NOT reimplement attribution as (city tons / site tons) x site total. + + This year's emissions come from decades of deposit cohorts, each with its own + city/other split. Where the split moves over time -- a growing neighbour, a + city that started diverting -- the same-year ratio is wrong by tens of + percent, and wrong in a direction that looks plausible. The kernel has to run + on the city's own deposit series, which is what the implementation does by + giving each stream its own Landfill object. + + This test fails if someone replaces that with the ratio. + """ + result = run_advanced_dst_city( + _request([_spec(accepted=GATE_GROWING)], (1.0,), diversion=0.0), + with_site_emissions=True, + ) + site = result["site_emissions"]["baseline"][0] + city, total = _total(site.city), _total(site.total) + + settled = total > 0.01 * total.max() + emission_share = (city / np.where(total == 0, 1.0, total))[settled] + + # No diversion in this run, so the city's allocation is its whole stream. + city_tons = np.array([GENERATED for _ in YEARS]) + gate_tons = np.array([GATE_GROWING[y] for y in YEARS]) + mass_share = (city_tons / gate_tons)[settled] + + # The two shares must visibly disagree: emissions lag mass, so a shrinking + # city holds a larger share of the emissions than of this year's intake. + assert np.max(np.abs(emission_share - mass_share)) > 0.02, ( + "mass share and emission share agree -- either the test city is too " + "static to be a regression guard, or attribution has been reduced to a ratio" + ) + assert np.all(emission_share[-10:] > mass_share[-10:]) + + +# --------------------------------------------------------------------------- # +# Assumptions the rest of the engine is entitled to keep making +# --------------------------------------------------------------------------- # + +def test_no_surplus_emits_exactly_zero(): + """`==`, not `approx`. E(0) is exactly 0.0, so any residual is a bug.""" + result = run_advanced_dst_city( + _request([_spec(accepted={y: 0.0 for y in YEARS})], (1.0,)), + with_site_emissions=True, + ) + + assert (_total(result["site_emissions"]["baseline"][0].other) == 0.0).all() + + +def test_the_mass_flows_sites_band_is_still_only_the_city(): + """`sum(sites) == landfilled` is asserted in eleven places elsewhere. + + The band reports where the *city's* waste went, and other-source waste is + not the city's. Folding it in here would be the easy mistake, and it would + break the one identity the mass flow exists to keep. + """ + plain = run_advanced_dst_city(_request([_spec()], (1.0,)), with_mass_flow=True) + with_other = run_advanced_dst_city( + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,)), with_mass_flow=True + ) + + for variant in ("baseline", "scenario"): + a = plain["mass_flow"][variant] + b = with_other["mass_flow"][variant] + assert a["sites"][0].equals(b["sites"][0]) + assert a["landfilled"].equals(b["landfilled"]) + + +def test_a_combusting_site_burns_everything_it_accepts(): + """The surplus meets the same furnace the city's waste does. + + Compared at the gate but deposited after the burn, so a site stated at + 150,000 t buries the reject of 150,000 t -- not 150,000 t of residue. + """ + burning = run_advanced_dst_city( + _request( + [_spec(accepted=GATE_WITH_NEIGHBOUR, combusts=True)], + (1.0,), + diversion=0.25, + ), + with_site_emissions=True, + ) + depositing = run_advanced_dst_city( + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,), diversion=0.25), + with_site_emissions=True, + ) + + burnt = _total(burning["site_emissions"]["baseline"][0].other) + kept = _total(depositing["site_emissions"]["baseline"][0].other) + + assert burnt.sum() > 0, "an incinerator's residue still decays" + # The unburnable reject is 10%, the same rate the combustion pathway uses. + assert burnt.sum() == pytest.approx(kept.sum() * 0.1, rel=1e-9) + + +def test_the_limits_endpoint_ignores_the_new_field(): + """Bounds are about what the city generates, which a gate total says + nothing about.""" + from SWEET_python.advanced_dst_city import run_advanced_dst_city_limits + + plain = run_advanced_dst_city_limits(_request([_spec()], (1.0,))) + with_other = run_advanced_dst_city_limits( + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,)) + ) + + assert plain.keys() == with_other.keys() + for variant in plain: + assert str(plain[variant]) == str(with_other[variant]) + + +# --------------------------------------------------------------------------- # +# The mix the surplus is given, where the city has none to lend it +# --------------------------------------------------------------------------- # + +def test_outside_waste_still_has_a_composition_when_the_city_generates_none(): + """A regional site whose city has not started collecting yet. + + The surplus is split by the city's post-diversion residual mix, and in a + year the city buries nothing that mix is 0/0. The fallback has to be the + composition as *shares*: the composition as masses is all-zero in exactly + the years it would be needed, which left every share at zero and deposited + none of the site's inflow. Thirty years of a 200,000 t/yr site vanished, and + nothing said so. + """ + late = {y: (0.0 if y < 2030 else GENERATED) for y in YEARS} + result = run_advanced_dst_city( + _request( + [_spec(accepted={y: 200_000.0 for y in YEARS})], + (1.0,), + generated=late, + ), + with_site_emissions=True, + ) + site = result["site_emissions"]["baseline"][0] + other = _total(site.other) + before = np.array([other[i] for i, y in enumerate(YEARS) if y < 2030]) + + assert before.sum() > 0, "the site's own inflow produced no methane at all" + # It is the whole of the site in those years, the city having sent nothing. + city = _total(site.city) + city_before = np.array([city[i] for i, y in enumerate(YEARS) if y < 2030]) + assert np.allclose(city_before, 0.0) + assert np.allclose( + before, + np.array([_total(site.total)[i] for i, y in enumerate(YEARS) if y < 2030]), + ) + + +# --------------------------------------------------------------------------- # +# The scenario half is spliced to baseline before the implement year +# --------------------------------------------------------------------------- # + +def test_a_changed_gate_total_takes_effect_only_from_the_implement_year(): + """The surplus follows the same baseline-until-implement rule as everything else. + + The splice that enforces this for a changed *intake* is upstream, in + ``dst_common.variant_series`` -- it builds the scenario series equal to + baseline before ``implement_year`` already. This pins the behaviour end to + end rather than the line that implements it; the line is pinned by + ``test_a_site_that_starts_burning_...`` below, which exercises a difference + ``variant_series`` cannot see. + """ + doubled = {y: 300_000.0 for y in YEARS} + result = run_advanced_dst_city( + _request( + [_spec(accepted=GATE_WITH_NEIGHBOUR, accepted_scenario=doubled)], + (1.0,), + ), + with_site_emissions=True, + ) + baseline_other = _total(result["site_emissions"]["baseline"][0].other) + scenario_other = _total(result["site_emissions"]["scenario"][0].other) + + before = [i for i, y in enumerate(YEARS) if y < IMPLEMENT_YEAR] + after = [i for i, y in enumerate(YEARS) if y >= IMPLEMENT_YEAR] + + # Identical deposits before the implement year, so identical emissions. + assert np.allclose( + baseline_other[before], scenario_other[before], rtol=0, atol=1e-9 + ), "the scenario rewrote history" + # And strictly more afterwards, since the scenario takes 3x the waste. The + # gap opens gradually: the extra tonnage has to decay before it shows up. + assert scenario_other[after[-1]] > baseline_other[after[-1]] * 1.5 + + +def test_a_site_that_starts_burning_still_buries_the_surplus_until_then(): + """The frame-level splice, on a difference the series-level one cannot see. + + ``variant_series`` splices the intake, so a variant-differing gate total is + already baseline-before-implement by the time this module sees it. Two + things are not: the open/close window and ``combusts``, both applied to the + mass frame afterwards. A site that starts incinerating at the implement year + must still deposit the surplus in full before then -- and without the splice + it deposits only the 10% reject for the whole run, quietly erasing 90% of + decades of history that the scenario is not supposed to be able to rewrite. + """ + result = run_advanced_dst_city( + _request( + # 200 kt against an undiverted 100 kt city, so half the gate is + # surplus and there is something for the burning to bite on. + [_spec(accepted={y: 200_000.0 for y in YEARS}, combusts=False, combusts_scenario=True)], + (1.0,), + ), + with_site_emissions=True, + ) + baseline_other = _total(result["site_emissions"]["baseline"][0].other) + scenario_other = _total(result["site_emissions"]["scenario"][0].other) + + before = [i for i, y in enumerate(YEARS) if y < IMPLEMENT_YEAR] + after = [i for i, y in enumerate(YEARS) if y >= IMPLEMENT_YEAR] + + assert baseline_other[before].sum() > 0, "nothing was buried to compare" + assert np.allclose( + baseline_other[before], scenario_other[before], rtol=0, atol=1e-9 + ), "burning from the implement year reached back and unburied earlier waste" + # And from the implement year the scenario buries only the reject, so its + # emissions fall away from baseline's. + assert scenario_other[after[-1]] < baseline_other[after[-1]]