From a2e7b3b538e7c9a6596d62b0cbb0ca52784d6daf Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Wed, 9 Sep 2026 11:01:18 -0700 Subject: [PATCH 1/5] The contract for a site that receives waste the city did not send Tests only; the feature they describe does not exist yet, so they fail. Written first because the two guarantees are the whole design and are easy to lose to a plausible-looking shortcut: - the city's emissions must not move when other-source waste is added, which the implementation has to make structural rather than arithmetic; - a site's total must be its streams summed exactly, which is only a physical quantity because the kernel is linear in deposited mass. The last test is the regression guard for the shortcut: attributing by this year's tonnage ratio instead of running the kernel on the city's own deposit series. Measured here at up to 28% error where the city's share of a site moves over time. Co-Authored-By: Claude Opus 5 --- tests/test_site_other_source.py | 260 ++++++++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 tests/test_site_other_source.py diff --git a/tests/test_site_other_source.py b/tests/test_site_other_source.py new file mode 100644 index 0000000..a4682f8 --- /dev/null +++ b/tests/test_site_other_source.py @@ -0,0 +1,260 @@ +"""A named site can receive waste the city did not send. + +Until now a city's disposal sites shared out exactly the city's landfilled +residual: ``landfill_split_timeline`` is *fractions that must sum to 1.0* +(``_validate_shares``), so a site receiving mass from anywhere else was +structurally unsayable. Real sites take waste from neighbouring municipalities, +from private haulers, from a regional catchment the baseline never describes -- +and the site's own operator knows its gate total, not the city's share of it. + +So a spec may now carry ``other_source_mass``: absolute tons per year arriving +at that site from outside this baseline. Two consequences, and this module +exists to pin both: + +**The city's own emissions must not move.** A city is responsible for the waste +it generates, wherever that waste goes. Other-source waste is somebody else's +inventory. The guarantee is structural rather than arithmetic: the other-source +landfills are never added to ``parameters.landfills``, which is the list +``City.sum_landfill_emissions`` iterates -- so the city total is bit-identical, +not merely close. + +**A site's total is the sum of its streams, exactly.** SWEET's landfill +emissions are exactly linear in deposited mass -- deposited mass enters the +first-order-decay kernel once, multiplicatively (``model_v2`` ``ch4_produce = +ks_values * L_0[waste] * waste_masses * exp_term * mcf_values``), and every step +after it is a mass-independent factor. Measured here at ~1e-16 relative. That is +what makes "the city's share of this site's emissions" a physical quantity +rather than an allocation convention someone had to invent. + +The linearity has one precondition and it is load-bearing: every stream at a +site must share one ``k``. ``k`` is a step function of composition, so two +streams at one landfill with different mixes do not superpose. Other-source +waste therefore carries the city's own post-diversion residual composition -- +which is also the physically right answer, since a gate observation is +downstream of whatever diversion happened upstream of it. +""" + +import numpy as np +import pytest + +from SWEET_python.advanced_dst_city import ( + AdvancedDSTCityRequest, + run_advanced_dst_city, +) + +YEARS = list(range(1990, 2051)) +# Food-heavy, so diversion has something to bite on and the residual +# composition is visibly different from the generated one. +FRACTIONS = [0.5, 0.1, 0.05, 0.1, 0.05, 0.1, 0.02, 0.03, 0.0, 0.05] +IMPLEMENT_YEAR = 2025 +GENERATED = 100_000.0 + + +def _spec(*, open_close=(1990, 2051), other=None, combusts=None): + spec = dict( + landfill_type={"baseline": 2, "scenario": 2}, + landfill_open_close={"baseline": list(open_close), "scenario": list(open_close)}, + gas_capture_efficiency={ + "baseline": {y: 0.0 for y in YEARS}, + "scenario": {y: 0.0 for y in YEARS}, + }, + ) + if other is not None: + spec["other_source_mass"] = {"baseline": dict(other), "scenario": dict(other)} + if combusts is not None: + spec["combusts"] = {"baseline": combusts, "scenario": combusts} + return spec + + +def _request(specs, shares, *, diversion=None): + extra = {} + if diversion is not None: + extra["diversion_fractions"] = { + "baseline": {"compost": {y: diversion for y in YEARS}}, + "scenario": {"compost": {y: diversion for y in YEARS}}, + } + return AdvancedDSTCityRequest( + city_name="Overflow", + precipitation=1200.0, + temperature=20.0, + implement_year=IMPLEMENT_YEAR, + waste_mass={ + "baseline": {y: GENERATED for y in YEARS}, + "scenario": {y: GENERATED for y in YEARS}, + }, + waste_fractions={ + "baseline": {y: FRACTIONS for y in YEARS}, + "scenario": {y: FRACTIONS for y in YEARS}, + }, + landfills=specs, + landfill_split_timeline={ + "baseline": {y: list(shares) for y in YEARS}, + "scenario": {y: list(shares) for y in YEARS}, + }, + country="USA", + **extra, + ) + + +def _total(frame): + return np.asarray(frame["total"], dtype=float) + + +# A neighbour whose tonnage RISES while the city's stays flat, so the city's +# share of the site falls over time. Any attribution built on a same-year ratio +# gets this case badly wrong; see the last test. +RISING_NEIGHBOUR = {y: 3_000.0 * (1.05 ** (y - 1990)) for y in YEARS} +FLAT_NEIGHBOUR = {y: 25_000.0 for y in YEARS} + + +# --------------------------------------------------------------------------- # +# The city's emissions are the city's waste, wherever it goes +# --------------------------------------------------------------------------- # + +def test_other_source_waste_does_not_change_the_city_total(): + """The whole feature, in one assertion. Bit-identical, not merely close.""" + without = run_advanced_dst_city(_request([_spec()], (1.0,), diversion=0.25)) + with_other = run_advanced_dst_city( + _request([_spec(other=FLAT_NEIGHBOUR)], (1.0,), diversion=0.25) + ) + + for variant in ("baseline", "scenario"): + assert without[variant].equals(with_other[variant]), ( + f"{variant}: other-source waste moved the city's own emissions" + ) + + +def test_an_absent_field_and_a_zero_series_are_the_same_request(): + """No silent second code path for the city that never uses this.""" + absent = run_advanced_dst_city(_request([_spec()], (1.0,), diversion=0.25)) + zeros = run_advanced_dst_city( + _request([_spec(other={y: 0.0 for y in YEARS})], (1.0,), diversion=0.25) + ) + + for variant in ("baseline", "scenario"): + assert absent[variant].equals(zeros[variant]) + + +def test_site_shares_still_have_to_sum_to_one(): + """Other-source mass is additive, not an escape hatch from the split. + + The city's landfilled waste still goes entirely to the city's own sites -- + this feature opens the *site* side, not the city side. + """ + from SWEET_python.city_params import CustomError + + with pytest.raises(CustomError): + run_advanced_dst_city( + _request([_spec(other=FLAT_NEIGHBOUR), _spec()], (0.5, 0.2)) + ) + + +# --------------------------------------------------------------------------- # +# A site's total is the sum of its streams +# --------------------------------------------------------------------------- # + +def test_a_sites_emissions_split_into_the_city_and_the_rest(): + result = run_advanced_dst_city( + _request([_spec(other=FLAT_NEIGHBOUR)], (1.0,), diversion=0.25), + with_site_emissions=True, + ) + site = result["site_emissions"]["baseline"][0] + + city, other, total = _total(site["city"]), _total(site["other"]), _total(site["total"]) + residual = np.abs(total - (city + other)) + scale = np.where(total == 0, 1.0, total) + + assert np.max(residual / scale) < 1e-12, "a site's streams do not add to its total" + assert other.sum() > 0, "the neighbour's waste produced no emissions" + + +def test_the_city_contribution_is_what_the_city_alone_would_have_produced(): + """Superposition, stated as the thing a reader actually wants to trust. + + The city's slice of a shared site emits exactly what it would emit at a site + nobody else used. If this fails, the attribution is a convention rather than + a measurement. + """ + alone = run_advanced_dst_city( + _request([_spec()], (1.0,), diversion=0.25), with_site_emissions=True + ) + shared = run_advanced_dst_city( + _request([_spec(other=FLAT_NEIGHBOUR)], (1.0,), diversion=0.25), + with_site_emissions=True, + ) + + a = _total(alone["site_emissions"]["baseline"][0]["city"]) + b = _total(shared["site_emissions"]["baseline"][0]["city"]) + scale = np.where(a == 0, 1.0, a) + + assert np.max(np.abs(a - b) / scale) < 1e-12 + + +def test_a_site_with_no_other_source_reports_a_total_equal_to_its_city_share(): + result = run_advanced_dst_city( + _request([_spec()], (1.0,), diversion=0.25), with_site_emissions=True + ) + site = result["site_emissions"]["baseline"][0] + + assert np.allclose(_total(site["other"]), 0.0) + assert np.allclose(_total(site["total"]), _total(site["city"]), rtol=0, atol=1e-12) + + +def test_the_city_shares_of_every_site_add_up_to_the_city_total(): + """The two outputs reconcile: city emissions are the per-site city shares + plus whatever the diversion pathways themselves emit.""" + request = _request( + [_spec(other=FLAT_NEIGHBOUR), _spec(other=RISING_NEIGHBOUR)], + (0.6, 0.4), + diversion=0.25, + ) + result = run_advanced_dst_city(request, with_site_emissions=True) + + per_site = sum( + _total(site["city"]) for site in result["site_emissions"]["baseline"] + ) + city_total = _total(result["baseline"]) + + # The gap is the compost/anaerobic emissions, which are the city's but are + # not at any landfill. It must be non-negative and smooth, not noise. + diversion_share = city_total - per_site + assert np.all(diversion_share > -1e-9) + assert diversion_share.sum() > 0 + + +# --------------------------------------------------------------------------- # +# The bug this feature is most likely to be "simplified" into +# --------------------------------------------------------------------------- # + +def test_the_city_share_is_not_this_years_tonnage_ratio(): + """Do NOT reimplement attribution as (city tons / site tons) x site total. + + This year's emissions come from decades of deposit cohorts, each with its own + city/other split. Where the split moves over time -- a growing neighbour, a + city that started diverting -- the same-year ratio is wrong by tens of + percent, and wrong in a direction that looks plausible. The kernel has to run + on the city's own deposit series, which is what the implementation does by + giving each stream its own Landfill object. + + This test fails if someone replaces that with the ratio. + """ + result = run_advanced_dst_city( + _request([_spec(other=RISING_NEIGHBOUR)], (1.0,), diversion=0.0), + with_site_emissions=True, + ) + site = result["site_emissions"]["baseline"][0] + city, total = _total(site["city"]), _total(site["total"]) + + settled = total > 0.01 * total.max() + emission_share = (city / np.where(total == 0, 1.0, total))[settled] + + city_tons = np.array([GENERATED for _ in YEARS]) + mass_share = (city_tons / (city_tons + np.array([RISING_NEIGHBOUR[y] for y in YEARS])))[settled] + + # The two shares must visibly disagree: emissions lag mass, so a shrinking + # city holds a larger share of the emissions than of this year's intake. + assert np.max(np.abs(emission_share - mass_share)) > 0.02, ( + "mass share and emission share agree -- either the test city is too " + "static to be a regression guard, or attribution has been reduced to a ratio" + ) + assert np.all(emission_share[-10:] > mass_share[-10:]) From 908c2b78678d36966bd8f139bde94a57da5dcb71 Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Wed, 9 Sep 2026 11:10:23 -0700 Subject: [PATCH 2/5] A site can accept waste the city did not send it `landfill_split_timeline` is fractions summing to 1, so until now every ton at every site came from the city by construction. A spec may now state `accepted_waste_mass` -- its whole gate total -- and whatever that exceeds the city's allocation is modelled as a second stream at the same site. The city's own emissions cannot move, and the reason is structural rather than arithmetic: `City.sum_landfill_emissions` sums `parameters.landfills`, and the surplus landfills are never put in that list. The regression test asserts frame equality, not a tolerance. A site's total is its streams summed exactly, because the kernel is linear in deposited mass (~1e-16 relative, E(0) == 0.0). That holds only while every stream at a site shares one `k`, which is a step function of composition -- so the surplus carries the city's post-diversion residual mix, which is also what a gate observation physically is. New behaviour lives in `site_inflow.py`, following the convention `dst_common.build_landfill` already sets: a landfill's construction lives outside `landfill.py`. `city_params.py` is untouched. Co-Authored-By: Claude Opus 5 --- SWEET_python/advanced_dst_city.py | 153 +++++++++++++++++++++++- SWEET_python/site_inflow.py | 187 ++++++++++++++++++++++++++++++ tests/test_site_other_source.py | 39 ++++--- 3 files changed, 358 insertions(+), 21 deletions(-) create mode 100644 SWEET_python/site_inflow.py diff --git a/SWEET_python/advanced_dst_city.py b/SWEET_python/advanced_dst_city.py index 1c70b47..6b79324 100644 --- a/SWEET_python/advanced_dst_city.py +++ b/SWEET_python/advanced_dst_city.py @@ -48,6 +48,7 @@ from SWEET_python.city_params import City, CityParameters, CustomError from SWEET_python.class_defs import DivsDF, LandfillType, LandfillWasteMassDF, Variant from SWEET_python import dst_common as common +from SWEET_python import site_inflow from SWEET_python.dst_common import YearlyFloat, YearlyFractions __all__ = ["AdvancedDSTCityRequest", "CityLandfillSpec", "run_advanced_dst_city"] @@ -106,6 +107,28 @@ class CityLandfillSpec(BaseModel): "decays." ), ) + accepted_waste_mass: Optional[Variant[YearlyFloat]] = Field( + None, + description=( + "Total waste crossing this site's weighbridge each year, in tons, " + "from every source -- a fact measured at the gate rather than a " + "share of anything. Omit (the default) and the site receives " + "exactly the city's allocation, which is the existing behaviour.\n\n" + "This does not replace ``landfill_split_timeline``, which still " + "divides the city's own landfilled waste across the city's sites " + "and must still sum to ~1 per year. Whatever this figure exceeds " + "the city's allocation is modeled as a second stream arriving from " + "outside the baseline: it carries the city's post-diversion " + "residual composition, and is combusted and windowed on exactly the " + "same terms as the city's stream, because it is the same hole in " + "the ground.\n\n" + "It never changes the city's own emissions -- a city answers for " + "the waste it generates, wherever that waste goes. A figure *below* " + "the city's allocation is read as 'no waste from outside', not as a " + "reduction of the city's; see ``site_inflow.surplus_intake``. Ask " + "for the split with ``with_site_emissions=True``." + ), + ) class AdvancedDSTCityRequest(BaseModel): @@ -852,7 +875,10 @@ def run_advanced_dst_city_limits( # Public entry point # --------------------------------------------------------------------------- # def run_advanced_dst_city( - request: AdvancedDSTCityRequest, *, with_mass_flow: bool = False + request: AdvancedDSTCityRequest, + *, + with_mass_flow: bool = False, + with_site_emissions: bool = False, ) -> dict: """Run the city-level advanced DST. @@ -872,6 +898,17 @@ def run_advanced_dst_city( *show* the mass flow should read it from here rather than reimplementing the diversion split, which is per waste type and does not survive being approximated as a scalar on the total. + + With ``with_site_emissions=True`` the result carries a ``"site_emissions"`` + key: per variant, one entry per landfill in request order, each + ``{"city", "other", "total"}`` in tons of methane per year. ``city`` is the + methane this city's waste produced at that site and is what the city's own + total is built from; ``other`` is what waste from outside the baseline + produced there (see ``CityLandfillSpec.accepted_waste_mass``), and is zero + for a site that only takes the city's waste. They are two independent passes + through the decay kernel, summed -- never a ratio applied to one of them, + which is wrong by tens of percent wherever the city's share of a site moves + over time. Asking for them changes no emissions. """ implement_year = int(request.implement_year) @@ -965,6 +1002,15 @@ def run_advanced_dst_city( ) net_masses[label] = wgen.sub(divs.sum(), fill_value=0.0) + # The mix the city is left to bury, as per-year component shares. Any waste + # arriving at a site from outside the baseline is given this same mix, so + # that both streams at a site decay at one `k` and therefore superpose -- + # see `site_inflow` for why that is load-bearing rather than tidy. + residual_mix = { + "baseline": site_inflow.residual_composition(net_masses["baseline"], wgen_baseline), + "scenario": site_inflow.residual_composition(net_masses["scenario"], wgen_scenario), + } + # --- City-wide decomposition rates + compost emission factors --- ref_year = min(max(implement_year, int(years.min())), int(years.max())) ks_baseline, ks_scenario = common.decomposition_rates( @@ -1024,6 +1070,15 @@ def run_advanced_dst_city( scenario_masses: List[pd.DataFrame] = [] baseline_ox: List[pd.Series] = [] scenario_ox: List[pd.Series] = [] + # One entry per site, `None` where the site takes only the city's waste. + # Deliberately kept out of `*_parameters.landfills`: that list is what + # `City.sum_landfill_emissions` sums, and a city does not answer for a + # neighbour's waste. Keeping these out of it is the whole reason the city's + # total is bit-identical rather than merely close. + baseline_surplus: List[Optional[object]] = [] + scenario_surplus: List[Optional[object]] = [] + baseline_surplus_masses: List[Optional[pd.DataFrame]] = [] + scenario_surplus_masses: List[Optional[pd.DataFrame]] = [] for index, spec in enumerate(request.landfills): base_type = int(spec.landfill_type["baseline"]) @@ -1064,6 +1119,24 @@ def run_advanced_dst_city( # Net-of-diversion city waste, scaled to this landfill's per-year share. mass_base = LandfillWasteMassDF.create_advanced(wgen_baseline, divs_baseline, share_base.copy()).df mass_scen = LandfillWasteMassDF.create_advanced(wgen_scenario, divs_scenario, share_scen.copy()).df + + # What the city sends here, at the gate -- read before the combustion + # and window lines below, because a stated gate total is measured at the + # weighbridge and a site that burns its intake still accepted all of it. + surplus_base_masses = surplus_scen_masses = None + if spec.accepted_waste_mass is not None: + accepted_base, accepted_scen = common.variant_series( + spec.accepted_waste_mass, years, implement_year, default=0.0 + ) + surplus_base_masses = site_inflow.surplus_masses( + site_inflow.surplus_intake(accepted_base, mass_base.sum(axis=1)), + residual_mix["baseline"], + ) + surplus_scen_masses = site_inflow.surplus_masses( + site_inflow.surplus_intake(accepted_scen, mass_scen.sum(axis=1)), + residual_mix["scenario"], + ) + # A combusting facility burns its intake and deposits only the reject. # Scaled per variant here, ahead of both the window and the pre-implement # splice below, so a site that starts combusting at implement_year keeps @@ -1082,13 +1155,41 @@ def run_advanced_dst_city( baseline_masses.append(mass_base) scenario_masses.append(mass_scen) - baseline_landfills.append(common.build_landfill( + # The surplus is the same waste in the same hole, so it takes the same + # three transformations in the same order. Anything else and the two + # streams disagree at a boundary -- a closure year, an implement year, + # the year a site starts burning -- which is exactly where a reader + # would notice and could not explain it. + if surplus_base_masses is not None: + surplus_base_masses = surplus_base_masses * _deposited_share(base_combusts, city) + surplus_scen_masses = surplus_scen_masses * _deposited_share(scen_combusts, city) + surplus_base_masses = common.apply_window(surplus_base_masses, b_open, b_close) + surplus_scen_masses = common.apply_window(surplus_scen_masses, s_open, s_close) + surplus_scen_masses.loc[: implement_year - 1, :] = surplus_base_masses.loc[ + : implement_year - 1, : + ] + # A stated total at or below the city's allocation in every year + # leaves nothing to model, and running the kernel on an all-zero + # frame would only produce an all-zero frame. + if not ( + surplus_base_masses.to_numpy().any() or surplus_scen_masses.to_numpy().any() + ): + surplus_base_masses = surplus_scen_masses = None + baseline_surplus_masses.append(surplus_base_masses) + scenario_surplus_masses.append(surplus_scen_masses) + + # Lifted into a dict so the surplus stream below is built from the very + # same arguments rather than a copy of them that has to be kept in step: + # two streams at one site must agree on every parameter but their mass, + # or they decay differently and stop adding up to the site. + base_landfill_kwargs = dict( open_year=b_open, close_year=b_close, site_type_idx=base_type, mcf=mcf_base, gas_capture_efficiency=gas_base, flaring=flare_base, oxidation_factor=ox_base, ks=ks_baseline, city_params_dict=baseline_params_dict, city_instance_attrs=city_instance_attrs, implement_year=implement_year, scenario=0, landfill_index=index, - )) + ) + baseline_landfills.append(common.build_landfill(**base_landfill_kwargs)) # The model evaluates from `open_date` onward (`model_v2.estimate_emissions2` # builds its year range from it), so the scenario has to start wherever # its mass frame can first be nonzero -- and the splice above puts @@ -1098,13 +1199,29 @@ def run_advanced_dst_city( # the scenario's emissions frame entirely, leaving the two halves a # different shape for a caller trying to subtract them. scenario_open_for_model = min(b_open, s_open) - scenario_landfills.append(common.build_landfill( + scen_landfill_kwargs = dict( open_year=scenario_open_for_model, close_year=s_close, site_type_idx=scen_type, mcf=mcf_scen, gas_capture_efficiency=gas_scen, flaring=flare_scen, oxidation_factor=ox_scen, ks=ks_scenario, city_params_dict=scenario_params_dict, city_instance_attrs=city_instance_attrs, implement_year=implement_year, scenario=1, landfill_index=index, - )) + ) + scenario_landfills.append(common.build_landfill(**scen_landfill_kwargs)) + + # The surplus twin. Same open year above all else: `model_v2` builds its + # year range from `open_date`, so a twin opened anywhere else returns a + # differently indexed frame and the site total silently gains NaNs where + # the two failed to line up. + baseline_surplus.append( + common.build_landfill(**base_landfill_kwargs) + if surplus_base_masses is not None + else None + ) + scenario_surplus.append( + common.build_landfill(**scen_landfill_kwargs) + if surplus_scen_masses is not None + else None + ) # --- Wire up and run the engine --- baseline_parameters.landfills = baseline_landfills @@ -1121,6 +1238,21 @@ def run_advanced_dst_city( landfill.oxidation_factor = ox landfill.estimate_emissions(skip_ox=True) + # The surplus streams, run the same way -- but only after + # `repopulate_attr_dicts` above has settled the city's parameter dict, which + # these never receive because they are not in `parameters.landfills`. + for streams, landfills, masses, oxidations in ( + (baseline_surplus, baseline_landfills, baseline_surplus_masses, baseline_ox), + (scenario_surplus, scenario_landfills, scenario_surplus_masses, scenario_ox), + ): + for surplus, city_stream, mass, ox in zip(streams, landfills, masses, oxidations): + if surplus is None: + continue + site_inflow.adopt_city_params(city_stream, surplus) + surplus.waste_mass_df = mass + surplus.oxidation_factor = ox + surplus.estimate_emissions(skip_ox=True) + city.baseline_parameters = baseline_parameters city.scenario_parameters[0] = scenario_parameters @@ -1144,6 +1276,17 @@ def run_advanced_dst_city( "baseline": baseline_parameters.total_emissions, "scenario": scenario_parameters.total_emissions, } + if with_site_emissions: + result["site_emissions"] = { + "baseline": [ + site_inflow.emission_frames(city_stream, surplus) + for city_stream, surplus in zip(baseline_landfills, baseline_surplus) + ], + "scenario": [ + site_inflow.emission_frames(city_stream, surplus) + for city_stream, surplus in zip(scenario_landfills, scenario_surplus) + ], + } if with_mass_flow: result["mass_flow"] = { "baseline": _mass_flow( diff --git a/SWEET_python/site_inflow.py b/SWEET_python/site_inflow.py new file mode 100644 index 0000000..403b4c4 --- /dev/null +++ b/SWEET_python/site_inflow.py @@ -0,0 +1,187 @@ +"""Waste arriving at a disposal site that the city did not send it. + +A city-level ADST run shares the city's landfilled residual out across the +city's own sites: ``landfill_split_timeline`` is *fractions that must sum to +1.0* (``advanced_dst_city._validate_shares``), so every ton at every site came +from the city by construction. Real sites are not like that. A regional +landfill takes waste from neighbouring municipalities, from private haulers, +from a catchment the baseline never describes -- and the operator knows the +gate total, not this city's share of it. + +So a site may state ``accepted_waste_mass``: everything crossing its weighbridge +in a year, from every source. Whatever that exceeds the city's own allocation is +modeled as a second stream at the same site. + +Two things then have to be true, and this module is arranged so that both are +structural facts about the code rather than arithmetic that happens to come out +right. + +**The city's emissions do not move.** A city is answerable for the waste it +generates, wherever that waste ends up; a neighbour's waste is the neighbour's +inventory. ``City.sum_landfill_emissions`` sums ``.emissions`` over every entry +of ``parameters.landfills`` -- so the surplus landfills built here are never put +in that list. Nothing about the city's total is recomputed, re-derived or +carefully cancelled: it is the same expression over the same objects, and the +test asserts frame equality rather than a tolerance. + +**A site's total is its streams, summed exactly.** Deposited mass enters the +first-order-decay kernel once and multiplicatively (``model_v2``: +``ch4_produce = ks_values * L_0[waste] * waste_masses * exp_term * mcf_values``) +and every step after it -- capture, flare, oxidation, the unit conversion -- is +a mass-independent factor. ``E(0)`` is exactly ``0.0``. So emissions superpose, +measured at ~1e-16 relative, and "the city's share of this site's methane" is a +physical quantity rather than an allocation convention somebody had to invent +and defend. + +That second invariant has a precondition, and it is the thing most likely to be +broken by a well-meaning later change: **every stream at a site must share one +``k``**. ``k`` is a step function of composition -- an 8x8x8 lookup keyed on +``int(share * 8)`` -- so two streams at one landfill with different mixes do not +superpose, and the error is tens of percent rather than rounding. The surplus +therefore carries the city's own post-diversion residual composition, which is +also the physically defensible answer: a gate observation is downstream of +whatever diversion happened upstream of it, so a landfill's intake is residual +in character whoever sent it. + +The other way to get this wrong is to skip the second kernel run and attribute +by tonnage: ``city tons / site tons * site total``. That is wrong whenever the +city's share moves over time, because this year's emissions come from decades of +deposit cohorts each with their own split -- measured at up to 28% error for a +site with a growing neighbour, and in a direction that looks entirely plausible. +Each stream gets its own ``Landfill`` and its own pass through the kernel. + +Why the functions here are free functions rather than ``Landfill`` methods: the +package already separates a landfill's construction from its definition -- +``dst_common.build_landfill`` builds a :class:`~SWEET_python.landfill.Landfill` +from a different module and leaves the caller to assign ``waste_mass_df``. This +follows that convention rather than growing either ``landfill.py`` or the +9,000-line ``city_params.py``. +""" + +from typing import Dict, List, Optional + +import pandas as pd + +from SWEET_python.city_params import City +from SWEET_python.landfill import Landfill + +__all__ = [ + "residual_composition", + "surplus_intake", + "surplus_masses", + "adopt_city_params", + "emission_frames", +] + + +def residual_composition( + residual: pd.DataFrame, generated: pd.DataFrame +) -> pd.DataFrame: + """Per-year component shares of what the city is left to bury; rows sum to 1. + + The mix the surplus is given, so that both streams at a site decay at the + same ``k`` -- see the module docstring on why that is not optional. + + Not the *generated* mix. Composting and anaerobic digestion draw only on + organics and recycling only on recyclables, so the residual differs from + what the city generates in shape and not merely in scale; splitting a + gate-observed tonnage by the generated mix overstates its degradable half + and with it the methane. ``generated`` is the fallback for a year the city + buries nothing at all, which would otherwise be 0/0 -- and in such a year + the tonnage these shares scale is itself zero, so the choice is cosmetic. + """ + totals = residual.sum(axis=1) + shares = residual.div(totals, axis=0) + + empty = totals <= 0 + if empty.any(): + fallback_totals = generated.sum(axis=1) + fallback = generated.div(fallback_totals.where(fallback_totals > 0), axis=0) + shares.loc[empty, :] = fallback.loc[empty, :] + + return shares.fillna(0.0) + + +def surplus_intake( + accepted: pd.Series, city_intake: pd.Series +) -> pd.Series: + """Tons per year reaching a site's gate from outside this baseline. + + Compared at the *gate*, before combustion and before the open/close window, + because that is where the stated total was measured: a site that burns its + intake still accepted all of it. + + Floored at zero, which is the answer to a stated total *below* what the city + sends. That is a real condition -- a gate figure and an allocation are + measured by different people in different years -- and it is not a licence + to shrink the city's stream. The city's waste has to go somewhere, this + feature does not add anywhere else for it to go, and a city's emissions are + not reduced by a neighbour's paperwork. So the site simply has no outside + waste, and the caller is free to show the city's figure as the floor it is. + """ + return (accepted - city_intake).clip(lower=0.0) + + +def surplus_masses( + surplus: pd.Series, composition: pd.DataFrame +) -> pd.DataFrame: + """The surplus tonnage split by component, at the city's residual mix.""" + return composition.mul(surplus, axis=0) + + +def adopt_city_params(source: Landfill, *surplus: Landfill) -> None: + """Give the surplus landfills their site's own ``city_params_dict``. + + ``CityParameters.repopulate_attr_dicts`` pushes a freshly dumped parameter + dict onto every landfill in ``parameters.landfills`` just before the engine + runs. The surplus landfills are deliberately not in that list -- that list is + what the city's total is summed over -- so they miss that pass, and would + otherwise decay against whatever dict they were constructed with. + + Copying the object off the site's city-stream landfill, rather than + rebuilding it, is what makes them exactly equal: two streams at one site + must agree on every non-mass parameter or they do not superpose, and an + equality that has to be maintained by hand is one that will eventually drift. + """ + for landfill in surplus: + landfill.city_params_dict = source.city_params_dict + + +def emission_frames( + city_stream: Landfill, surplus_stream: Optional[Landfill] +) -> Dict[str, pd.DataFrame]: + """One site's emissions, split by whose waste produced them. + + Returns ``{"city": frame, "other": frame, "total": frame}``, each indexed by + year with the model's degradable components plus ``total``, in **tons of + methane per year** -- the same units and the same conversion + ``City.sum_landfill_emissions`` applies to the city's own figure, so the two + outputs can be read side by side. + + ``other`` is an all-zero frame rather than ``None`` for a site with no + outside waste, so a caller never has to special-case the ordinary site. + """ + city = _to_tons_ch4(city_stream.emissions) + + if surplus_stream is None or surplus_stream.emissions is None: + other = city * 0.0 + else: + # Both streams were built on the same open year, so the kernel gave them + # the same index -- but reindex rather than trust it, because a silent + # NaN here would read as a plausible number downstream. + other = _to_tons_ch4(surplus_stream.emissions).reindex( + index=city.index, columns=city.columns, fill_value=0.0 + ) + + return {"city": city, "other": other, "total": city + other} + + +def _to_tons_ch4(emissions: pd.DataFrame) -> pd.DataFrame: + """m^3 of methane to tons of it, the way the city's own total is converted. + + ``City.sum_landfill_emissions`` converts to tons of CO2e and then divides by + 28, which lands back on tons of methane. Reproduced rather than simplified + so that a per-site figure and the city figure it has to reconcile with pass + through identical arithmetic -- including the same rounding. + """ + return emissions.map(City.convert_methane_m3_to_ton_co2e) / 28 diff --git a/tests/test_site_other_source.py b/tests/test_site_other_source.py index a4682f8..637ba6a 100644 --- a/tests/test_site_other_source.py +++ b/tests/test_site_other_source.py @@ -50,7 +50,7 @@ GENERATED = 100_000.0 -def _spec(*, open_close=(1990, 2051), other=None, combusts=None): +def _spec(*, open_close=(1990, 2051), accepted=None, combusts=None): spec = dict( landfill_type={"baseline": 2, "scenario": 2}, landfill_open_close={"baseline": list(open_close), "scenario": list(open_close)}, @@ -59,8 +59,8 @@ def _spec(*, open_close=(1990, 2051), other=None, combusts=None): "scenario": {y: 0.0 for y in YEARS}, }, ) - if other is not None: - spec["other_source_mass"] = {"baseline": dict(other), "scenario": dict(other)} + if accepted is not None: + spec["accepted_waste_mass"] = {"baseline": dict(accepted), "scenario": dict(accepted)} if combusts is not None: spec["combusts"] = {"baseline": combusts, "scenario": combusts} return spec @@ -100,11 +100,16 @@ def _total(frame): return np.asarray(frame["total"], dtype=float) -# A neighbour whose tonnage RISES while the city's stays flat, so the city's -# share of the site falls over time. Any attribution built on a same-year ratio -# gets this case badly wrong; see the last test. -RISING_NEIGHBOUR = {y: 3_000.0 * (1.05 ** (y - 1990)) for y in YEARS} -FLAT_NEIGHBOUR = {y: 25_000.0 for y in YEARS} +# `accepted_waste_mass` is the whole gate total, not the neighbour's share of +# it, so every fixture here has to clear whatever the city sends. With 25% +# composted the city buries ~78,154 t/yr of its 100,000 t; a site taking the +# city's entire residual and 100,000 t is therefore comfortably over. +GATE_WITH_NEIGHBOUR = {y: 100_000.0 for y in YEARS} + +# A gate total that GROWS away from the city's flat allocation, so the city's +# share of the site falls year on year. Any attribution built on a same-year +# tonnage ratio gets this case badly wrong; see the last test. +GATE_GROWING = {y: 100_000.0 + 3_000.0 * (1.05 ** (y - 1990)) for y in YEARS} # --------------------------------------------------------------------------- # @@ -115,7 +120,7 @@ def test_other_source_waste_does_not_change_the_city_total(): """The whole feature, in one assertion. Bit-identical, not merely close.""" without = run_advanced_dst_city(_request([_spec()], (1.0,), diversion=0.25)) with_other = run_advanced_dst_city( - _request([_spec(other=FLAT_NEIGHBOUR)], (1.0,), diversion=0.25) + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,), diversion=0.25) ) for variant in ("baseline", "scenario"): @@ -128,7 +133,7 @@ def test_an_absent_field_and_a_zero_series_are_the_same_request(): """No silent second code path for the city that never uses this.""" absent = run_advanced_dst_city(_request([_spec()], (1.0,), diversion=0.25)) zeros = run_advanced_dst_city( - _request([_spec(other={y: 0.0 for y in YEARS})], (1.0,), diversion=0.25) + _request([_spec(accepted={y: 0.0 for y in YEARS})], (1.0,), diversion=0.25) ) for variant in ("baseline", "scenario"): @@ -145,7 +150,7 @@ def test_site_shares_still_have_to_sum_to_one(): with pytest.raises(CustomError): run_advanced_dst_city( - _request([_spec(other=FLAT_NEIGHBOUR), _spec()], (0.5, 0.2)) + _request([_spec(accepted=GATE_WITH_NEIGHBOUR), _spec()], (0.5, 0.2)) ) @@ -155,7 +160,7 @@ def test_site_shares_still_have_to_sum_to_one(): def test_a_sites_emissions_split_into_the_city_and_the_rest(): result = run_advanced_dst_city( - _request([_spec(other=FLAT_NEIGHBOUR)], (1.0,), diversion=0.25), + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,), diversion=0.25), with_site_emissions=True, ) site = result["site_emissions"]["baseline"][0] @@ -179,7 +184,7 @@ def test_the_city_contribution_is_what_the_city_alone_would_have_produced(): _request([_spec()], (1.0,), diversion=0.25), with_site_emissions=True ) shared = run_advanced_dst_city( - _request([_spec(other=FLAT_NEIGHBOUR)], (1.0,), diversion=0.25), + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,), diversion=0.25), with_site_emissions=True, ) @@ -204,7 +209,7 @@ def test_the_city_shares_of_every_site_add_up_to_the_city_total(): """The two outputs reconcile: city emissions are the per-site city shares plus whatever the diversion pathways themselves emit.""" request = _request( - [_spec(other=FLAT_NEIGHBOUR), _spec(other=RISING_NEIGHBOUR)], + [_spec(accepted=GATE_WITH_NEIGHBOUR), _spec(accepted=GATE_GROWING)], (0.6, 0.4), diversion=0.25, ) @@ -239,7 +244,7 @@ def test_the_city_share_is_not_this_years_tonnage_ratio(): This test fails if someone replaces that with the ratio. """ result = run_advanced_dst_city( - _request([_spec(other=RISING_NEIGHBOUR)], (1.0,), diversion=0.0), + _request([_spec(accepted=GATE_GROWING)], (1.0,), diversion=0.0), with_site_emissions=True, ) site = result["site_emissions"]["baseline"][0] @@ -248,8 +253,10 @@ def test_the_city_share_is_not_this_years_tonnage_ratio(): settled = total > 0.01 * total.max() emission_share = (city / np.where(total == 0, 1.0, total))[settled] + # No diversion in this run, so the city's allocation is its whole stream. city_tons = np.array([GENERATED for _ in YEARS]) - mass_share = (city_tons / (city_tons + np.array([RISING_NEIGHBOUR[y] for y in YEARS])))[settled] + gate_tons = np.array([GATE_GROWING[y] for y in YEARS]) + mass_share = (city_tons / gate_tons)[settled] # The two shares must visibly disagree: emissions lag mass, so a shrinking # city holds a larger share of the emissions than of this year's intake. From b1e1e63f5ff02d2dc8e52d0cd722755da250f6b6 Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Wed, 9 Sep 2026 11:25:33 -0700 Subject: [PATCH 3/5] Changelog, and stop overclaiming why the surplus shares the city's mix The module said two streams at one site must share a composition or they stop superposing, citing #785's 27.2% figure. That figure is about blending two mixes into one kernel run, which this does not do: `k` is computed once per variant from the city's generated mix and handed to every landfill, so a stream's own mix moves only its per-component masses and superposition holds regardless. Verified against the code and by measurement. The residual mix is still the default -- a gate observation is downstream of whatever diversion happened upstream of it -- but it is a modelling judgement, not a constraint, and a later PR can lift it. Also adds `require_linear`: `doing_fancy_ox` is the one thing that would genuinely break superposition, so the split refuses to run rather than returning an attribution nobody could defend. And four tests pinning what the rest of the engine is entitled to keep assuming. Co-Authored-By: Claude Opus 5 --- SWEET_python/site_inflow.py | 107 ++++++++++++++++++++++++-------- changelog/2026-09.md | 49 +++++++++++++++ changelog/README.md | 2 +- tests/test_site_other_source.py | 102 ++++++++++++++++++++++++++---- 4 files changed, 221 insertions(+), 39 deletions(-) diff --git a/SWEET_python/site_inflow.py b/SWEET_python/site_inflow.py index 403b4c4..739d2f1 100644 --- a/SWEET_python/site_inflow.py +++ b/SWEET_python/site_inflow.py @@ -33,15 +33,26 @@ physical quantity rather than an allocation convention somebody had to invent and defend. -That second invariant has a precondition, and it is the thing most likely to be -broken by a well-meaning later change: **every stream at a site must share one -``k``**. ``k`` is a step function of composition -- an 8x8x8 lookup keyed on -``int(share * 8)`` -- so two streams at one landfill with different mixes do not -superpose, and the error is tens of percent rather than rounding. The surplus -therefore carries the city's own post-diversion residual composition, which is -also the physically defensible answer: a gate observation is downstream of -whatever diversion happened upstream of it, so a landfill's intake is residual -in character whoever sent it. +That invariant needs every stream at a site to agree on every parameter but its +mass, which is why the surplus twin is built from the city stream's own kwargs +dict rather than a copy of it. Note what is *not* on that list: composition. +``k`` is computed once per variant from the city's generated mix +(``advanced_dst_city`` around the ``decomposition_rates`` call) and handed +identically to every landfill, so a stream's own mix moves only its +per-component deposited masses -- which the kernel is linear in. Two streams at +one site with entirely different mixes still superpose exactly. + +So the surplus carrying the city's post-diversion residual composition is a +**modelling default, not a numerical requirement**. It is the default because it +is the defensible reading of a gate observation -- a landfill's intake is +residual in character whoever sent it, since somebody's diversion happened +upstream of it -- and because the user has no second composition to hand. A +later PR can let a site state its own mix for the surplus, and +``residual_composition`` is the one function it has to replace. + +(The 27.2% figure quoted in PR #785 is about blending two mixes into a *single* +kernel run, where one ``k`` has to stand for both. That is not what happens +here: each stream gets its own ``Landfill`` and its own pass.) The other way to get this wrong is to skip the second kernel run and attribute by tonnage: ``city tons / site tons * site total``. That is wrong whenever the @@ -58,14 +69,16 @@ 9,000-line ``city_params.py``. """ -from typing import Dict, List, Optional +from typing import Optional import pandas as pd +from pydantic import BaseModel, ConfigDict -from SWEET_python.city_params import City +from SWEET_python.city_params import City, CustomError from SWEET_python.landfill import Landfill __all__ = [ + "SiteEmissions", "residual_composition", "surplus_intake", "surplus_masses", @@ -74,13 +87,36 @@ ] +class SiteEmissions(BaseModel): + """One site's methane, split by whose waste produced it. + + Each frame is years x the model's degradable components plus ``total``, in + **tons of methane per year** -- the same units and the same conversion + ``City.sum_landfill_emissions`` applies to the city's own figure, so the two + outputs can be read side by side. + + ``city`` is the share the city's own headline total is built from; ``other`` + is waste that reached the gate from outside the baseline, and is an all-zero + frame rather than ``None`` for the ordinary site, so no caller has to + special-case it. + """ + + city: pd.DataFrame + other: pd.DataFrame + total: pd.DataFrame + + model_config = ConfigDict(arbitrary_types_allowed=True) + + def residual_composition( residual: pd.DataFrame, generated: pd.DataFrame ) -> pd.DataFrame: """Per-year component shares of what the city is left to bury; rows sum to 1. - The mix the surplus is given, so that both streams at a site decay at the - same ``k`` -- see the module docstring on why that is not optional. + The mix the surplus is given. This is the single place that decision is + made, and the one function a later "a site states its own mix" PR replaces + -- nothing downstream of it assumes the two streams at a site agree, because + ``k`` is city-wide and they superpose whether they agree or not. Not the *generated* mix. Composting and anaerobic digestion draw only on organics and recycling only on recyclables, so the residual differs from @@ -142,25 +178,46 @@ def adopt_city_params(source: Landfill, *surplus: Landfill) -> None: rebuilding it, is what makes them exactly equal: two streams at one site must agree on every non-mass parameter or they do not superpose, and an equality that has to be maintained by hand is one that will eventually drift. + + Mirrors all three attributes ``repopulate_attr_dicts`` sets, not just the + first. The other two only exist once a landfill has run, which a surplus + stream has not yet -- but copying one of three is the kind of near-miss that + survives a refactor and then stops being harmless. """ for landfill in surplus: landfill.city_params_dict = source.city_params_dict + if hasattr(landfill, "model"): + landfill.model.city_params_dict = source.city_params_dict + landfill.model.landfill_instance_attrs = landfill.model_dump() -def emission_frames( - city_stream: Landfill, surplus_stream: Optional[Landfill] -) -> Dict[str, pd.DataFrame]: - """One site's emissions, split by whose waste produced them. +def require_linear(landfill: Landfill) -> None: + """Refuse to split a site whose emissions are not linear in deposited mass. - Returns ``{"city": frame, "other": frame, "total": frame}``, each indexed by - year with the model's degradable components plus ``total``, in **tons of - methane per year** -- the same units and the same conversion - ``City.sum_landfill_emissions`` applies to the city's own figure, so the two - outputs can be read side by side. + ``Landfill.doing_fancy_ox`` is hardcoded ``False`` and its body derives an + oxidation factor from one year's available methane and then clips it three + times -- genuinely nonlinear in mass. Everything here assumes it stays off: + with it on, two streams at a site stop superposing and "the city's share" + silently becomes an allocation convention rather than a measurement, with no + error and no visible symptom. - ``other`` is an all-zero frame rather than ``None`` for a site with no - outside waste, so a caller never has to special-case the ordinary site. + So this raises rather than warns, and it is a runtime check rather than a + test: the flag is an attribute anybody can set, and a test only fails for + whoever runs the suite. """ + if getattr(landfill, "doing_fancy_ox", False): + raise CustomError( + "nonlinear_oxidation", + "This site uses CALMIM oxidation, whose emissions are not linear in " + "deposited mass, so its waste cannot be split by source.", + ) + + +def emission_frames( + city_stream: Landfill, surplus_stream: Optional[Landfill] +) -> SiteEmissions: + """One site's emissions, split by whose waste produced them.""" + require_linear(city_stream) city = _to_tons_ch4(city_stream.emissions) if surplus_stream is None or surplus_stream.emissions is None: @@ -173,7 +230,7 @@ def emission_frames( index=city.index, columns=city.columns, fill_value=0.0 ) - return {"city": city, "other": other, "total": city + other} + return SiteEmissions(city=city, other=other, total=city + other) def _to_tons_ch4(emissions: pd.DataFrame) -> pd.DataFrame: diff --git a/changelog/2026-09.md b/changelog/2026-09.md index d171107..7750189 100644 --- a/changelog/2026-09.md +++ b/changelog/2026-09.md @@ -47,6 +47,55 @@ landfill received — instead of re-deriving it. ## Added +- **A disposal site can accept waste the city did not send it.** `CityLandfillSpec` + takes an optional `accepted_waste_mass`: everything crossing that site's + weighbridge in a year, from every source. `landfill_split_timeline` is + fractions that must sum to 1, so until now every ton at every site came from + the city by construction — a regional landfill taking waste from a + neighbouring municipality was structurally unsayable, and site operators know + their gate total rather than one city's share of it. + + Whatever the stated total exceeds the city's own allocation is modelled as a + second stream at the same site: its own `Landfill`, its own pass through the + decay kernel, deposited and windowed on exactly the same terms as the city's + waste because it is the same hole in the ground. A figure at or below the + city's allocation is read as "no waste from outside" rather than as a + reduction of the city's — the city's waste still has to go somewhere, and this + does not add anywhere else for it to go. + + **The city's own emissions do not move**, and the reason is structural rather + than arithmetic: `City.sum_landfill_emissions` sums `parameters.landfills`, + and the surplus landfills are never put in that list. The test asserts frame + equality, not a tolerance. Every request without the field takes the path it + took before, with `city_params.py` untouched. + + `run_advanced_dst_city(request, with_site_emissions=True)` returns the split: + per landfill in request order, `city`, `other` and `total` in tons of methane + per year. The two are separate kernel runs summed, never a ratio applied to + one of them — this year's emissions come from decades of deposit cohorts each + with their own split, and attributing by this year's tonnage was measured at + up to 28% error where a site's mix of sources moves over time. A test pins + that case so the shortcut cannot be reintroduced quietly. + + Superposition is exact because the kernel is linear in deposited mass + (measured at ~1e-16 relative; `E(0)` is exactly `0.0`), so "the city's share + of this site's methane" is a measured quantity rather than an allocation + convention. `Landfill.doing_fancy_ox` is the one thing that would break it — + its body is genuinely nonlinear in mass — so the split now refuses to run + rather than silently returning an attribution nobody could defend. + + The surplus carries the city's post-diversion residual composition. That is a + modelling default, not a numerical constraint: `k` is city-wide, so a stream's + own mix moves only its per-component masses. It is the default because a gate + observation is downstream of whatever diversion happened upstream of it. + `site_inflow.residual_composition` is the single function a later "a site + states its own mix" change replaces. + + New behaviour lives in `SWEET_python/site_inflow.py`, following the convention + `dst_common.build_landfill` already sets — a landfill's construction lives + outside `landfill.py` — rather than growing the 9,000-line `city_params.py`. + ([#NN](https://github.com/RMI/SWEET_python/pull/NN)) + - **`run_advanced_dst_city_limits` — what a city's composition allows.** Takes the same request as `run_advanced_dst_city` and returns, per year: the fraction of each waste component's own mass the diversion pathways claim, the diff --git a/changelog/README.md b/changelog/README.md index 0393293..992f2b9 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -15,7 +15,7 @@ The project does not publish semantic version tags, so releases are tracked by Newest first: -- [2026-09](2026-09.md) — Model output frames get one fixed waste-type column order instead of a per-process one (a `set` fed the ordering, and Python randomizes set iteration per process), moving values in the last bit only, where a sum runs across the reordered columns; City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart +- [2026-09](2026-09.md) — A disposal site can accept waste the city did not send it (`accepted_waste_mass`), modelled as a second stream at the same site, with per-site emissions returned split into the city's share and the rest; the city's own total is untouched by construction. Model output frames get one fixed waste-type column order instead of a per-process one (a `set` fed the ordering, and Python randomizes set iteration per process), moving values in the last bit only, where a sum runs across the reordered columns; City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart - [2026-08](2026-08.md) — Single-site adst gains optional `depth` (deep-dump MCF bump) and `k_override` (caller-supplied decomposition rate) inputs, restoring the last two site-DST levers; `/sdst` flaring efficiency reaches the model again after a variable-name bug silently forced flare destruction to 0.98; annual model applies cover oxidation by emission year not deposit year, fixing biocover having no effect on closed landfills (WasteMAP #719); `City.sdst_v1_5` custom-site path holds the scenario equal to the baseline before the implementation year even when composition changes (was back-dating the new composition onto pre-implementation deposits) (model-output change); MCF consolidated into a new `SWEET_python.mcf` module and both dump types moved to the IPCC uncategorised-SWDS 0.6 (open dumps up from 0.4, controlled dumps down from 0.7), with a supplied depth now selecting the deep/shallow category (model-output change) - [2026-07](2026-07.md) — All ten waste types eligible for combustion (metal/glass/other added); methane-only model treats combustion as landfill diversion (model-output change) - [2026-06](2026-06.md) — New single-site and city-level ADST modeling modules, min-cost max-flow rewrite of the city DST diversion allocator, physical-k fix for cold/dry sites, no more spurious negative food-waste mass diff --git a/tests/test_site_other_source.py b/tests/test_site_other_source.py index 637ba6a..3f642c0 100644 --- a/tests/test_site_other_source.py +++ b/tests/test_site_other_source.py @@ -26,12 +26,13 @@ what makes "the city's share of this site's emissions" a physical quantity rather than an allocation convention someone had to invent. -The linearity has one precondition and it is load-bearing: every stream at a -site must share one ``k``. ``k`` is a step function of composition, so two -streams at one landfill with different mixes do not superpose. Other-source -waste therefore carries the city's own post-diversion residual composition -- -which is also the physically right answer, since a gate observation is -downstream of whatever diversion happened upstream of it. +Linearity needs the two streams to agree on every parameter but mass, which the +implementation gets by building the surplus twin from the city stream's own +kwargs. Composition is not one of those parameters: ``k`` is computed once from +the city's generated mix and handed to every landfill, so a stream's own mix +moves only its per-component masses. The surplus carries the city's +post-diversion residual mix because that is the defensible reading of a gate +observation, not because superposition would otherwise fail. """ import numpy as np @@ -165,7 +166,7 @@ def test_a_sites_emissions_split_into_the_city_and_the_rest(): ) site = result["site_emissions"]["baseline"][0] - city, other, total = _total(site["city"]), _total(site["other"]), _total(site["total"]) + city, other, total = _total(site.city), _total(site.other), _total(site.total) residual = np.abs(total - (city + other)) scale = np.where(total == 0, 1.0, total) @@ -188,8 +189,8 @@ def test_the_city_contribution_is_what_the_city_alone_would_have_produced(): with_site_emissions=True, ) - a = _total(alone["site_emissions"]["baseline"][0]["city"]) - b = _total(shared["site_emissions"]["baseline"][0]["city"]) + a = _total(alone["site_emissions"]["baseline"][0].city) + b = _total(shared["site_emissions"]["baseline"][0].city) scale = np.where(a == 0, 1.0, a) assert np.max(np.abs(a - b) / scale) < 1e-12 @@ -201,8 +202,8 @@ def test_a_site_with_no_other_source_reports_a_total_equal_to_its_city_share(): ) site = result["site_emissions"]["baseline"][0] - assert np.allclose(_total(site["other"]), 0.0) - assert np.allclose(_total(site["total"]), _total(site["city"]), rtol=0, atol=1e-12) + assert np.allclose(_total(site.other), 0.0) + assert np.allclose(_total(site.total), _total(site.city), rtol=0, atol=1e-12) def test_the_city_shares_of_every_site_add_up_to_the_city_total(): @@ -216,7 +217,7 @@ def test_the_city_shares_of_every_site_add_up_to_the_city_total(): result = run_advanced_dst_city(request, with_site_emissions=True) per_site = sum( - _total(site["city"]) for site in result["site_emissions"]["baseline"] + _total(site.city) for site in result["site_emissions"]["baseline"] ) city_total = _total(result["baseline"]) @@ -248,7 +249,7 @@ def test_the_city_share_is_not_this_years_tonnage_ratio(): with_site_emissions=True, ) site = result["site_emissions"]["baseline"][0] - city, total = _total(site["city"]), _total(site["total"]) + city, total = _total(site.city), _total(site.total) settled = total > 0.01 * total.max() emission_share = (city / np.where(total == 0, 1.0, total))[settled] @@ -265,3 +266,78 @@ def test_the_city_share_is_not_this_years_tonnage_ratio(): "static to be a regression guard, or attribution has been reduced to a ratio" ) assert np.all(emission_share[-10:] > mass_share[-10:]) + + +# --------------------------------------------------------------------------- # +# Assumptions the rest of the engine is entitled to keep making +# --------------------------------------------------------------------------- # + +def test_no_surplus_emits_exactly_zero(): + """`==`, not `approx`. E(0) is exactly 0.0, so any residual is a bug.""" + result = run_advanced_dst_city( + _request([_spec(accepted={y: 0.0 for y in YEARS})], (1.0,)), + with_site_emissions=True, + ) + + assert (_total(result["site_emissions"]["baseline"][0].other) == 0.0).all() + + +def test_the_mass_flows_sites_band_is_still_only_the_city(): + """`sum(sites) == landfilled` is asserted in eleven places elsewhere. + + The band reports where the *city's* waste went, and other-source waste is + not the city's. Folding it in here would be the easy mistake, and it would + break the one identity the mass flow exists to keep. + """ + plain = run_advanced_dst_city(_request([_spec()], (1.0,)), with_mass_flow=True) + with_other = run_advanced_dst_city( + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,)), with_mass_flow=True + ) + + for variant in ("baseline", "scenario"): + a = plain["mass_flow"][variant] + b = with_other["mass_flow"][variant] + assert a["sites"][0].equals(b["sites"][0]) + assert a["landfilled"].equals(b["landfilled"]) + + +def test_a_combusting_site_burns_everything_it_accepts(): + """The surplus meets the same furnace the city's waste does. + + Compared at the gate but deposited after the burn, so a site stated at + 150,000 t buries the reject of 150,000 t -- not 150,000 t of residue. + """ + burning = run_advanced_dst_city( + _request( + [_spec(accepted=GATE_WITH_NEIGHBOUR, combusts=True)], + (1.0,), + diversion=0.25, + ), + with_site_emissions=True, + ) + depositing = run_advanced_dst_city( + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,), diversion=0.25), + with_site_emissions=True, + ) + + burnt = _total(burning["site_emissions"]["baseline"][0].other) + kept = _total(depositing["site_emissions"]["baseline"][0].other) + + assert burnt.sum() > 0, "an incinerator's residue still decays" + # The unburnable reject is 10%, the same rate the combustion pathway uses. + assert burnt.sum() == pytest.approx(kept.sum() * 0.1, rel=1e-9) + + +def test_the_limits_endpoint_ignores_the_new_field(): + """Bounds are about what the city generates, which a gate total says + nothing about.""" + from SWEET_python.advanced_dst_city import run_advanced_dst_city_limits + + plain = run_advanced_dst_city_limits(_request([_spec()], (1.0,))) + with_other = run_advanced_dst_city_limits( + _request([_spec(accepted=GATE_WITH_NEIGHBOUR)], (1.0,)) + ) + + assert plain.keys() == with_other.keys() + for variant in plain: + assert str(plain[variant]) == str(with_other[variant]) From ee7c3cf4e1daffc684e09061f984ad3daaf8b6b0 Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Wed, 9 Sep 2026 11:26:25 -0700 Subject: [PATCH 4/5] Changelog links #65 --- changelog/2026-09.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/2026-09.md b/changelog/2026-09.md index 7750189..14b57ee 100644 --- a/changelog/2026-09.md +++ b/changelog/2026-09.md @@ -94,7 +94,7 @@ landfill received — instead of re-deriving it. New behaviour lives in `SWEET_python/site_inflow.py`, following the convention `dst_common.build_landfill` already sets — a landfill's construction lives outside `landfill.py` — rather than growing the 9,000-line `city_params.py`. - ([#NN](https://github.com/RMI/SWEET_python/pull/NN)) + ([#65](https://github.com/RMI/SWEET_python/pull/65)) - **`run_advanced_dst_city_limits` — what a city's composition allows.** Takes the same request as `run_advanced_dst_city` and returns, per year: the From 64c7093d56e71745d5c3c9c803c00e2cfd2079be Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Wed, 9 Sep 2026 13:04:34 -0700 Subject: [PATCH 5/5] Outside waste keeps its composition when the city generates none Three review findings, all real. The surplus is split by the city's post-diversion residual mix, and in a year the city buries nothing that mix is 0/0. The fallback was the composition as *masses*, which is all-zero in exactly the years it was needed -- so every share stayed zero and the surplus was deposited as nothing at all. A site taking regional waste while its city has not started collecting yet is an ordinary shape: 200,000 t/yr across thirty years came back as 0 t CH4, silently. The fallback is now the composition as shares, which survives a zero-mass year. The frame-level pre-implement splice had no test that could fail. A variant-differing gate total cannot exercise it -- `variant_series` already splices the series upstream -- so the case that reaches it is a window or `combusts` that differs between variants. Pinned with a site that starts incinerating at the implement year, which without the splice unburies 90% of its own history. And the test module documented an `other_source_mass` field that never existed, describing it as the outside share rather than the gate total. Co-Authored-By: Claude Opus 5 --- SWEET_python/advanced_dst_city.py | 8 +- SWEET_python/site_inflow.py | 18 ++-- changelog/2026-09.md | 8 +- tests/test_site_other_source.py | 146 ++++++++++++++++++++++++++++-- 4 files changed, 161 insertions(+), 19 deletions(-) diff --git a/SWEET_python/advanced_dst_city.py b/SWEET_python/advanced_dst_city.py index 6b79324..b0dfc50 100644 --- a/SWEET_python/advanced_dst_city.py +++ b/SWEET_python/advanced_dst_city.py @@ -1006,9 +1006,13 @@ def run_advanced_dst_city( # arriving at a site from outside the baseline is given this same mix, so # that both streams at a site decay at one `k` and therefore superpose -- # see `site_inflow` for why that is load-bearing rather than tidy. + # The fallback is the composition as *shares* rather than `wgen_*`, the + # composition as masses: a year the city generates nothing has an all-zero + # mass frame and so no fallback at all, which left a site's outside waste + # with a zero mix and deposited none of it. residual_mix = { - "baseline": site_inflow.residual_composition(net_masses["baseline"], wgen_baseline), - "scenario": site_inflow.residual_composition(net_masses["scenario"], wgen_scenario), + "baseline": site_inflow.residual_composition(net_masses["baseline"], baseline_fractions), + "scenario": site_inflow.residual_composition(net_masses["scenario"], scenario_fractions), } # --- City-wide decomposition rates + compost emission factors --- diff --git a/SWEET_python/site_inflow.py b/SWEET_python/site_inflow.py index 739d2f1..d810086 100644 --- a/SWEET_python/site_inflow.py +++ b/SWEET_python/site_inflow.py @@ -109,7 +109,7 @@ class SiteEmissions(BaseModel): def residual_composition( - residual: pd.DataFrame, generated: pd.DataFrame + residual: pd.DataFrame, fractions: pd.DataFrame ) -> pd.DataFrame: """Per-year component shares of what the city is left to bury; rows sum to 1. @@ -122,17 +122,23 @@ def residual_composition( organics and recycling only on recyclables, so the residual differs from what the city generates in shape and not merely in scale; splitting a gate-observed tonnage by the generated mix overstates its degradable half - and with it the methane. ``generated`` is the fallback for a year the city - buries nothing at all, which would otherwise be 0/0 -- and in such a year - the tonnage these shares scale is itself zero, so the choice is cosmetic. + and with it the methane. + + ``fractions`` is the fallback for a year the city buries nothing, which + would otherwise be 0/0. It must be the city's composition **as shares**, not + as masses: a year in which the city generates nothing has an all-zero mass + frame, so a mass fallback is no fallback at all -- it leaves every share at + zero and the surplus is then deposited as nothing. That is not a hypothetical + tidy-up. A site taking regional waste while the city it belongs to collects + none yet is an ordinary shape, and it silently lost every ton of it. """ totals = residual.sum(axis=1) shares = residual.div(totals, axis=0) empty = totals <= 0 if empty.any(): - fallback_totals = generated.sum(axis=1) - fallback = generated.div(fallback_totals.where(fallback_totals > 0), axis=0) + fallback_totals = fractions.sum(axis=1) + fallback = fractions.div(fallback_totals.where(fallback_totals > 0), axis=0) shares.loc[empty, :] = fallback.loc[empty, :] return shares.fillna(0.0) diff --git a/changelog/2026-09.md b/changelog/2026-09.md index 14b57ee..4b52fe6 100644 --- a/changelog/2026-09.md +++ b/changelog/2026-09.md @@ -89,7 +89,13 @@ landfill received — instead of re-deriving it. own mix moves only its per-component masses. It is the default because a gate observation is downstream of whatever diversion happened upstream of it. `site_inflow.residual_composition` is the single function a later "a site - states its own mix" change replaces. + states its own mix" change replaces. In a year the city buries nothing that + mix is 0/0, and the fallback is the composition **as shares** — the + composition as masses is all-zero in exactly the years it would be needed, + which left the surplus with a zero mix and deposited none of it. A site taking + regional waste while its city has not started collecting yet is an ordinary + shape, and it silently lost every ton: 200,000 t/yr over thirty years came + back as 0 t CH4. Pinned by test. New behaviour lives in `SWEET_python/site_inflow.py`, following the convention `dst_common.build_landfill` already sets — a landfill's construction lives diff --git a/tests/test_site_other_source.py b/tests/test_site_other_source.py index 3f642c0..e91ae32 100644 --- a/tests/test_site_other_source.py +++ b/tests/test_site_other_source.py @@ -7,9 +7,12 @@ from private haulers, from a regional catchment the baseline never describes -- and the site's own operator knows its gate total, not the city's share of it. -So a spec may now carry ``other_source_mass``: absolute tons per year arriving -at that site from outside this baseline. Two consequences, and this module -exists to pin both: +So a spec may now carry ``accepted_waste_mass``: everything crossing that site's +weighbridge in a year, **from every source**, city waste included. It is a gate +total, not the outside share -- the outside share is what it exceeds the city's +own allocation by, which is the model's subtraction to do rather than the user's. +A figure at or below that allocation therefore means "no outside waste", not a +smaller city stream. Two consequences, and this module exists to pin both: **The city's own emissions must not move.** A city is responsible for the waste it generates, wherever that waste goes. Other-source waste is somebody else's @@ -51,7 +54,14 @@ GENERATED = 100_000.0 -def _spec(*, open_close=(1990, 2051), accepted=None, combusts=None): +def _spec( + *, + open_close=(1990, 2051), + accepted=None, + accepted_scenario=None, + combusts=None, + combusts_scenario=None, +): spec = dict( landfill_type={"baseline": 2, "scenario": 2}, landfill_open_close={"baseline": list(open_close), "scenario": list(open_close)}, @@ -61,13 +71,19 @@ def _spec(*, open_close=(1990, 2051), accepted=None, combusts=None): }, ) if accepted is not None: - spec["accepted_waste_mass"] = {"baseline": dict(accepted), "scenario": dict(accepted)} - if combusts is not None: - spec["combusts"] = {"baseline": combusts, "scenario": combusts} + spec["accepted_waste_mass"] = { + "baseline": dict(accepted), + "scenario": dict(accepted_scenario if accepted_scenario is not None else accepted), + } + if combusts is not None or combusts_scenario is not None: + spec["combusts"] = { + "baseline": bool(combusts), + "scenario": bool(combusts if combusts_scenario is None else combusts_scenario), + } return spec -def _request(specs, shares, *, diversion=None): +def _request(specs, shares, *, diversion=None, generated=None): extra = {} if diversion is not None: extra["diversion_fractions"] = { @@ -80,8 +96,8 @@ def _request(specs, shares, *, diversion=None): temperature=20.0, implement_year=IMPLEMENT_YEAR, waste_mass={ - "baseline": {y: GENERATED for y in YEARS}, - "scenario": {y: GENERATED for y in YEARS}, + "baseline": dict(generated) if generated else {y: GENERATED for y in YEARS}, + "scenario": dict(generated) if generated else {y: GENERATED for y in YEARS}, }, waste_fractions={ "baseline": {y: FRACTIONS for y in YEARS}, @@ -341,3 +357,113 @@ def test_the_limits_endpoint_ignores_the_new_field(): assert plain.keys() == with_other.keys() for variant in plain: assert str(plain[variant]) == str(with_other[variant]) + + +# --------------------------------------------------------------------------- # +# The mix the surplus is given, where the city has none to lend it +# --------------------------------------------------------------------------- # + +def test_outside_waste_still_has_a_composition_when_the_city_generates_none(): + """A regional site whose city has not started collecting yet. + + The surplus is split by the city's post-diversion residual mix, and in a + year the city buries nothing that mix is 0/0. The fallback has to be the + composition as *shares*: the composition as masses is all-zero in exactly + the years it would be needed, which left every share at zero and deposited + none of the site's inflow. Thirty years of a 200,000 t/yr site vanished, and + nothing said so. + """ + late = {y: (0.0 if y < 2030 else GENERATED) for y in YEARS} + result = run_advanced_dst_city( + _request( + [_spec(accepted={y: 200_000.0 for y in YEARS})], + (1.0,), + generated=late, + ), + with_site_emissions=True, + ) + site = result["site_emissions"]["baseline"][0] + other = _total(site.other) + before = np.array([other[i] for i, y in enumerate(YEARS) if y < 2030]) + + assert before.sum() > 0, "the site's own inflow produced no methane at all" + # It is the whole of the site in those years, the city having sent nothing. + city = _total(site.city) + city_before = np.array([city[i] for i, y in enumerate(YEARS) if y < 2030]) + assert np.allclose(city_before, 0.0) + assert np.allclose( + before, + np.array([_total(site.total)[i] for i, y in enumerate(YEARS) if y < 2030]), + ) + + +# --------------------------------------------------------------------------- # +# The scenario half is spliced to baseline before the implement year +# --------------------------------------------------------------------------- # + +def test_a_changed_gate_total_takes_effect_only_from_the_implement_year(): + """The surplus follows the same baseline-until-implement rule as everything else. + + The splice that enforces this for a changed *intake* is upstream, in + ``dst_common.variant_series`` -- it builds the scenario series equal to + baseline before ``implement_year`` already. This pins the behaviour end to + end rather than the line that implements it; the line is pinned by + ``test_a_site_that_starts_burning_...`` below, which exercises a difference + ``variant_series`` cannot see. + """ + doubled = {y: 300_000.0 for y in YEARS} + result = run_advanced_dst_city( + _request( + [_spec(accepted=GATE_WITH_NEIGHBOUR, accepted_scenario=doubled)], + (1.0,), + ), + with_site_emissions=True, + ) + baseline_other = _total(result["site_emissions"]["baseline"][0].other) + scenario_other = _total(result["site_emissions"]["scenario"][0].other) + + before = [i for i, y in enumerate(YEARS) if y < IMPLEMENT_YEAR] + after = [i for i, y in enumerate(YEARS) if y >= IMPLEMENT_YEAR] + + # Identical deposits before the implement year, so identical emissions. + assert np.allclose( + baseline_other[before], scenario_other[before], rtol=0, atol=1e-9 + ), "the scenario rewrote history" + # And strictly more afterwards, since the scenario takes 3x the waste. The + # gap opens gradually: the extra tonnage has to decay before it shows up. + assert scenario_other[after[-1]] > baseline_other[after[-1]] * 1.5 + + +def test_a_site_that_starts_burning_still_buries_the_surplus_until_then(): + """The frame-level splice, on a difference the series-level one cannot see. + + ``variant_series`` splices the intake, so a variant-differing gate total is + already baseline-before-implement by the time this module sees it. Two + things are not: the open/close window and ``combusts``, both applied to the + mass frame afterwards. A site that starts incinerating at the implement year + must still deposit the surplus in full before then -- and without the splice + it deposits only the 10% reject for the whole run, quietly erasing 90% of + decades of history that the scenario is not supposed to be able to rewrite. + """ + result = run_advanced_dst_city( + _request( + # 200 kt against an undiverted 100 kt city, so half the gate is + # surplus and there is something for the burning to bite on. + [_spec(accepted={y: 200_000.0 for y in YEARS}, combusts=False, combusts_scenario=True)], + (1.0,), + ), + with_site_emissions=True, + ) + baseline_other = _total(result["site_emissions"]["baseline"][0].other) + scenario_other = _total(result["site_emissions"]["scenario"][0].other) + + before = [i for i, y in enumerate(YEARS) if y < IMPLEMENT_YEAR] + after = [i for i, y in enumerate(YEARS) if y >= IMPLEMENT_YEAR] + + assert baseline_other[before].sum() > 0, "nothing was buried to compare" + assert np.allclose( + baseline_other[before], scenario_other[before], rtol=0, atol=1e-9 + ), "burning from the implement year reached back and unburied earlier waste" + # And from the implement year the scenario buries only the reject, so its + # emissions fall away from baseline's. + assert scenario_other[after[-1]] < baseline_other[after[-1]]