From bfbb551699bb8ce2ff2ea746a4428f2f619d1240 Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Tue, 8 Sep 2026 15:20:30 -0700 Subject: [PATCH 1/3] Model output frames have one column order, whatever the hash seed The waste-type eligibility collections on City -- `components` and each value of `div_components` -- were plain `set`s. They answer a membership question, so a set is the right shape; but they are also what the engines iterate to build DataFrame columns, and a `set` of `str` iterates in hash order, which Python randomizes per process (PEP 456). The same code on the same inputs therefore produced a different column order in every process: PYTHONHASHSEED=0 ['food', 'paper_cardboard', 'textiles', 'green', 'wood'] PYTHONHASHSEED=1 ['paper_cardboard', 'food', 'green', 'wood', 'textiles'] `WASTE_TYPES` in constants.py is now the single definition of that order -- the field order of WasteFractions/WasteMasses, so the frames built from a pydantic `model_dump()` and the frames built from an eligibility set finally agree -- and `WasteTypeSet` is a frozenset that iterates in it. Fixing this in the type rather than at the ~30 call sites means every `list(city.components)` and `for waste in div_components[div]` becomes deterministic with nothing to remember, here and in the Climate TRACE pipeline, which reaches into the same attributes. advanced_dst_city switches from `sorted()` (deterministic, but alphabetical and so a second order) to the same canonical one. Verified on 749 dumped model outputs / 297,806 numbers across five city-DST runs: the three hash seeds now produce a byte-identical dump, sha256 ab37f4d0. Column-aligned against the pre-change code the numbers are unchanged, except 307 cells of the derived `total` column at one seed, each off by exactly 1 ULP -- `total` is a sum ACROSS the columns and floating-point addition is not associative, so its last bit used to follow the hash seed too. --- SWEET_python/advanced_dst_city.py | 6 +- SWEET_python/city_params.py | 134 +++++++------ SWEET_python/constants.py | 106 +++++++++- tests/test_waste_type_order.py | 321 ++++++++++++++++++++++++++++++ 4 files changed, 506 insertions(+), 61 deletions(-) create mode 100644 tests/test_waste_type_order.py diff --git a/SWEET_python/advanced_dst_city.py b/SWEET_python/advanced_dst_city.py index b576132..1c70b47 100644 --- a/SWEET_python/advanced_dst_city.py +++ b/SWEET_python/advanced_dst_city.py @@ -309,7 +309,11 @@ def _diverted_masses( div_dfs: Dict[str, pd.DataFrame] = {} gross_dfs: Dict[str, pd.DataFrame] = {} for pathway in DIVERSION_PATHWAYS: - components = sorted(city.div_components[pathway]) # deterministic column order + # div_components values are WasteTypeSet, so iteration is already in + # canonical WASTE_TYPES order -- the same order every other frame in the + # package uses. This used to be sorted() (alphabetical), which was + # deterministic but disagreed with the rest of the model. + components = list(city.div_components[pathway]) sub = fractions_df[components] denom = sub.sum(axis=1) diff --git a/SWEET_python/city_params.py b/SWEET_python/city_params.py index 85fec97..489f0bc 100644 --- a/SWEET_python/city_params.py +++ b/SWEET_python/city_params.py @@ -29,7 +29,12 @@ from sqlalchemy.exc import OperationalError as SQLAlchemyOperationalError from datetime import datetime import time -from SWEET_python.constants import MODEL_START_YEAR, MODEL_END_YEAR +from SWEET_python.constants import ( + MODEL_START_YEAR, + MODEL_END_YEAR, + WASTE_TYPES, + WasteTypeSet, +) def _build_oxidation_series(default_value, canonical_row, time_series_rows, years_range): @@ -298,45 +303,40 @@ def __init__(self, city_name: str): self.iso3 = None self.baseline_parameters = None self.scenario_parameters = {} - self.components = {"food", "green", "wood", "paper_cardboard", "textiles"} + self.components = WasteTypeSet( + {"food", "green", "wood", "paper_cardboard", "textiles"} + ) self.div_components = { - "compost": {"food", "green", "wood", "paper_cardboard"}, - "anaerobic": {"food", "green", "wood", "paper_cardboard"}, - "combustion": { - "food", - "green", - "wood", - "paper_cardboard", - "textiles", - "plastic", - "rubber", - "metal", - "glass", - "other", - }, - "recycling": { - "wood", - "paper_cardboard", - "textiles", - "plastic", - "rubber", - "metal", - "glass", - "other", - }, + "compost": WasteTypeSet({"food", "green", "wood", "paper_cardboard"}), + "anaerobic": WasteTypeSet({"food", "green", "wood", "paper_cardboard"}), + "combustion": WasteTypeSet( + { + "food", + "green", + "wood", + "paper_cardboard", + "textiles", + "plastic", + "rubber", + "metal", + "glass", + "other", + } + ), + "recycling": WasteTypeSet( + { + "wood", + "paper_cardboard", + "textiles", + "plastic", + "rubber", + "metal", + "glass", + "other", + } + ), } - self.waste_types = [ - "food", - "green", - "wood", - "paper_cardboard", - "textiles", - "plastic", - "metal", - "glass", - "rubber", - "other", - ] + self.waste_types = list(WASTE_TYPES) self.unprocessable = { "food": 0.0192, "green": 0.042522, @@ -1098,22 +1098,28 @@ def load_andre_params(self, row, backfill=False): # ks = defaults_2019.k_defaults[precip_zone] # Model components - components = set(["food", "green", "wood", "paper_cardboard", "textiles"]) + components = WasteTypeSet( + ["food", "green", "wood", "paper_cardboard", "textiles"] + ) # Compost params - compost_components = set(["food", "green", "wood", "paper_cardboard"]) + compost_components = WasteTypeSet( + ["food", "green", "wood", "paper_cardboard"] + ) compost_fraction = float(row["waste_treatment_compost_percent"]) / 100 if np.isnan(compost_fraction): compost_fraction = 0.0 # Anaerobic digestion params - anaerobic_components = set(["food", "green", "wood", "paper_cardboard"]) + anaerobic_components = WasteTypeSet( + ["food", "green", "wood", "paper_cardboard"] + ) anaerobic_fraction = ( float(row["waste_treatment_anaerobic_digestion_percent"]) / 100 ) # Combustion params - combustion_components = set( + combustion_components = WasteTypeSet( [ "food", "green", @@ -1135,7 +1141,7 @@ def load_andre_params(self, row, backfill=False): combustion_fraction = (np.nan_to_num(value1) + np.nan_to_num(value2)) / 100 # Recycling params - recycling_components = set( + recycling_components = WasteTypeSet( [ "wood", "paper_cardboard", @@ -3199,12 +3205,16 @@ def import_basics(self, row) -> None: mef_compost = 0 # Model components - self.components = set(["food", "green", "wood", "paper_cardboard", "textiles"]) - self.compost_components = set( + self.components = WasteTypeSet( + ["food", "green", "wood", "paper_cardboard", "textiles"] + ) + self.compost_components = WasteTypeSet( ["food", "green", "wood", "paper_cardboard"] ) # Double check we don't want to include paper - self.anaerobic_components = set(["food", "green", "wood", "paper_cardboard"]) - self.combustion_components = set( + self.anaerobic_components = WasteTypeSet( + ["food", "green", "wood", "paper_cardboard"] + ) + self.combustion_components = WasteTypeSet( [ "food", "green", @@ -3218,7 +3228,7 @@ def import_basics(self, row) -> None: "other", ] ) - self.recycling_components = set( + self.recycling_components = WasteTypeSet( [ "wood", "paper_cardboard", @@ -3628,12 +3638,16 @@ def _is_transient_db_error(err: Exception) -> bool: mef_compost = 0 # Model components - self.components = set(["food", "green", "wood", "paper_cardboard", "textiles"]) - self.compost_components = set( + self.components = WasteTypeSet( + ["food", "green", "wood", "paper_cardboard", "textiles"] + ) + self.compost_components = WasteTypeSet( ["food", "green", "wood", "paper_cardboard"] ) # Double check we don't want to include paper - self.anaerobic_components = set(["food", "green", "wood", "paper_cardboard"]) - self.combustion_components = set( + self.anaerobic_components = WasteTypeSet( + ["food", "green", "wood", "paper_cardboard"] + ) + self.combustion_components = WasteTypeSet( [ "food", "green", @@ -3647,7 +3661,7 @@ def _is_transient_db_error(err: Exception) -> bool: "other", ] ) - self.recycling_components = set( + self.recycling_components = WasteTypeSet( [ "wood", "paper_cardboard", @@ -3917,12 +3931,16 @@ def _is_transient_db_error(err: Exception) -> bool: mef_compost = 0 # Model components - self.components = set(["food", "green", "wood", "paper_cardboard", "textiles"]) - self.compost_components = set( + self.components = WasteTypeSet( + ["food", "green", "wood", "paper_cardboard", "textiles"] + ) + self.compost_components = WasteTypeSet( ["food", "green", "wood", "paper_cardboard"] ) # Double check we don't want to include paper - self.anaerobic_components = set(["food", "green", "wood", "paper_cardboard"]) - self.combustion_components = set( + self.anaerobic_components = WasteTypeSet( + ["food", "green", "wood", "paper_cardboard"] + ) + self.combustion_components = WasteTypeSet( [ "food", "green", @@ -3936,7 +3954,7 @@ def _is_transient_db_error(err: Exception) -> bool: "other", ] ) - self.recycling_components = set( + self.recycling_components = WasteTypeSet( [ "wood", "paper_cardboard", diff --git a/SWEET_python/constants.py b/SWEET_python/constants.py index f47b5d7..d2f96b0 100644 --- a/SWEET_python/constants.py +++ b/SWEET_python/constants.py @@ -1,5 +1,6 @@ -"""Canonical modeling window shared across SWEET_python, the Climate TRACE -waste methane pipeline, and the WasteMAP backend. +"""Canonical definitions shared across SWEET_python, the Climate TRACE waste +methane pipeline, and the WasteMAP backend: the modeling window, and the order +of the waste types. Waste deposited before MODEL_START_YEAR is assumed to be zero everywhere: sites with earlier reported opening years keep their true opening year in @@ -29,3 +30,104 @@ MODEL_START_YEAR: int = 1970 MODEL_END_YEAR: int = 2050 + + +# --------------------------------------------------------------------------- +# Canonical waste-type ordering +# --------------------------------------------------------------------------- + +WASTE_TYPES: tuple[str, ...] = ( + "food", + "green", + "wood", + "paper_cardboard", + "textiles", + "plastic", + "metal", + "glass", + "rubber", + "other", +) +"""The one true order of the ten waste types. + +This is not an arbitrary choice: it is the field order of ``WasteFractions`` / +``WasteMasses`` in ``class_defs.py``. Those pydantic models already impose it on +every frame built from a ``model_dump()`` (the ``divs_df`` frames, for one), so +adopting the same sequence here makes the whole model agree on one column order +instead of two. ``tests/test_waste_type_order.py`` pins the two together. + +Waste-type collections in the model are *sets* -- eligibility answers the +question "can this type be composted?", which is membership, not sequence. But +they are also iterated to build DataFrame columns, and a ``set`` of ``str`` +iterates in hash order, which Python randomizes per process (PEP 456). Column +order therefore differed between two runs of the same code on the same inputs. +``WasteTypeSet`` below keeps the set semantics and fixes the iteration order. +""" + +_WASTE_TYPE_INDEX: dict[str, int] = {w: i for i, w in enumerate(WASTE_TYPES)} + + +class WasteTypeSet(frozenset): + """A set of waste types that always iterates in ``WASTE_TYPES`` order. + + Every ``list(city.components)``, ``for waste in city.div_components[div]`` + and ``{w: ... for w in components}`` in this package -- and in the Climate + TRACE pipeline and the WasteMAP backend, which reach into the same + attributes -- becomes deterministic by construction, with nothing to + remember at the call site. That is the point of doing it in the type rather + than sprinkling ``sorted()`` around: a ``sorted()`` that someone forgets to + add is silent, and it would also order columns alphabetically rather than in + the model's own order. + + A member outside ``WASTE_TYPES`` is not an error -- it sorts alphabetically + after the known types, so a caller experimenting with a new stream still + gets a stable order. + + Set operations return a ``WasteTypeSet`` rather than the base ``frozenset``, + so the guarantee survives ``eligible & combustible`` -- as long as the + ``WasteTypeSet`` is the left operand, since Python resolves the left + operand's ``__and__`` first and a plain ``set``'s wins. + """ + + __slots__ = () + + def __iter__(self): + known = [w for w in WASTE_TYPES if frozenset.__contains__(self, w)] + unknown = sorted( + w for w in frozenset.__iter__(self) if w not in _WASTE_TYPE_INDEX + ) + return iter(known + unknown) + + def __repr__(self) -> str: + return f"{type(self).__name__}({list(self)!r})" + + # frozenset's operators return the base class; re-wrap so a derived set is + # still ordered. Note the asymmetry Python imposes: in `plain_set & ordered` + # the LEFT operand's __and__ runs first and returns a plain set, so keep the + # WasteTypeSet on the left (or use the named methods below). + def __and__(self, other): + return type(self)(frozenset.__and__(self, other)) + + def __or__(self, other): + return type(self)(frozenset.__or__(self, other)) + + def __sub__(self, other): + return type(self)(frozenset.__sub__(self, other)) + + def __xor__(self, other): + return type(self)(frozenset.__xor__(self, other)) + + def union(self, *others): + return type(self)(frozenset.union(self, *others)) + + def intersection(self, *others): + return type(self)(frozenset.intersection(self, *others)) + + def difference(self, *others): + return type(self)(frozenset.difference(self, *others)) + + def symmetric_difference(self, other): + return type(self)(frozenset.symmetric_difference(self, other)) + + def copy(self): + return self diff --git a/tests/test_waste_type_order.py b/tests/test_waste_type_order.py new file mode 100644 index 0000000..a17baa7 --- /dev/null +++ b/tests/test_waste_type_order.py @@ -0,0 +1,321 @@ +"""Model output frames carry their waste-type columns in one fixed order. + +The eligibility collections on ``City`` (``components``, ``div_components``) +were plain ``set``s, and a ``set`` of ``str`` iterates in hash order, which +Python randomizes per process (PEP 456). They are also what the engines iterate +to build DataFrame columns, so the same code on the same inputs produced a +different column order in every process:: + + PYTHONHASHSEED=0 ['food', 'paper_cardboard', 'textiles', 'green', 'wood'] + PYTHONHASHSEED=1 ['paper_cardboard', 'food', 'green', 'wood', 'textiles'] + +Values were unaffected -- except for the derived ``total`` column, which is a +sum ACROSS those columns, and floating-point addition is not associative: 307 +of 297,806 dumped numbers moved by exactly 1 ULP depending on the seed. + +``WasteTypeSet`` keeps set semantics and pins iteration to ``WASTE_TYPES``. +These tests pin both halves: the type's own contract, and the column order of +real model output. +""" + +import copy +import json +import os +import subprocess +import sys +import textwrap + +import pytest + +from SWEET_python.class_defs import WasteFractions, WasteMasses +from SWEET_python.constants import WASTE_TYPES, WasteTypeSet +from SWEET_python.city_params import City, DiversionFractions + + +# The five degradable types the FOD engine models, in canonical order. +DEGRADABLE = ["food", "green", "wood", "paper_cardboard", "textiles"] + + +# --------------------------------------------------------------------------- # +# The canonical order itself +# --------------------------------------------------------------------------- # +def test_canonical_order_matches_the_pydantic_models(): + """WASTE_TYPES is the field order of WasteFractions / WasteMasses. + + Frames built from a ``model_dump()`` (every ``divs_df``) take their order + from those models; frames built from a component collection take it from + ``WASTE_TYPES``. If the two drift apart the package is back to two column + orders, so pin them together. + """ + assert WASTE_TYPES == tuple(WasteFractions.model_fields) + assert WASTE_TYPES == tuple(WasteMasses.model_fields) + + +def test_canonical_order_is_the_expected_ten_types(): + assert WASTE_TYPES == ( + "food", + "green", + "wood", + "paper_cardboard", + "textiles", + "plastic", + "metal", + "glass", + "rubber", + "other", + ) + + +# --------------------------------------------------------------------------- # +# WasteTypeSet +# --------------------------------------------------------------------------- # +class TestWasteTypeSet: + def test_iterates_in_canonical_order_whatever_the_construction_order(self): + forwards = WasteTypeSet(DEGRADABLE) + backwards = WasteTypeSet(reversed(DEGRADABLE)) + assert list(forwards) == DEGRADABLE + assert list(backwards) == DEGRADABLE + + def test_every_iteration_protocol_sees_the_same_order(self): + s = WasteTypeSet(DEGRADABLE) + assert list(s) == DEGRADABLE + assert tuple(s) == tuple(DEGRADABLE) + assert [w for w in s] == DEGRADABLE + # The dict-comprehension form the model uses to build frames. + assert list({w: 0.0 for w in s}) == DEGRADABLE + + def test_the_order_is_not_alphabetical(self): + """Guards against 'fixing' this with sorted(), which is a different order.""" + assert list(WasteTypeSet(DEGRADABLE)) != sorted(DEGRADABLE) + + def test_still_behaves_as_a_set(self): + s = WasteTypeSet(DEGRADABLE) + assert "food" in s + assert "plastic" not in s + assert len(s) == 5 + assert s == set(DEGRADABLE) + assert s == frozenset(DEGRADABLE) + + @pytest.mark.parametrize( + "op", + [ + lambda a, b: a & b, + lambda a, b: a | b, + lambda a, b: a - b, + lambda a, b: a ^ b, + lambda a, b: a.intersection(b), + lambda a, b: a.union(b), + lambda a, b: a.difference(b), + lambda a, b: a.symmetric_difference(b), + ], + ) + def test_set_operations_stay_ordered(self, op): + """frozenset's operators return the base class; ours must not.""" + result = op(WasteTypeSet(DEGRADABLE), WasteTypeSet(["food", "green", "metal"])) + assert isinstance(result, WasteTypeSet) + assert list(result) == [w for w in WASTE_TYPES if w in result] + + def test_mixing_with_a_plain_set_keeps_the_order_on_the_left(self): + """Python resolves the LEFT operand's __and__ first, and set's wins. + + So ``ordered & plain`` is ordered and ``plain & ordered`` is not -- + a language rule, not something the type can override. Documented on + WasteTypeSet; asserted here so the asymmetry is not a surprise. + """ + s = WasteTypeSet(DEGRADABLE) + assert isinstance(s & {"green", "food"}, WasteTypeSet) + assert not isinstance({"green", "food"} & s, WasteTypeSet) + + def test_unknown_members_sort_after_the_known_ones(self): + s = WasteTypeSet(["zebra", "other", "food", "aardvark"]) + assert list(s) == ["food", "other", "aardvark", "zebra"] + + def test_survives_deepcopy(self): + """The DST deep-copies baseline parameters to build a scenario.""" + s = copy.deepcopy(WasteTypeSet(DEGRADABLE)) + assert isinstance(s, WasteTypeSet) + assert list(s) == DEGRADABLE + + +# --------------------------------------------------------------------------- # +# The City collections +# --------------------------------------------------------------------------- # +class TestCityCollections: + def test_components_and_eligibility_are_ordered_sets(self): + """A plain set here is the bug; catch it at the source, not downstream.""" + city = City("x") + assert isinstance(city.components, WasteTypeSet) + for div, eligible in city.div_components.items(): + assert isinstance(eligible, WasteTypeSet), div + + def test_components_iterate_in_canonical_order(self): + city = City("x") + assert list(city.components) == DEGRADABLE + assert list(city.div_components["compost"]) == [ + "food", + "green", + "wood", + "paper_cardboard", + ] + assert list(city.div_components["recycling"]) == [ + "wood", + "paper_cardboard", + "textiles", + "plastic", + "metal", + "glass", + "rubber", + "other", + ] + + def test_waste_types_list_is_the_canonical_order(self): + assert City("x").waste_types == list(WASTE_TYPES) + + +# --------------------------------------------------------------------------- # +# Real model output +# --------------------------------------------------------------------------- # +@pytest.fixture(scope="module") +def dst_run(): + """One city-DST run: blank-city baseline plus a four-stream scenario.""" + city = City("x") + city.dst_baseline_blank("Algeria", 2_594_000, 716.81, 18.38) + city.implement_dst_changes_simple_v1_5( + DiversionFractions(compost=0.20, anaerobic=0.10, combustion=0.10, recycling=0.20), + 0, + 0, + 0.0, + 0.0, + 2026, + 1, + 0.10, + ) + return city + + +class TestModelOutputColumnOrder: + def test_landfill_emissions_frames_are_pinned(self, dst_run): + """The frame in the bug report: a landfill's per-waste-type CH4.""" + landfill = dst_run.baseline_parameters.landfills[0] + assert list(landfill.ch4.columns) == DEGRADABLE + assert list(landfill.captured.columns) == DEGRADABLE + assert list(landfill.waste_mass_after_degredation.columns) == DEGRADABLE + # `emissions` carries the derived cross-column sum as a final column. + assert list(landfill.emissions.columns) == DEGRADABLE + ["total"] + + def test_divs_frames_are_pinned(self, dst_run): + """Diverted mass per stream -- each stream's eligible types, in order. + + The baseline frames are built from ``div_components``, so they carry + only the eligible types: this is the path the hash seed used to move. + """ + divs = dst_run.baseline_parameters.divs_df + assert list(divs.compost.columns) == ["food", "green", "wood", "paper_cardboard"] + assert list(divs.anaerobic.columns) == ["food", "green", "wood", "paper_cardboard"] + assert list(divs.combustion.columns) == list(WASTE_TYPES) + assert list(divs.recycling.columns) == [ + "wood", + "paper_cardboard", + "textiles", + "plastic", + "metal", + "glass", + "rubber", + "other", + ] + + def test_both_divs_construction_paths_agree_on_the_order(self, dst_run): + """The scenario frames come from a pydantic ``model_dump()`` instead. + + That path was always deterministic -- but it ordered columns by the + model's field order while the eligibility path ordered them by hash. + Now both are WASTE_TYPES, which is the point of pinning the canonical + order to the pydantic models. + """ + divs = dst_run.scenario_parameters[0].divs_df + for div in ("compost", "anaerobic", "combustion", "recycling"): + assert list(getattr(divs, div).columns) == list(WASTE_TYPES), div + + def test_component_fraction_frames_are_pinned(self, dst_run): + fracs = dst_run.baseline_parameters.div_component_fractions + for div in ("compost", "anaerobic", "combustion", "recycling"): + cols = list(getattr(fracs, div).columns) + assert cols == [w for w in WASTE_TYPES if w in set(cols)], div + + def test_every_waste_type_frame_follows_the_canonical_order(self, dst_run): + """The general invariant, over every frame the two runs expose.""" + checked = 0 + for params in (dst_run.baseline_parameters, dst_run.scenario_parameters[0]): + for landfill in params.landfills: + for attr in ("ch4", "captured", "emissions", "waste_mass_after_degredation"): + frame = getattr(landfill, attr, None) + if frame is None: + continue + cols = [c for c in frame.columns if c != "total"] + assert cols == [w for w in WASTE_TYPES if w in set(cols)], (attr, cols) + checked += 1 + assert checked > 0 + + def test_waste_mass_df_is_alphabetical_by_index_union(self, dst_run): + """A KNOWN second order, deliberately left alone -- not the bug. + + ``waste_mass_df`` is ``waste_generated_df - DivsDF.sum()``, and + ``sum()`` builds its columns with ``Index.union``, which sorts. That is + deterministic, so it was never part of the hash-seed problem, and those + lines are being edited by an open PR. Pinned here so the remaining + inconsistency is visible rather than folklore; it affects nothing + downstream, because every consumer slices this frame by name. + """ + frame = dst_run.baseline_parameters.landfills[0].waste_mass_df + assert list(frame.columns) == sorted(WASTE_TYPES) + + +# --------------------------------------------------------------------------- # +# The property itself: same inputs, different process, same columns +# --------------------------------------------------------------------------- # +_CHILD = textwrap.dedent( + """ + import json + from SWEET_python.city_params import City, DiversionFractions + + city = City("x") + city.dst_baseline_blank("Algeria", 2_594_000, 716.81, 18.38) + city.implement_dst_changes_simple_v1_5( + DiversionFractions(compost=0.20, anaerobic=0.10, combustion=0.10, recycling=0.20), + 0, 0, 0.0, 0.0, 2026, 1, 0.10) + + baseline = city.baseline_parameters + scenario = city.scenario_parameters[0] + print(json.dumps({ + "components": list(city.components), + "ch4": list(baseline.landfills[0].ch4.columns), + "emissions": list(baseline.landfills[0].emissions.columns), + "divs_compost": list(scenario.divs_df.compost.columns), + "divs_recycling": list(scenario.divs_df.recycling.columns), + # The derived total is a sum ACROSS the columns, so its last bit + # followed the column order. repr() round-trips a float exactly. + "total_1971": repr(float(baseline.landfills[0].emissions["total"].iloc[1])), + })) + """ +) + + +def _columns_under_hash_seed(seed): + env = dict(os.environ, PYTHONHASHSEED=str(seed)) + out = subprocess.run( + [sys.executable, "-c", _CHILD], + env=env, + capture_output=True, + text=True, + check=True, + ) + return json.loads(out.stdout) + + +def test_column_order_does_not_depend_on_the_hash_seed(): + """The regression itself. Fails on the pre-fix code, whatever the seeds. + + Two subprocesses because PYTHONHASHSEED is read once at interpreter start: + it cannot be varied in-process. + """ + assert _columns_under_hash_seed(0) == _columns_under_hash_seed(1) From 359eae01ab7d9819b80aa9d3483e6cb9ad7a4745 Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Tue, 8 Sep 2026 15:22:52 -0700 Subject: [PATCH 2/3] Changelog for #63 --- changelog/2026-09.md | 39 ++++++++++++++++++++++++++++++++++++++- changelog/README.md | 2 +- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/changelog/2026-09.md b/changelog/2026-09.md index e3af7e7..6868b95 100644 --- a/changelog/2026-09.md +++ b/changelog/2026-09.md @@ -1,6 +1,10 @@ # SWEET_python Changelog — September 2026 -**Highlights:** The annual model was charging a flare's inefficiency twice — a +**Highlights:** Model output frames now come out with their waste-type columns +in one fixed order instead of a per-process one — a `set` fed the ordering, and +Python randomizes set iteration per process, so the same code on the same inputs +diffed dirty against itself. No output-value change. Earlier in the month: the +annual model was charging a flare's inefficiency twice — a hardcoded 2% slip stacked on top of a `flaring` destruction efficiency that already accounted for it — which also made the annual engine disagree with the monthly one by ~3.7% on identical inputs. A site with gas capture now emits @@ -164,6 +168,28 @@ landfill received — instead of re-deriving it. ## Fixed +- **Model output column order no longer depends on `PYTHONHASHSEED`.** + `City.components` and the values of `City.div_components` were plain `set`s. + A set is the right shape for eligibility — that is a membership question — but + they are also what the engines iterate to build DataFrame columns, and a `set` + of `str` iterates in hash order, which Python randomizes per process (PEP 456). + The same code on the same inputs therefore produced a different column order in + every process: 356 of 749 dumped model outputs changed order between two seeds. + `WASTE_TYPES` in `constants.py` is now the single definition of the order — the + field order of `WasteFractions`/`WasteMasses`, so the frames built from a + pydantic `model_dump()` and the frames built from an eligibility set finally + agree — and `WasteTypeSet` is a `frozenset` that iterates in it, which makes all + 35 `list(...)` conversions and 45 iterations deterministic without touching a + call site (including the Climate TRACE pipeline's, which reach into the same + attributes). `advanced_dst_city` moves off its lone alphabetical `sorted()` onto + the same order. **No output-value change:** 749 outputs / 297,806 numbers dumped + at full precision are byte-identical across three hash seeds, and column-aligned + against the previous code every per-waste-type number is unchanged. The only + cells that move are 307 in the derived `total` column at one seed, each by + exactly 1 ULP — `total` is a sum *across* the columns and floating-point + addition is not associative, so its last bit used to follow the hash seed too. + ([#63](https://github.com/RMI/SWEET_python/pull/63)) + - **Over-diversion is measured before rejection.** The guard compared *net* diverted mass against what the city generates, but rejects stay in the waste stream and are landfilled as their own material — so a pathway fed more of a @@ -237,6 +263,7 @@ landfill received — instead of re-deriving it. [RMI/WasteMAP#776](https://github.com/RMI/WasteMAP/pull/776)) + - **A caller can now say a landfill never closes.** `MODEL_YEAR_MAX` was the hard ceiling on `landfill_open_close`, but a closure year is not an intake year (`apply_window` zeroes from it inclusive), so a site submitted as closing @@ -325,6 +352,15 @@ landfill received — instead of re-deriving it. ## Known limitations +- **`DivsDF.sum()` still orders its columns alphabetically.** It builds them with + `Index.union`, which sorts, so that frame and the `waste_mass_df` derived from + it use a different order from the rest of the model. Deterministic, so it was + never part of the hash-seed problem, and it affects nothing downstream because + every consumer of `waste_mass_df` slices it by name — but it is a second order, + pinned by a test until it can be unified. + ([#63](https://github.com/RMI/SWEET_python/pull/63)) + + - **An incineration facility's residue carries the city's waste composition**, exactly as the combustion diversion pathway's reject always has. Real incinerator bottom ash is far more inert than that, so the @@ -338,6 +374,7 @@ landfill received — instead of re-deriving it. frontend calls the city-level endpoint for both its city and site modes, so nothing user-facing depends on this. ([#53](https://github.com/RMI/SWEET_python/pull/53)) + - **Collected gas that is vented is still oxidised as if it left through the cover.** It left through a stack. At `capture 0.6`, `oxidation 0.22` and `flaring 0`, the model gives `0.78 × generation` where the physical answer is diff --git a/changelog/README.md b/changelog/README.md index ed87b39..ab277d5 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -15,7 +15,7 @@ The project does not publish semantic version tags, so releases are tracked by Newest first: -- [2026-09](2026-09.md) — City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart +- [2026-09](2026-09.md) — Model output frames get one fixed waste-type column order instead of a per-process one (a `set` fed the ordering, and Python randomizes set iteration per process), with no output-value change; City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart - [2026-08](2026-08.md) — Single-site adst gains optional `depth` (deep-dump MCF bump) and `k_override` (caller-supplied decomposition rate) inputs, restoring the last two site-DST levers; `/sdst` flaring efficiency reaches the model again after a variable-name bug silently forced flare destruction to 0.98; annual model applies cover oxidation by emission year not deposit year, fixing biocover having no effect on closed landfills (WasteMAP #719); `City.sdst_v1_5` custom-site path holds the scenario equal to the baseline before the implementation year even when composition changes (was back-dating the new composition onto pre-implementation deposits) (model-output change); MCF consolidated into a new `SWEET_python.mcf` module and both dump types moved to the IPCC uncategorised-SWDS 0.6 (open dumps up from 0.4, controlled dumps down from 0.7), with a supplied depth now selecting the deep/shallow category (model-output change) - [2026-07](2026-07.md) — All ten waste types eligible for combustion (metal/glass/other added); methane-only model treats combustion as landfill diversion (model-output change) - [2026-06](2026-06.md) — New single-site and city-level ADST modeling modules, min-cost max-flow rewrite of the city DST diversion allocator, physical-k fix for cold/dry sites, no more spurious negative food-waste mass From f44c100192023021dd5202c62ea4b38a266ac1c2 Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Tue, 8 Sep 2026 18:13:17 -0700 Subject: [PATCH 3/3] Say that the reorder moves values in the last bit, because it does Copilot review on #63: the entry claimed "No output-value change" and "the only cells that move are 307 in the derived `total` column", while the same paragraph described `advanced_dst_city` moving off its alphabetical `sorted()`. Those two statements cannot both hold -- `_diverted_masses` does `denom = sub.sum(axis=1)` across exactly those reordered columns, and floating-point addition is not associative. Verified rather than argued. Summing a Dirichlet-random composition in canonical versus alphabetical order differs for 24-52% of draws per pathway (max 5.5e-16 relative). End to end on a 137,414 t/yr city diverting to compost, recycling and combustion: per-component diverted mass moves by up to 1.5e-16 relative (9.1e-13 t) and modelled emissions by up to 8.9e-16. So per-waste-type numbers do move, not just `total`. The entry now separates the two claims: byte-identical across hash seeds (the point of the PR), and a 1-ULP change against the previous code wherever a sum runs across the columns, with both sites named. Highlights and the changelog README summary say the same. Labelled model-output-change accordingly. 206 passed. Co-Authored-By: Claude Opus 5 --- changelog/2026-09.md | 26 +++++++++++++++++++------- changelog/README.md | 2 +- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/changelog/2026-09.md b/changelog/2026-09.md index 6868b95..d171107 100644 --- a/changelog/2026-09.md +++ b/changelog/2026-09.md @@ -3,7 +3,8 @@ **Highlights:** Model output frames now come out with their waste-type columns in one fixed order instead of a per-process one — a `set` fed the ordering, and Python randomizes set iteration per process, so the same code on the same inputs -diffed dirty against itself. No output-value change. Earlier in the month: the +diffed dirty against itself. Values move in the last bit only, where a sum runs +across the reordered columns. Earlier in the month: the annual model was charging a flare's inefficiency twice — a hardcoded 2% slip stacked on top of a `flaring` destruction efficiency that already accounted for it — which also made the annual engine disagree with the @@ -182,12 +183,23 @@ landfill received — instead of re-deriving it. 35 `list(...)` conversions and 45 iterations deterministic without touching a call site (including the Climate TRACE pipeline's, which reach into the same attributes). `advanced_dst_city` moves off its lone alphabetical `sorted()` onto - the same order. **No output-value change:** 749 outputs / 297,806 numbers dumped - at full precision are byte-identical across three hash seeds, and column-aligned - against the previous code every per-waste-type number is unchanged. The only - cells that move are 307 in the derived `total` column at one seed, each by - exactly 1 ULP — `total` is a sum *across* the columns and floating-point - addition is not associative, so its last bit used to follow the hash seed too. + the same order. + + **Deterministic across hash seeds:** 749 outputs / 297,806 numbers dumped at + full precision are byte-identical across three hash seeds, where 356 of those + 749 changed column order between two seeds before. + + **Against the previous code, values move in the last bit only.** Anywhere a + sum runs *across* the waste-type columns, reordering them changes the result + by an ULP or so, because floating-point addition is not associative. Two + places: the derived `total` column (307 cells at one seed, 1 ULP each), and + `_diverted_masses` in `advanced_dst_city`, where `denom = sub.sum(axis=1)` is + the pool a pathway draws on and divides into its mass. Measured on a city + diverting to compost, recycling and combustion, canonical order against the + old alphabetical one: per-component diverted mass moves by up to 1.5e-16 + relative (9.1e-13 t on a 137,414 t/yr city) and modelled emissions by up to + 8.9e-16. So this is not a no-op against the old outputs — it is a 1-ULP + change, in exchange for the same numbers in every process. ([#63](https://github.com/RMI/SWEET_python/pull/63)) - **Over-diversion is measured before rejection.** The guard compared *net* diff --git a/changelog/README.md b/changelog/README.md index ab277d5..0393293 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -15,7 +15,7 @@ The project does not publish semantic version tags, so releases are tracked by Newest first: -- [2026-09](2026-09.md) — Model output frames get one fixed waste-type column order instead of a per-process one (a `set` fed the ordering, and Python randomizes set iteration per process), with no output-value change; City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart +- [2026-09](2026-09.md) — Model output frames get one fixed waste-type column order instead of a per-process one (a `set` fed the ordering, and Python randomizes set iteration per process), moving values in the last bit only, where a sum runs across the reordered columns; City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart - [2026-08](2026-08.md) — Single-site adst gains optional `depth` (deep-dump MCF bump) and `k_override` (caller-supplied decomposition rate) inputs, restoring the last two site-DST levers; `/sdst` flaring efficiency reaches the model again after a variable-name bug silently forced flare destruction to 0.98; annual model applies cover oxidation by emission year not deposit year, fixing biocover having no effect on closed landfills (WasteMAP #719); `City.sdst_v1_5` custom-site path holds the scenario equal to the baseline before the implementation year even when composition changes (was back-dating the new composition onto pre-implementation deposits) (model-output change); MCF consolidated into a new `SWEET_python.mcf` module and both dump types moved to the IPCC uncategorised-SWDS 0.6 (open dumps up from 0.4, controlled dumps down from 0.7), with a supplied depth now selecting the deep/shallow category (model-output change) - [2026-07](2026-07.md) — All ten waste types eligible for combustion (metal/glass/other added); methane-only model treats combustion as landfill diversion (model-output change) - [2026-06](2026-06.md) — New single-site and city-level ADST modeling modules, min-cost max-flow rewrite of the city DST diversion allocator, physical-k fix for cold/dry sites, no more spurious negative food-waste mass