Skip to content

Model output frames have one column order, whatever the hash seed #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HughRunyan merged 3 commits into from
Sep 9, 2026
Merged

Model output frames have one column order, whatever the hash seed #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bfbb551
Model output frames have one column order, whatever the hash seed
HughRunyan Sep 8, 2026
359eae0
Changelog for #63
HughRunyan Sep 8, 2026
f44c100
Say that the reorder moves values in the last bit, because it does
HughRunyan Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion SWEET_python/advanced_dst_city.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,11 @@ def _diverted_masses(
div_dfs: Dict[str, pd.DataFrame] = {}
gross_dfs: Dict[str, pd.DataFrame] = {}
for pathway in DIVERSION_PATHWAYS:
components = sorted(city.div_components[pathway]) # deterministic column order
# div_components values are WasteTypeSet, so iteration is already in
# canonical WASTE_TYPES order -- the same order every other frame in the
# package uses. This used to be sorted() (alphabetical), which was
# deterministic but disagreed with the rest of the model.
components = list(city.div_components[pathway])
sub = fractions_df[components]
denom = sub.sum(axis=1)

Expand Down
134 changes: 76 additions & 58 deletions SWEET_python/city_params.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,12 @@
from sqlalchemy.exc import OperationalError as SQLAlchemyOperationalError
from datetime import datetime
import time
from SWEET_python.constants import MODEL_START_YEAR, MODEL_END_YEAR
from SWEET_python.constants import (
MODEL_START_YEAR,
MODEL_END_YEAR,
WASTE_TYPES,
WasteTypeSet,
)


def _build_oxidation_series(default_value, canonical_row, time_series_rows, years_range):
Expand Down Expand Up @@ -298,45 +303,40 @@ def __init__(self, city_name: str):
self.iso3 = None
self.baseline_parameters = None
self.scenario_parameters = {}
self.components = {"food", "green", "wood", "paper_cardboard", "textiles"}
self.components = WasteTypeSet(
{"food", "green", "wood", "paper_cardboard", "textiles"}
)
self.div_components = {
"compost": {"food", "green", "wood", "paper_cardboard"},
"anaerobic": {"food", "green", "wood", "paper_cardboard"},
"combustion": {
"food",
"green",
"wood",
"paper_cardboard",
"textiles",
"plastic",
"rubber",
"metal",
"glass",
"other",
},
"recycling": {
"wood",
"paper_cardboard",
"textiles",
"plastic",
"rubber",
"metal",
"glass",
"other",
},
"compost": WasteTypeSet({"food", "green", "wood", "paper_cardboard"}),
"anaerobic": WasteTypeSet({"food", "green", "wood", "paper_cardboard"}),
"combustion": WasteTypeSet(
{
"food",
"green",
"wood",
"paper_cardboard",
"textiles",
"plastic",
"rubber",
"metal",
"glass",
"other",
}
),
"recycling": WasteTypeSet(
{
"wood",
"paper_cardboard",
"textiles",
"plastic",
"rubber",
"metal",
"glass",
"other",
}
),
}
self.waste_types = [
"food",
"green",
"wood",
"paper_cardboard",
"textiles",
"plastic",
"metal",
"glass",
"rubber",
"other",
]
self.waste_types = list(WASTE_TYPES)
self.unprocessable = {
"food": 0.0192,
"green": 0.042522,
Expand Down Expand Up @@ -1098,22 +1098,28 @@ def load_andre_params(self, row, backfill=False):
# ks = defaults_2019.k_defaults[precip_zone]

# Model components
components = set(["food", "green", "wood", "paper_cardboard", "textiles"])
components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard", "textiles"]
)

# Compost params
compost_components = set(["food", "green", "wood", "paper_cardboard"])
compost_components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard"]
)
compost_fraction = float(row["waste_treatment_compost_percent"]) / 100
if np.isnan(compost_fraction):
compost_fraction = 0.0

# Anaerobic digestion params
anaerobic_components = set(["food", "green", "wood", "paper_cardboard"])
anaerobic_components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard"]
)
anaerobic_fraction = (
float(row["waste_treatment_anaerobic_digestion_percent"]) / 100
)

# Combustion params
combustion_components = set(
combustion_components = WasteTypeSet(
[
"food",
"green",
Expand All @@ -1135,7 +1141,7 @@ def load_andre_params(self, row, backfill=False):
combustion_fraction = (np.nan_to_num(value1) + np.nan_to_num(value2)) / 100

# Recycling params
recycling_components = set(
recycling_components = WasteTypeSet(
[
"wood",
"paper_cardboard",
Expand Down Expand Up @@ -3199,12 +3205,16 @@ def import_basics(self, row) -> None:
mef_compost = 0

# Model components
self.components = set(["food", "green", "wood", "paper_cardboard", "textiles"])
self.compost_components = set(
self.components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard", "textiles"]
)
self.compost_components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard"]
) # Double check we don't want to include paper
self.anaerobic_components = set(["food", "green", "wood", "paper_cardboard"])
self.combustion_components = set(
self.anaerobic_components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard"]
)
self.combustion_components = WasteTypeSet(
[
"food",
"green",
Expand All @@ -3218,7 +3228,7 @@ def import_basics(self, row) -> None:
"other",
]
)
self.recycling_components = set(
self.recycling_components = WasteTypeSet(
[
"wood",
"paper_cardboard",
Expand Down Expand Up @@ -3628,12 +3638,16 @@ def _is_transient_db_error(err: Exception) -> bool:
mef_compost = 0

# Model components
self.components = set(["food", "green", "wood", "paper_cardboard", "textiles"])
self.compost_components = set(
self.components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard", "textiles"]
)
self.compost_components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard"]
) # Double check we don't want to include paper
self.anaerobic_components = set(["food", "green", "wood", "paper_cardboard"])
self.combustion_components = set(
self.anaerobic_components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard"]
)
self.combustion_components = WasteTypeSet(
[
"food",
"green",
Expand All @@ -3647,7 +3661,7 @@ def _is_transient_db_error(err: Exception) -> bool:
"other",
]
)
self.recycling_components = set(
self.recycling_components = WasteTypeSet(
[
"wood",
"paper_cardboard",
Expand Down Expand Up @@ -3917,12 +3931,16 @@ def _is_transient_db_error(err: Exception) -> bool:
mef_compost = 0

# Model components
self.components = set(["food", "green", "wood", "paper_cardboard", "textiles"])
self.compost_components = set(
self.components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard", "textiles"]
)
self.compost_components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard"]
) # Double check we don't want to include paper
self.anaerobic_components = set(["food", "green", "wood", "paper_cardboard"])
self.combustion_components = set(
self.anaerobic_components = WasteTypeSet(
["food", "green", "wood", "paper_cardboard"]
)
self.combustion_components = WasteTypeSet(
[
"food",
"green",
Expand All @@ -3936,7 +3954,7 @@ def _is_transient_db_error(err: Exception) -> bool:
"other",
]
)
self.recycling_components = set(
self.recycling_components = WasteTypeSet(
[
"wood",
"paper_cardboard",
Expand Down
106 changes: 104 additions & 2 deletions SWEET_python/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Canonical modeling window shared across SWEET_python, the Climate TRACE
waste methane pipeline, and the WasteMAP backend.
"""Canonical definitions shared across SWEET_python, the Climate TRACE waste
methane pipeline, and the WasteMAP backend: the modeling window, and the order
of the waste types.

Waste deposited before MODEL_START_YEAR is assumed to be zero everywhere:
sites with earlier reported opening years keep their true opening year in
Expand Down Expand Up @@ -29,3 +30,104 @@

MODEL_START_YEAR: int = 1970
MODEL_END_YEAR: int = 2050


# ---------------------------------------------------------------------------
# Canonical waste-type ordering
# ---------------------------------------------------------------------------

WASTE_TYPES: tuple[str, ...] = (
"food",
"green",
"wood",
"paper_cardboard",
"textiles",
"plastic",
"metal",
"glass",
"rubber",
"other",
)
"""The one true order of the ten waste types.

This is not an arbitrary choice: it is the field order of ``WasteFractions`` /
``WasteMasses`` in ``class_defs.py``. Those pydantic models already impose it on
every frame built from a ``model_dump()`` (the ``divs_df`` frames, for one), so
adopting the same sequence here makes the whole model agree on one column order
instead of two. ``tests/test_waste_type_order.py`` pins the two together.

Waste-type collections in the model are *sets* -- eligibility answers the
question "can this type be composted?", which is membership, not sequence. But
they are also iterated to build DataFrame columns, and a ``set`` of ``str``
iterates in hash order, which Python randomizes per process (PEP 456). Column
order therefore differed between two runs of the same code on the same inputs.
``WasteTypeSet`` below keeps the set semantics and fixes the iteration order.
"""

_WASTE_TYPE_INDEX: dict[str, int] = {w: i for i, w in enumerate(WASTE_TYPES)}


class WasteTypeSet(frozenset):
"""A set of waste types that always iterates in ``WASTE_TYPES`` order.

Every ``list(city.components)``, ``for waste in city.div_components[div]``
and ``{w: ... for w in components}`` in this package -- and in the Climate
TRACE pipeline and the WasteMAP backend, which reach into the same
attributes -- becomes deterministic by construction, with nothing to
remember at the call site. That is the point of doing it in the type rather
than sprinkling ``sorted()`` around: a ``sorted()`` that someone forgets to
add is silent, and it would also order columns alphabetically rather than in
the model's own order.

A member outside ``WASTE_TYPES`` is not an error -- it sorts alphabetically
after the known types, so a caller experimenting with a new stream still
gets a stable order.

Set operations return a ``WasteTypeSet`` rather than the base ``frozenset``,
so the guarantee survives ``eligible & combustible`` -- as long as the
``WasteTypeSet`` is the left operand, since Python resolves the left
operand's ``__and__`` first and a plain ``set``'s wins.
"""

__slots__ = ()

def __iter__(self):
known = [w for w in WASTE_TYPES if frozenset.__contains__(self, w)]
unknown = sorted(
w for w in frozenset.__iter__(self) if w not in _WASTE_TYPE_INDEX
)
return iter(known + unknown)

def __repr__(self) -> str:
return f"{type(self).__name__}({list(self)!r})"

# frozenset's operators return the base class; re-wrap so a derived set is
# still ordered. Note the asymmetry Python imposes: in `plain_set & ordered`
# the LEFT operand's __and__ runs first and returns a plain set, so keep the
# WasteTypeSet on the left (or use the named methods below).
def __and__(self, other):
return type(self)(frozenset.__and__(self, other))

def __or__(self, other):
return type(self)(frozenset.__or__(self, other))

def __sub__(self, other):
return type(self)(frozenset.__sub__(self, other))

def __xor__(self, other):
return type(self)(frozenset.__xor__(self, other))

def union(self, *others):
return type(self)(frozenset.union(self, *others))

def intersection(self, *others):
return type(self)(frozenset.intersection(self, *others))

def difference(self, *others):
return type(self)(frozenset.difference(self, *others))

def symmetric_difference(self, other):
return type(self)(frozenset.symmetric_difference(self, other))

def copy(self):
return self
Loading