From 0f46e249bc47a5ff8bcbc0b60ea48361def10eda Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Tue, 8 Sep 2026 15:03:13 -0700 Subject: [PATCH 1/2] The diversion sum keeps working when pandas removes the copy keyword The four reindex(...).infer_objects(...).fillna(0) chains in DivsDF.sum passed copy=False. Since pandas 3.0 made Copy-on-Write unconditional the keyword is ignored: infer_objects hands it to _check_copy_deprecation, which warns and returns, and nothing else in the method reads it. So dropping it is exactly the call that was already running. Keeping it was not free. It emitted 96 Pandas4Warnings per test run, and in pandas 4 it raises TypeError. The break is worse than a TypeError, because every caller of DivsDF.sum wraps it in a bare `except:` whose fallback -- sum(divs_df.values()) -- does not work on a pydantic model. The real failure would surface as AttributeError: 'DivsDF' object has no attribute 'values' pointing nowhere near the cause. The new test simulates the removal and pins both the sum and that caller. infer_objects() itself stays. It does nothing for the reindex -- columns that reindex adds arrive as float64 NaN whatever the source dtypes, which is checked against float, int, 0-row and 0-column sources -- but it is what stops an object-dtype input column from surviving fillna(0) and dragging the summed frame to object. No output change. Five city-DST scenarios, including the food-waste prevention path that drives this code, produce a byte-identical dump of 73,760 numbers across 165 model outputs before and after (at a fixed PYTHONHASHSEED; column order varies with the hash seed either way, which is a separate pre-existing quirk and does not touch values). Co-Authored-By: Claude Opus 5 --- SWEET_python/class_defs.py | 29 +++++---- tests/test_divs_sum_dtypes.py | 117 ++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 tests/test_divs_sum_dtypes.py diff --git a/SWEET_python/class_defs.py b/SWEET_python/class_defs.py index 6f3b871..93a59d1 100644 --- a/SWEET_python/class_defs.py +++ b/SWEET_python/class_defs.py @@ -396,20 +396,23 @@ def sum(self) -> pd.DataFrame: .union(self.recycling.columns) ) - # Reindex and fill missing values, then infer object types + # Reindex each stream to the union of columns, then fill the gaps with 0. + # + # infer_objects() is not here for the reindex: columns that reindex adds + # always arrive as float64 NaN, whatever the source dtypes. It is here for + # an input frame that already carries its numbers in an object column + # (a Series built from None, a value that arrived boxed), where a bare + # fillna(0) would leave the column object-dtyped and the summed frame + # would inherit that. It is a no-op on the float64 frames this repo + # builds, and cheap insurance on the ones callers hand in. + # + # No copy= keyword: it has been ignored since pandas 3.0 made + # Copy-on-Write unconditional, and passing it is deprecated for removal. divs_list = [ - self.compost.reindex(columns=all_columns) - .infer_objects(copy=False) - .fillna(0), - self.anaerobic.reindex(columns=all_columns) - .infer_objects(copy=False) - .fillna(0), - self.combustion.reindex(columns=all_columns) - .infer_objects(copy=False) - .fillna(0), - self.recycling.reindex(columns=all_columns) - .infer_objects(copy=False) - .fillna(0), + self.compost.reindex(columns=all_columns).infer_objects().fillna(0), + self.anaerobic.reindex(columns=all_columns).infer_objects().fillna(0), + self.combustion.reindex(columns=all_columns).infer_objects().fillna(0), + self.recycling.reindex(columns=all_columns).infer_objects().fillna(0), ] return sum(divs_list) diff --git a/tests/test_divs_sum_dtypes.py b/tests/test_divs_sum_dtypes.py new file mode 100644 index 0000000..8942494 --- /dev/null +++ b/tests/test_divs_sum_dtypes.py @@ -0,0 +1,117 @@ +"""``DivsDF.sum`` keeps its dtypes, and stops asking pandas for a removed keyword. + +The four ``reindex(...).infer_objects(...).fillna(0)`` chains in ``DivsDF.sum`` +used to pass ``copy=False``. That keyword has been ignored since pandas 3.0 made +Copy-on-Write unconditional -- ``_check_copy_deprecation`` warns and nothing else +consults it -- and it is slated for removal in pandas 4, at which point the call +raises ``TypeError``. + +The removal is worse than a plain ``TypeError`` here, because every caller of +``DivsDF.sum`` wraps it in a bare ``except:`` whose fallback (``sum(divs_df.values())``) +does not work on a pydantic model. The real failure surfaces as +``AttributeError: 'DivsDF' object has no attribute 'values'``, pointing nowhere +near the cause -- so ``test_removal_of_the_keyword_is_not_masked`` pins that too. + +``infer_objects()`` itself stays. It does nothing for the reindex (added columns +are float64 NaN regardless of source dtype), but it is what keeps an object-dtype +input column from surviving ``fillna(0)`` and dragging the sum to object. +""" + +import warnings + +import pandas as pd +import pytest + +from SWEET_python.class_defs import DivsDF, LandfillWasteMassDF + + +WASTE_TYPES = ["food", "green", "wood", "paper_cardboard", "plastic"] +YEARS = [2000, 2001, 2002] + + +def _frame(value, columns=WASTE_TYPES): + return pd.DataFrame({c: [float(value)] * len(YEARS) for c in columns}, index=YEARS) + + +def _divs(**overrides): + frames = { + "compost": _frame(1), + "anaerobic": _frame(2), + "combustion": _frame(3), + "recycling": _frame(4), + } + frames.update(overrides) + return DivsDF(**frames) + + +class TestNoDeprecatedKeyword: + def test_sum_emits_no_pandas_deprecation(self): + """The 96 Pandas4Warnings this suite used to raise, pinned at zero.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _divs().sum() + + pandas_deprecations = [ + w for w in caught if isinstance(w.message, pd.errors.PandasChangeWarning) + ] + assert pandas_deprecations == [], [str(w.message) for w in pandas_deprecations] + + def test_removal_of_the_keyword_is_not_masked(self, monkeypatch): + """With the keyword gone, sum() must still work rather than fail obliquely. + + Simulates pandas 4 by making ``infer_objects`` reject every argument. + Before the fix this raised ``TypeError`` inside ``sum()``, which the + caller's bare ``except:`` turned into an unrelated ``AttributeError``. + """ + original = pd.DataFrame.infer_objects + + def no_keywords_accepted(self, *args, **kwargs): + if args or kwargs: + raise TypeError( + "infer_objects() takes 1 positional argument but 2 were given" + ) + return original(self) + + monkeypatch.setattr(pd.DataFrame, "infer_objects", no_keywords_accepted) + + divs = _divs() + assert divs.sum().loc[2000, "food"] == pytest.approx(10.0) + + # And through the real caller, whose bare `except:` did the masking. + landfilled = LandfillWasteMassDF.create( + waste_generated_df=_frame(100), + divs_df=divs, + fraction_of_waste=0.5, + waste_types=WASTE_TYPES, + ) + assert landfilled.df.loc[2000, "food"] == pytest.approx(45.0) + + +class TestDtypesAndValues: + def test_float_inputs_sum_to_float(self): + summed = _divs().sum() + + assert list(summed.columns) == WASTE_TYPES + assert (summed.dtypes == "float64").all(), summed.dtypes.to_dict() + assert (summed == 10.0).all().all() + + def test_reindex_fills_missing_columns_with_zero_as_float(self): + """A stream missing a waste type contributes 0 to it, not NaN or object.""" + summed = _divs(compost=_frame(1, columns=["food", "green"])).sum() + + assert list(summed.columns) == sorted(WASTE_TYPES) + assert (summed.dtypes == "float64").all(), summed.dtypes.to_dict() + assert summed.loc[2000, "food"] == pytest.approx(10.0) + # compost sat out: 2 + 3 + 4, no NaN leaking through. + assert summed.loc[2000, "plastic"] == pytest.approx(9.0) + + def test_object_dtype_input_is_inferred_back_to_float(self): + """Why infer_objects() stays: a boxed column must not survive fillna(0).""" + boxed = _frame(1) + boxed["food"] = pd.Series([1.0] * len(YEARS), index=YEARS, dtype=object) + assert boxed.dtypes["food"] == object + + summed = _divs(compost=boxed).sum() + + assert summed.dtypes["food"] == "float64", summed.dtypes.to_dict() + assert summed.loc[2000, "food"] == pytest.approx(10.0) From 89a5e21db71bf42fb4f6eb6115d7f55fecc3e723 Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Tue, 8 Sep 2026 15:04:42 -0700 Subject: [PATCH 2/2] Changelog for #62 Co-Authored-By: Claude Opus 5 --- changelog/2026-09.md | 29 +++++++++++++++++++++++++++++ changelog/README.md | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/changelog/2026-09.md b/changelog/2026-09.md index 47e50c8..51bfc17 100644 --- a/changelog/2026-09.md +++ b/changelog/2026-09.md @@ -7,6 +7,34 @@ monthly one by ~3.7% on identical inputs. A site with gas capture now emits about 3.5% less; sites without capture are unaffected. Separately, `gas_capture_efficiency` is now bounded to [0, 1] on both advanced-DST paths, where an out-of-range value used to produce negative emissions silently. The +city DST's diversion sum also stops passing pandas a keyword pandas is removing, +which would have broken the model path on pandas 4 with an error pointing +nowhere near the cause. + +## Changed + +- **`DivsDF.sum` no longer passes `copy=False` to `infer_objects`.** The four + `reindex(...).infer_objects(...).fillna(0)` chains that combine the compost / + anaerobic / combustion / recycling streams into the diverted-mass frame passed + a keyword pandas has ignored since 3.0 made Copy-on-Write unconditional — + `infer_objects` hands it to `_check_copy_deprecation`, which warns and returns, + and nothing else in the method reads it. It emitted 96 `Pandas4Warning`s per + test run, and pandas 4 removes it, at which point the call raises `TypeError`. + That break would have been worse than a `TypeError`: all three callers wrap + `divs_df.sum()` in a bare `except:` whose fallback, `sum(divs_df.values())`, + does not work on a pydantic model, so the failure would have surfaced as + `AttributeError: 'DivsDF' object has no attribute 'values'`. A new test + simulates the removal and pins both the sum and that caller. + `infer_objects()` itself is kept — not for the reindex, which never produces an + object column (added columns arrive as float64 NaN whatever the source dtypes), + but to stop an object-dtype *input* column from surviving `fillna(0)` and + dragging the summed frame with it. **No output change**, verified end to end: + five city-DST scenarios, including the food-waste-prevention path that drives + this code, produce a byte-identical dump of 73,760 numbers across 165 model + outputs before and after. + ([#62](https://github.com/RMI/SWEET_python/pull/62)) + + city-level advanced DST also gains a `food_waste_prevention` input, so callers stop having to pre-shrink the waste stream themselves, and can now ask it where the tonnage went — the diversion split per waste type, and the total each @@ -87,6 +115,7 @@ landfill received — instead of re-deriving it. parity with the single-site one (`advanced_dst_city.py`, `tests/test_adst_city_incineration.py`). ([#53](https://github.com/RMI/SWEET_python/pull/53)) + ## Fixed - **A caller can now say a landfill never closes.** `MODEL_YEAR_MAX` was the diff --git a/changelog/README.md b/changelog/README.md index 89ce0d9..ed87b39 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -15,7 +15,7 @@ The project does not publish semantic version tags, so releases are tracked by Newest first: -- [2026-09](2026-09.md) — Annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart +- [2026-09](2026-09.md) — City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart - [2026-08](2026-08.md) — Single-site adst gains optional `depth` (deep-dump MCF bump) and `k_override` (caller-supplied decomposition rate) inputs, restoring the last two site-DST levers; `/sdst` flaring efficiency reaches the model again after a variable-name bug silently forced flare destruction to 0.98; annual model applies cover oxidation by emission year not deposit year, fixing biocover having no effect on closed landfills (WasteMAP #719); `City.sdst_v1_5` custom-site path holds the scenario equal to the baseline before the implementation year even when composition changes (was back-dating the new composition onto pre-implementation deposits) (model-output change); MCF consolidated into a new `SWEET_python.mcf` module and both dump types moved to the IPCC uncategorised-SWDS 0.6 (open dumps up from 0.4, controlled dumps down from 0.7), with a supplied depth now selecting the deep/shallow category (model-output change) - [2026-07](2026-07.md) — All ten waste types eligible for combustion (metal/glass/other added); methane-only model treats combustion as landfill diversion (model-output change) - [2026-06](2026-06.md) — New single-site and city-level ADST modeling modules, min-cost max-flow rewrite of the city DST diversion allocator, physical-k fix for cold/dry sites, no more spurious negative food-waste mass