diff --git a/SWEET_python/class_defs.py b/SWEET_python/class_defs.py index 6f3b871..93a59d1 100644 --- a/SWEET_python/class_defs.py +++ b/SWEET_python/class_defs.py @@ -396,20 +396,23 @@ def sum(self) -> pd.DataFrame: .union(self.recycling.columns) ) - # Reindex and fill missing values, then infer object types + # Reindex each stream to the union of columns, then fill the gaps with 0. + # + # infer_objects() is not here for the reindex: columns that reindex adds + # always arrive as float64 NaN, whatever the source dtypes. It is here for + # an input frame that already carries its numbers in an object column + # (a Series built from None, a value that arrived boxed), where a bare + # fillna(0) would leave the column object-dtyped and the summed frame + # would inherit that. It is a no-op on the float64 frames this repo + # builds, and cheap insurance on the ones callers hand in. + # + # No copy= keyword: it has been ignored since pandas 3.0 made + # Copy-on-Write unconditional, and passing it is deprecated for removal. divs_list = [ - self.compost.reindex(columns=all_columns) - .infer_objects(copy=False) - .fillna(0), - self.anaerobic.reindex(columns=all_columns) - .infer_objects(copy=False) - .fillna(0), - self.combustion.reindex(columns=all_columns) - .infer_objects(copy=False) - .fillna(0), - self.recycling.reindex(columns=all_columns) - .infer_objects(copy=False) - .fillna(0), + self.compost.reindex(columns=all_columns).infer_objects().fillna(0), + self.anaerobic.reindex(columns=all_columns).infer_objects().fillna(0), + self.combustion.reindex(columns=all_columns).infer_objects().fillna(0), + self.recycling.reindex(columns=all_columns).infer_objects().fillna(0), ] return sum(divs_list) diff --git a/changelog/2026-09.md b/changelog/2026-09.md index 47e50c8..51bfc17 100644 --- a/changelog/2026-09.md +++ b/changelog/2026-09.md @@ -7,6 +7,34 @@ monthly one by ~3.7% on identical inputs. A site with gas capture now emits about 3.5% less; sites without capture are unaffected. Separately, `gas_capture_efficiency` is now bounded to [0, 1] on both advanced-DST paths, where an out-of-range value used to produce negative emissions silently. The +city DST's diversion sum also stops passing pandas a keyword pandas is removing, +which would have broken the model path on pandas 4 with an error pointing +nowhere near the cause. + +## Changed + +- **`DivsDF.sum` no longer passes `copy=False` to `infer_objects`.** The four + `reindex(...).infer_objects(...).fillna(0)` chains that combine the compost / + anaerobic / combustion / recycling streams into the diverted-mass frame passed + a keyword pandas has ignored since 3.0 made Copy-on-Write unconditional — + `infer_objects` hands it to `_check_copy_deprecation`, which warns and returns, + and nothing else in the method reads it. It emitted 96 `Pandas4Warning`s per + test run, and pandas 4 removes it, at which point the call raises `TypeError`. + That break would have been worse than a `TypeError`: all three callers wrap + `divs_df.sum()` in a bare `except:` whose fallback, `sum(divs_df.values())`, + does not work on a pydantic model, so the failure would have surfaced as + `AttributeError: 'DivsDF' object has no attribute 'values'`. A new test + simulates the removal and pins both the sum and that caller. + `infer_objects()` itself is kept — not for the reindex, which never produces an + object column (added columns arrive as float64 NaN whatever the source dtypes), + but to stop an object-dtype *input* column from surviving `fillna(0)` and + dragging the summed frame with it. **No output change**, verified end to end: + five city-DST scenarios, including the food-waste-prevention path that drives + this code, produce a byte-identical dump of 73,760 numbers across 165 model + outputs before and after. + ([#62](https://github.com/RMI/SWEET_python/pull/62)) + + city-level advanced DST also gains a `food_waste_prevention` input, so callers stop having to pre-shrink the waste stream themselves, and can now ask it where the tonnage went — the diversion split per waste type, and the total each @@ -87,6 +115,7 @@ landfill received — instead of re-deriving it. parity with the single-site one (`advanced_dst_city.py`, `tests/test_adst_city_incineration.py`). ([#53](https://github.com/RMI/SWEET_python/pull/53)) + ## Fixed - **A caller can now say a landfill never closes.** `MODEL_YEAR_MAX` was the diff --git a/changelog/README.md b/changelog/README.md index 89ce0d9..ed87b39 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -15,7 +15,7 @@ The project does not publish semantic version tags, so releases are tracked by Newest first: -- [2026-09](2026-09.md) — Annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart +- [2026-09](2026-09.md) — City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart - [2026-08](2026-08.md) — Single-site adst gains optional `depth` (deep-dump MCF bump) and `k_override` (caller-supplied decomposition rate) inputs, restoring the last two site-DST levers; `/sdst` flaring efficiency reaches the model again after a variable-name bug silently forced flare destruction to 0.98; annual model applies cover oxidation by emission year not deposit year, fixing biocover having no effect on closed landfills (WasteMAP #719); `City.sdst_v1_5` custom-site path holds the scenario equal to the baseline before the implementation year even when composition changes (was back-dating the new composition onto pre-implementation deposits) (model-output change); MCF consolidated into a new `SWEET_python.mcf` module and both dump types moved to the IPCC uncategorised-SWDS 0.6 (open dumps up from 0.4, controlled dumps down from 0.7), with a supplied depth now selecting the deep/shallow category (model-output change) - [2026-07](2026-07.md) — All ten waste types eligible for combustion (metal/glass/other added); methane-only model treats combustion as landfill diversion (model-output change) - [2026-06](2026-06.md) — New single-site and city-level ADST modeling modules, min-cost max-flow rewrite of the city DST diversion allocator, physical-k fix for cold/dry sites, no more spurious negative food-waste mass diff --git a/tests/test_divs_sum_dtypes.py b/tests/test_divs_sum_dtypes.py new file mode 100644 index 0000000..8942494 --- /dev/null +++ b/tests/test_divs_sum_dtypes.py @@ -0,0 +1,117 @@ +"""``DivsDF.sum`` keeps its dtypes, and stops asking pandas for a removed keyword. + +The four ``reindex(...).infer_objects(...).fillna(0)`` chains in ``DivsDF.sum`` +used to pass ``copy=False``. That keyword has been ignored since pandas 3.0 made +Copy-on-Write unconditional -- ``_check_copy_deprecation`` warns and nothing else +consults it -- and it is slated for removal in pandas 4, at which point the call +raises ``TypeError``. + +The removal is worse than a plain ``TypeError`` here, because every caller of +``DivsDF.sum`` wraps it in a bare ``except:`` whose fallback (``sum(divs_df.values())``) +does not work on a pydantic model. The real failure surfaces as +``AttributeError: 'DivsDF' object has no attribute 'values'``, pointing nowhere +near the cause -- so ``test_removal_of_the_keyword_is_not_masked`` pins that too. + +``infer_objects()`` itself stays. It does nothing for the reindex (added columns +are float64 NaN regardless of source dtype), but it is what keeps an object-dtype +input column from surviving ``fillna(0)`` and dragging the sum to object. +""" + +import warnings + +import pandas as pd +import pytest + +from SWEET_python.class_defs import DivsDF, LandfillWasteMassDF + + +WASTE_TYPES = ["food", "green", "wood", "paper_cardboard", "plastic"] +YEARS = [2000, 2001, 2002] + + +def _frame(value, columns=WASTE_TYPES): + return pd.DataFrame({c: [float(value)] * len(YEARS) for c in columns}, index=YEARS) + + +def _divs(**overrides): + frames = { + "compost": _frame(1), + "anaerobic": _frame(2), + "combustion": _frame(3), + "recycling": _frame(4), + } + frames.update(overrides) + return DivsDF(**frames) + + +class TestNoDeprecatedKeyword: + def test_sum_emits_no_pandas_deprecation(self): + """The 96 Pandas4Warnings this suite used to raise, pinned at zero.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _divs().sum() + + pandas_deprecations = [ + w for w in caught if isinstance(w.message, pd.errors.PandasChangeWarning) + ] + assert pandas_deprecations == [], [str(w.message) for w in pandas_deprecations] + + def test_removal_of_the_keyword_is_not_masked(self, monkeypatch): + """With the keyword gone, sum() must still work rather than fail obliquely. + + Simulates pandas 4 by making ``infer_objects`` reject every argument. + Before the fix this raised ``TypeError`` inside ``sum()``, which the + caller's bare ``except:`` turned into an unrelated ``AttributeError``. + """ + original = pd.DataFrame.infer_objects + + def no_keywords_accepted(self, *args, **kwargs): + if args or kwargs: + raise TypeError( + "infer_objects() takes 1 positional argument but 2 were given" + ) + return original(self) + + monkeypatch.setattr(pd.DataFrame, "infer_objects", no_keywords_accepted) + + divs = _divs() + assert divs.sum().loc[2000, "food"] == pytest.approx(10.0) + + # And through the real caller, whose bare `except:` did the masking. + landfilled = LandfillWasteMassDF.create( + waste_generated_df=_frame(100), + divs_df=divs, + fraction_of_waste=0.5, + waste_types=WASTE_TYPES, + ) + assert landfilled.df.loc[2000, "food"] == pytest.approx(45.0) + + +class TestDtypesAndValues: + def test_float_inputs_sum_to_float(self): + summed = _divs().sum() + + assert list(summed.columns) == WASTE_TYPES + assert (summed.dtypes == "float64").all(), summed.dtypes.to_dict() + assert (summed == 10.0).all().all() + + def test_reindex_fills_missing_columns_with_zero_as_float(self): + """A stream missing a waste type contributes 0 to it, not NaN or object.""" + summed = _divs(compost=_frame(1, columns=["food", "green"])).sum() + + assert list(summed.columns) == sorted(WASTE_TYPES) + assert (summed.dtypes == "float64").all(), summed.dtypes.to_dict() + assert summed.loc[2000, "food"] == pytest.approx(10.0) + # compost sat out: 2 + 3 + 4, no NaN leaking through. + assert summed.loc[2000, "plastic"] == pytest.approx(9.0) + + def test_object_dtype_input_is_inferred_back_to_float(self): + """Why infer_objects() stays: a boxed column must not survive fillna(0).""" + boxed = _frame(1) + boxed["food"] = pd.Series([1.0] * len(YEARS), index=YEARS, dtype=object) + assert boxed.dtypes["food"] == object + + summed = _divs(compost=boxed).sum() + + assert summed.dtypes["food"] == "float64", summed.dtypes.to_dict() + assert summed.loc[2000, "food"] == pytest.approx(10.0)