From 824def455c1a8314cf5d7d32d1071d54ab306f92 Mon Sep 17 00:00:00 2001 From: ChrisW09 <50968720+ChrisW09@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:23:41 +0200 Subject: [PATCH] fix(metrics): count confidence 1.0 in ECE and stop thresholding 1-D integer labels ExpectedCalibrationError used half-open bins everywhere, including the last one, so samples at confidence exactly 1.0 (saturated softmax, or hard labels passed as probabilities) fell into no bin: a model that is always wrong at 100% confidence scored a perfect ECE of 0. The final bin is now right-inclusive. Accuracy and F1Score thresholded any 1-D input at 0.5 despite documenting 1-D integer-label support, mangling multiclass labels (label 2 -> 1). Accuracy()([0,1,2,2], [0,1,2,2]) -- perfect predictions -- returned 0.5. 1-D integer-valued input is now treated as labels; only fractional 1-D scores are thresholded, so binary probability inputs keep working. Fixes #421 Co-Authored-By: Claude Fable 5 --- deeptab/metrics/classification.py | 27 +++++++++++++++++++++++---- tests/test_metrics.py | 20 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/deeptab/metrics/classification.py b/deeptab/metrics/classification.py index afdcbdc7..e6836f6f 100644 --- a/deeptab/metrics/classification.py +++ b/deeptab/metrics/classification.py @@ -64,6 +64,21 @@ from .base import DeepTabMetric +def _labels_from_predictions(y_pred: np.ndarray) -> np.ndarray: + """Resolve predictions to class labels. + + 2-D arrays are class probabilities (argmax). 1-D integer-valued arrays are + already labels; only 1-D fractional arrays are positive-class scores and + get thresholded at 0.5. + """ + if y_pred.ndim == 2: + return np.argmax(y_pred, axis=1) + flat = y_pred.ravel() + if np.issubdtype(flat.dtype, np.integer) or np.array_equal(flat, np.round(flat)): + return flat.astype(int) + return (flat >= 0.5).astype(int) + + class Accuracy(DeepTabMetric): """Classification accuracy -- delegates to :func:`sklearn.metrics.accuracy_score`. @@ -76,7 +91,7 @@ class Accuracy(DeepTabMetric): def __call__(self, y_true: np.ndarray, y_pred: np.ndarray) -> float: y_true = np.asarray(y_true).ravel() y_pred = np.asarray(y_pred) - labels = np.argmax(y_pred, axis=1) if y_pred.ndim == 2 else (y_pred.ravel() >= 0.5).astype(int) + labels = _labels_from_predictions(y_pred) return float(_accuracy(y_true, labels)) @@ -101,7 +116,7 @@ def __init__(self, average: str = "binary") -> None: def __call__(self, y_true: np.ndarray, y_pred: np.ndarray) -> float: y_true = np.asarray(y_true).ravel() y_pred = np.asarray(y_pred) - labels = np.argmax(y_pred, axis=1) if y_pred.ndim == 2 else (y_pred.ravel() >= 0.5).astype(int) + labels = _labels_from_predictions(y_pred) return float(_f1(y_true, labels, average=self.average, zero_division=0)) # type: ignore[arg-type] def __repr__(self) -> str: @@ -217,8 +232,12 @@ def __call__(self, y_true: np.ndarray, y_pred: np.ndarray) -> float: bin_edges = np.linspace(0.0, 1.0, self.n_bins + 1) ece = 0.0 n = len(y_true) - for lo, hi in itertools.pairwise(bin_edges): - mask = (confidence >= lo) & (confidence < hi) + for i, (lo, hi) in enumerate(itertools.pairwise(bin_edges)): + # The last bin is right-inclusive so confidence == 1.0 is counted. + if i == self.n_bins - 1: + mask = (confidence >= lo) & (confidence <= hi) + else: + mask = (confidence >= lo) & (confidence < hi) if mask.sum() == 0: continue acc = correct[mask].mean() diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 1ef1bea5..c1f09493 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -319,6 +319,26 @@ def test_ece_zero_for_perfect_calibration(self): proba = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 0.0], [0.0, 1.0]]) assert ExpectedCalibrationError()(y_true, proba) == pytest.approx(0.0) + def test_ece_counts_full_confidence(self): + """Regression test: confidence == 1.0 must fall into the last bin. + + A model that is always wrong at 100% confidence has the worst + possible calibration (ECE = 1), not perfect calibration (ECE = 0). + """ + y_true = np.array([0, 0, 0, 0]) + proba = np.array([[0.0, 1.0]] * 4) + assert ExpectedCalibrationError()(y_true, proba) == pytest.approx(1.0) + + def test_accuracy_1d_multiclass_labels(self): + """Regression test: 1-D integer labels must not be thresholded at 0.5.""" + y = np.array([0, 1, 2, 2]) + assert Accuracy()(y, np.array([0, 1, 2, 2])) == pytest.approx(1.0) + + def test_f1_1d_multiclass_labels(self): + """Regression test: 1-D integer labels must not be thresholded at 0.5.""" + y = np.array([0, 1, 2, 2]) + assert F1Score(average="macro")(y, np.array([0, 1, 2, 2])) == pytest.approx(1.0) + def test_f1_perfect(self): y = np.array([0, 1, 0, 1]) proba = np.array([[0.9, 0.1], [0.1, 0.9], [0.9, 0.1], [0.1, 0.9]])