fix(metrics): count confidence 1.0 in ECE and stop thresholding 1-D integer labels

deeptab/metrics/classification.py

@@ -64,6 +64,21 @@
 from .base import DeepTabMetric
 
 
+def _labels_from_predictions(y_pred: np.ndarray) -> np.ndarray:
+    """Resolve predictions to class labels.
+
+    2-D arrays are class probabilities (argmax). 1-D integer-valued arrays are
+    already labels; only 1-D fractional arrays are positive-class scores and
+    get thresholded at 0.5.
+    """
+    if y_pred.ndim == 2:
+        return np.argmax(y_pred, axis=1)
+    flat = y_pred.ravel()
+    if np.issubdtype(flat.dtype, np.integer) or np.array_equal(flat, np.round(flat)):
+        return flat.astype(int)
+    return (flat >= 0.5).astype(int)
+
+
 class Accuracy(DeepTabMetric):
     """Classification accuracy -- delegates to :func:`sklearn.metrics.accuracy_score`.
 
@@ -76,7 +91,7 @@ class Accuracy(DeepTabMetric):
     def __call__(self, y_true: np.ndarray, y_pred: np.ndarray) -> float:
         y_true = np.asarray(y_true).ravel()
         y_pred = np.asarray(y_pred)
-        labels = np.argmax(y_pred, axis=1) if y_pred.ndim == 2 else (y_pred.ravel() >= 0.5).astype(int)
+        labels = _labels_from_predictions(y_pred)
         return float(_accuracy(y_true, labels))
 
 
@@ -101,7 +116,7 @@ def __init__(self, average: str = "binary") -> None:
     def __call__(self, y_true: np.ndarray, y_pred: np.ndarray) -> float:
         y_true = np.asarray(y_true).ravel()
         y_pred = np.asarray(y_pred)
-        labels = np.argmax(y_pred, axis=1) if y_pred.ndim == 2 else (y_pred.ravel() >= 0.5).astype(int)
+        labels = _labels_from_predictions(y_pred)
         return float(_f1(y_true, labels, average=self.average, zero_division=0))  # type: ignore[arg-type]
 
     def __repr__(self) -> str:
@@ -217,8 +232,12 @@ def __call__(self, y_true: np.ndarray, y_pred: np.ndarray) -> float:
         bin_edges = np.linspace(0.0, 1.0, self.n_bins + 1)
         ece = 0.0
         n = len(y_true)
-        for lo, hi in itertools.pairwise(bin_edges):
-            mask = (confidence >= lo) & (confidence < hi)
+        for i, (lo, hi) in enumerate(itertools.pairwise(bin_edges)):
+            # The last bin is right-inclusive so confidence == 1.0 is counted.
+            if i == self.n_bins - 1:
+                mask = (confidence >= lo) & (confidence <= hi)
+            else:
+                mask = (confidence >= lo) & (confidence < hi)
             if mask.sum() == 0:
                 continue
             acc = correct[mask].mean()

tests/test_metrics.py

@@ -319,6 +319,26 @@ def test_ece_zero_for_perfect_calibration(self):
         proba = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 0.0], [0.0, 1.0]])
         assert ExpectedCalibrationError()(y_true, proba) == pytest.approx(0.0)
 
+    def test_ece_counts_full_confidence(self):
+        """Regression test: confidence == 1.0 must fall into the last bin.
+
+        A model that is always wrong at 100% confidence has the worst
+        possible calibration (ECE = 1), not perfect calibration (ECE = 0).
+        """
+        y_true = np.array([0, 0, 0, 0])
+        proba = np.array([[0.0, 1.0]] * 4)
+        assert ExpectedCalibrationError()(y_true, proba) == pytest.approx(1.0)
+
+    def test_accuracy_1d_multiclass_labels(self):
+        """Regression test: 1-D integer labels must not be thresholded at 0.5."""
+        y = np.array([0, 1, 2, 2])
+        assert Accuracy()(y, np.array([0, 1, 2, 2])) == pytest.approx(1.0)
+
+    def test_f1_1d_multiclass_labels(self):
+        """Regression test: 1-D integer labels must not be thresholded at 0.5."""
+        y = np.array([0, 1, 2, 2])
+        assert F1Score(average="macro")(y, np.array([0, 1, 2, 2])) == pytest.approx(1.0)
+
     def test_f1_perfect(self):
         y = np.array([0, 1, 0, 1])
         proba = np.array([[0.9, 0.1], [0.1, 0.9], [0.9, 0.1], [0.1, 0.9]])