diff --git a/deeptab/metrics/classification.py b/deeptab/metrics/classification.py index afdcbdc7..e6836f6f 100644 --- a/deeptab/metrics/classification.py +++ b/deeptab/metrics/classification.py @@ -64,6 +64,21 @@ from .base import DeepTabMetric +def _labels_from_predictions(y_pred: np.ndarray) -> np.ndarray: + """Resolve predictions to class labels. + + 2-D arrays are class probabilities (argmax). 1-D integer-valued arrays are + already labels; only 1-D fractional arrays are positive-class scores and + get thresholded at 0.5. + """ + if y_pred.ndim == 2: + return np.argmax(y_pred, axis=1) + flat = y_pred.ravel() + if np.issubdtype(flat.dtype, np.integer) or np.array_equal(flat, np.round(flat)): + return flat.astype(int) + return (flat >= 0.5).astype(int) + + class Accuracy(DeepTabMetric): """Classification accuracy -- delegates to :func:`sklearn.metrics.accuracy_score`. @@ -76,7 +91,7 @@ class Accuracy(DeepTabMetric): def __call__(self, y_true: np.ndarray, y_pred: np.ndarray) -> float: y_true = np.asarray(y_true).ravel() y_pred = np.asarray(y_pred) - labels = np.argmax(y_pred, axis=1) if y_pred.ndim == 2 else (y_pred.ravel() >= 0.5).astype(int) + labels = _labels_from_predictions(y_pred) return float(_accuracy(y_true, labels)) @@ -101,7 +116,7 @@ def __init__(self, average: str = "binary") -> None: def __call__(self, y_true: np.ndarray, y_pred: np.ndarray) -> float: y_true = np.asarray(y_true).ravel() y_pred = np.asarray(y_pred) - labels = np.argmax(y_pred, axis=1) if y_pred.ndim == 2 else (y_pred.ravel() >= 0.5).astype(int) + labels = _labels_from_predictions(y_pred) return float(_f1(y_true, labels, average=self.average, zero_division=0)) # type: ignore[arg-type] def __repr__(self) -> str: @@ -217,8 +232,12 @@ def __call__(self, y_true: np.ndarray, y_pred: np.ndarray) -> float: bin_edges = np.linspace(0.0, 1.0, self.n_bins + 1) ece = 0.0 n = len(y_true) - for lo, hi in itertools.pairwise(bin_edges): - mask = (confidence >= lo) & (confidence < hi) + for i, (lo, hi) in enumerate(itertools.pairwise(bin_edges)): + # The last bin is right-inclusive so confidence == 1.0 is counted. + if i == self.n_bins - 1: + mask = (confidence >= lo) & (confidence <= hi) + else: + mask = (confidence >= lo) & (confidence < hi) if mask.sum() == 0: continue acc = correct[mask].mean() diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 1ef1bea5..c1f09493 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -319,6 +319,26 @@ def test_ece_zero_for_perfect_calibration(self): proba = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 0.0], [0.0, 1.0]]) assert ExpectedCalibrationError()(y_true, proba) == pytest.approx(0.0) + def test_ece_counts_full_confidence(self): + """Regression test: confidence == 1.0 must fall into the last bin. + + A model that is always wrong at 100% confidence has the worst + possible calibration (ECE = 1), not perfect calibration (ECE = 0). + """ + y_true = np.array([0, 0, 0, 0]) + proba = np.array([[0.0, 1.0]] * 4) + assert ExpectedCalibrationError()(y_true, proba) == pytest.approx(1.0) + + def test_accuracy_1d_multiclass_labels(self): + """Regression test: 1-D integer labels must not be thresholded at 0.5.""" + y = np.array([0, 1, 2, 2]) + assert Accuracy()(y, np.array([0, 1, 2, 2])) == pytest.approx(1.0) + + def test_f1_1d_multiclass_labels(self): + """Regression test: 1-D integer labels must not be thresholded at 0.5.""" + y = np.array([0, 1, 2, 2]) + assert F1Score(average="macro")(y, np.array([0, 1, 2, 2])) == pytest.approx(1.0) + def test_f1_perfect(self): y = np.array([0, 1, 0, 1]) proba = np.array([[0.9, 0.1], [0.1, 0.9], [0.9, 0.1], [0.1, 0.9]])