diff --git a/ignite/metrics/accuracy.py b/ignite/metrics/accuracy.py index 2d44a37d4afc..0ad8eb16aacb 100644 --- a/ignite/metrics/accuracy.py +++ b/ignite/metrics/accuracy.py @@ -1,4 +1,5 @@ from collections.abc import Callable, Sequence, Iterable +from typing import cast import torch @@ -167,6 +168,11 @@ class Accuracy(_BaseClassification): :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the form expected by the metric. This can be useful if, for example, you have a multi-output model and you want to compute the metric with respect to one of the outputs. + average: if None (default), computes subset accuracy for multilabel data + (a sample is correct only if all labels match exactly), returning a single float. + If False, computes per-label accuracy for multilabel data, returning a tensor + of shape ``(num_labels,)`` where each element is the fraction of samples correctly + predicted for that label. Only applicable when ``is_multilabel=True``. is_multilabel: flag to use in multilabel case. By default, False. device: specifies which device updates are accumulated on. Setting the metric's device to be the same as your ``update`` arguments ensures the ``update`` method is non-blocking. By @@ -246,6 +252,33 @@ class Accuracy(_BaseClassification): 0.2 + Multilabel case, per-label accuracy + + .. testcode:: 5 + + metric = Accuracy(is_multilabel=True, average=False) + metric.attach(default_evaluator, "label_accuracy") + y_true = torch.tensor([ + [0, 0, 1, 0, 1], + [1, 0, 1, 0, 0], + [0, 0, 0, 0, 1], + [1, 0, 0, 0, 1], + [0, 1, 1, 0, 1], + ]) + y_pred = torch.tensor([ + [1, 1, 0, 0, 0], + [1, 0, 1, 0, 0], + [1, 0, 0, 0, 0], + [1, 0, 1, 1, 1], + [1, 1, 0, 0, 1], + ]) + state = default_evaluator.run([[y_pred, y_true]]) + print(state.metrics["label_accuracy"]) + + .. testoutput:: 5 + + tensor([0.4000, 0.8000, 0.4000, 0.8000, 0.6000]) + In binary and multilabel cases, the elements of `y` and `y_pred` should have 0 or 1 values. Thresholding of predictions can be done as below: @@ -269,6 +302,9 @@ def thresholded_output_transform(output): .. versionchanged:: 0.5.1 ``skip_unrolling`` argument is added. + + .. versionchanged:: 0.6.0 + ``average`` argument is added. """ _state_dict_all_req_keys = ("_num_correct", "_num_examples") @@ -276,17 +312,23 @@ def thresholded_output_transform(output): def __init__( self, output_transform: Callable = lambda x: x, + average: bool | None = None, is_multilabel: bool = False, device: str | torch.device = torch.device("cpu"), skip_unrolling: bool = False, ): + if average is not None and average is not False: + raise ValueError("Argument average should be None or False.") + if average is False and not is_multilabel: + raise ValueError("Argument average=False is only applicable with is_multilabel=True.") + self._average = average super().__init__( output_transform=output_transform, is_multilabel=is_multilabel, device=device, skip_unrolling=skip_unrolling ) @reinit__is_reduced def reset(self) -> None: - self._num_correct = torch.tensor(0, device=self._device) + self._num_correct: int | torch.Tensor = 0 self._num_examples = 0 super().reset() @@ -307,6 +349,12 @@ def update(self, output: Sequence[torch.Tensor]) -> None: last_dim = y_pred.ndimension() y_pred = torch.transpose(y_pred, 1, last_dim - 1).reshape(-1, num_classes) y = torch.transpose(y, 1, last_dim - 1).reshape(-1, num_classes) + if self._average is False: + correct = (y == y_pred.type_as(y)).float() + self._num_correct += correct.sum(dim=0).to(self._device) + self._num_examples += correct.shape[0] + return + # Default: subset accuracy — entire label vector must match exactly correct = torch.all(y == y_pred.type_as(y), dim=-1) else: raise ValueError(f"Unexpected type: {self._type}") @@ -315,7 +363,9 @@ def update(self, output: Sequence[torch.Tensor]) -> None: self._num_examples += correct.shape[0] @sync_all_reduce("_num_examples", "_num_correct") - def compute(self) -> float: + def compute(self) -> float | torch.Tensor: if self._num_examples == 0: raise NotComputableError("Accuracy must have at least one example before it can be computed.") - return self._num_correct.item() / self._num_examples + if self._average is False: + return self._num_correct / self._num_examples + return cast(torch.Tensor, self._num_correct).item() / self._num_examples diff --git a/tests/ignite/metrics/test_accuracy.py b/tests/ignite/metrics/test_accuracy.py index cc87b72be36f..add2ba1ad040 100644 --- a/tests/ignite/metrics/test_accuracy.py +++ b/tests/ignite/metrics/test_accuracy.py @@ -154,6 +154,18 @@ def test_multilabel_wrong_inputs(): acc.update((torch.randint(0, 2, size=(10, 1)), torch.randint(0, 2, size=(10, 1)).long())) +def test_average_parameter(): + with pytest.raises(ValueError, match=r"Argument average should be None or False"): + Accuracy(average="macro") + + with pytest.raises(ValueError, match=r"Argument average=False is only applicable with is_multilabel=True"): + Accuracy(average=False) + + # both should be fine + Accuracy(average=None) + Accuracy(average=False, is_multilabel=True) + + @pytest.mark.parametrize("n_times", range(3)) def test_multilabel_input(n_times, available_device, test_data_multilabel): acc = Accuracy(is_multilabel=True, device=available_device) @@ -176,6 +188,110 @@ def test_multilabel_input(n_times, available_device, test_data_multilabel): assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute()) +def test_multilabel_input_average_false(): + y_true = torch.tensor( + [ + [0, 0, 1, 0, 1], + [1, 0, 1, 0, 0], + [0, 0, 0, 0, 1], + [1, 0, 0, 0, 1], + [0, 1, 1, 0, 1], + ] + ) + y_pred = torch.tensor( + [ + [1, 1, 0, 0, 0], + [1, 0, 1, 0, 0], + [1, 0, 0, 0, 0], + [1, 0, 1, 1, 1], + [1, 1, 0, 0, 1], + ] + ) + + acc = Accuracy(is_multilabel=True, average=False) + acc.update((y_pred, y_true)) + result = acc.compute() + + assert isinstance(result, torch.Tensor) + assert result.shape == (5,) + + expected = (y_true == y_pred).float().mean(dim=0) + assert torch.allclose(result, expected) + assert torch.allclose(result, torch.tensor([0.4, 0.8, 0.4, 0.8, 0.6])) + + +def test_multilabel_input_average_false_all_correct(): + y_true = torch.randint(0, 2, size=(8, 3)) + y_pred = y_true.clone() + + acc = Accuracy(is_multilabel=True, average=False) + acc.update((y_pred, y_true)) + assert torch.allclose(acc.compute(), torch.ones(3)) + + +def test_multilabel_input_average_false_all_wrong(): + y_true = torch.tensor([[0, 1, 0]] * 8) + y_pred = torch.tensor([[1, 0, 1]] * 8) + + acc = Accuracy(is_multilabel=True, average=False) + acc.update((y_pred, y_true)) + assert torch.allclose(acc.compute(), torch.zeros(3)) + + +def test_multilabel_input_average_false_label_with_no_positives(): + # label at index 1 has no positive samples in y_true + y_true = torch.tensor( + [ + [1, 0, 0], + [0, 0, 1], + [1, 0, 1], + [0, 0, 0], + ] + ) + y_pred = torch.tensor( + [ + [1, 0, 0], + [0, 1, 1], + [1, 0, 0], + [0, 0, 1], + ] + ) + + acc = Accuracy(is_multilabel=True, average=False) + acc.update((y_pred, y_true)) + result = acc.compute() + + expected = (y_true == y_pred).float().mean(dim=0) + assert torch.allclose(result, expected) + assert torch.isfinite(result).all() + + +@pytest.mark.parametrize("n_times", range(3)) +def test_multilabel_input_average_false_batched(n_times, available_device, test_data_multilabel): + acc = Accuracy(is_multilabel=True, average=False, device=available_device) + assert acc._device == torch.device(available_device) + + y_pred, y, batch_size = test_data_multilabel + if batch_size > 1: + n_iters = y.shape[0] // batch_size + 1 + for i in range(n_iters): + idx = i * batch_size + acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) + else: + acc.update((y_pred, y)) + + np_y_pred = to_numpy_multilabel(y_pred) + np_y = to_numpy_multilabel(y) + num_labels = np_y.shape[1] + + result = acc.compute() + assert isinstance(result, torch.Tensor) + assert result.shape == (num_labels,) + + expected = (np_y == np_y_pred).mean(axis=0) + assert result.cpu().numpy().tolist() == pytest.approx(expected.tolist()) + + def test_incorrect_type(): acc = Accuracy() @@ -192,6 +308,39 @@ def test_incorrect_type(): acc.update((y_pred, y)) +@pytest.mark.parametrize("n_epochs", [1, 2]) +def test_engine_integration_multilabel_average_false(n_epochs): + # Regression test: reset() must correctly re-initialize the per-label accumulator + # at the start of every epoch, not just the first one. + n_iters = 10 + batch_size = 16 + n_classes = 5 + + y_true = torch.randint(0, 2, size=(n_iters * batch_size, n_classes)) + y_preds = torch.randint(0, 2, size=(n_iters * batch_size, n_classes)) + + def update(engine, i): + return ( + y_preds[i * batch_size : (i + 1) * batch_size, :], + y_true[i * batch_size : (i + 1) * batch_size, :], + ) + + engine = Engine(update) + acc = Accuracy(is_multilabel=True, average=False) + acc.attach(engine, "acc") + + data = list(range(n_iters)) + engine.run(data=data, max_epochs=n_epochs) + + assert "acc" in engine.state.metrics + res = engine.state.metrics["acc"] + assert isinstance(res, torch.Tensor) + assert res.shape == (n_classes,) + + expected = (y_true == y_preds).float().mean(dim=0) + assert torch.allclose(res, expected) + + @pytest.mark.usefixtures("distributed") class TestDistributed: def test_multilabel_input_NHW(self): @@ -400,19 +549,71 @@ def update(engine, i): assert pytest.approx(res) == true_res - def test_accumulator_device(self): + @pytest.mark.parametrize("n_epochs", [1, 2]) + def test_integration_multilabel_average_false(self, n_epochs): + rank = idist.get_rank() + torch.manual_seed(12 + rank) + + n_iters = 80 + batch_size = 16 + n_classes = 10 + metric_devices = [torch.device("cpu")] device = idist.device() if device.type != "xla": metric_devices.append(device) for metric_device in metric_devices: - acc = Accuracy(device=metric_device) - assert acc._device == metric_device + metric_device = torch.device(metric_device) + + y_true = torch.randint(0, 2, size=(n_iters * batch_size, n_classes, 8, 10)).to(device) + y_preds = torch.randint(0, 2, size=(n_iters * batch_size, n_classes, 8, 10)).to(device) + + def update(engine, i): + return ( + y_preds[i * batch_size : (i + 1) * batch_size, ...], + y_true[i * batch_size : (i + 1) * batch_size, ...], + ) + + engine = Engine(update) + + acc = Accuracy(is_multilabel=True, average=False, device=metric_device) + acc.attach(engine, "acc") + + data = list(range(n_iters)) + engine.run(data=data, max_epochs=n_epochs) + + y_true = idist.all_gather(y_true) + y_preds = idist.all_gather(y_preds) + assert acc._num_correct.device == metric_device, ( f"{type(acc._num_correct.device)}:{acc._num_correct.device} vs {type(metric_device)}:{metric_device}" ) + assert "acc" in engine.state.metrics + res = engine.state.metrics["acc"] + assert isinstance(res, torch.Tensor) + assert res.shape == (n_classes,) + + np_y_true = to_numpy_multilabel(y_true) + np_y_preds = to_numpy_multilabel(y_preds) + expected = (np_y_true == np_y_preds).mean(axis=0) + + assert res.cpu().numpy().tolist() == pytest.approx(expected.tolist()) + + def test_accumulator_device(self): + metric_devices = [torch.device("cpu")] + device = idist.device() + if device.type != "xla": + metric_devices.append(device) + + for metric_device in metric_devices: + acc = Accuracy(device=metric_device) + assert acc._device == metric_device + # Since the shape of the accumulated amount isn't known before the first update + # call, the internal variable isn't a tensor on the right device yet (it's a + # plain int 0 right after reset()). + y_pred = torch.randint(0, 2, size=(10,), device=device, dtype=torch.long) y = torch.randint(0, 2, size=(10,), device=device, dtype=torch.long) acc.update((y_pred, y)) diff --git a/tests/ignite/metrics/test_running_average.py b/tests/ignite/metrics/test_running_average.py index 523b11b3dd86..c1930e91f2f7 100644 --- a/tests/ignite/metrics/test_running_average.py +++ b/tests/ignite/metrics/test_running_average.py @@ -121,7 +121,9 @@ def _(engine): acc_metric.load_state_dict(metric_state) assert acc_metric._value == saved__value assert acc_metric.src._num_examples == saved_src__num_examples - assert (acc_metric.src._num_correct == saved_src__num_correct).all() + # `_num_correct` may be a plain int (right after reset(), before any update()) or a tensor, + # depending on whether `src` was updated since its last reset — compare in a type-agnostic way. + assert torch.equal(torch.as_tensor(acc_metric.src._num_correct), torch.as_tensor(saved_src__num_correct)) metric_state = avg_output.state_dict() saved__value = avg_output._value