feat: Add average parameter to Accuracy for per-label multilabel output

ignite/metrics/accuracy.py

@@ -1,4 +1,5 @@
 from collections.abc import Callable, Sequence, Iterable
+from typing import cast
 
 import torch
 
@@ -167,6 +168,11 @@ class Accuracy(_BaseClassification):
             :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the
             form expected by the metric. This can be useful if, for example, you have a multi-output model and
             you want to compute the metric with respect to one of the outputs.
+        average: if None (default), computes subset accuracy for multilabel data
+            (a sample is correct only if all labels match exactly), returning a single float.
+            If False, computes per-label accuracy for multilabel data, returning a tensor
+            of shape ``(num_labels,)`` where each element is the fraction of samples correctly
+            predicted for that label. Only applicable when ``is_multilabel=True``.
         is_multilabel: flag to use in multilabel case. By default, False.
         device: specifies which device updates are accumulated on. Setting the metric's
             device to be the same as your ``update`` arguments ensures the ``update`` method is non-blocking. By
@@ -246,6 +252,33 @@ class Accuracy(_BaseClassification):
 
             0.2
 
+        Multilabel case, per-label accuracy
+
+        .. testcode:: 5
+
+            metric = Accuracy(is_multilabel=True, average=False)
+            metric.attach(default_evaluator, "label_accuracy")
+            y_true = torch.tensor([
+                [0, 0, 1, 0, 1],
+                [1, 0, 1, 0, 0],
+                [0, 0, 0, 0, 1],
+                [1, 0, 0, 0, 1],
+                [0, 1, 1, 0, 1],
+            ])
+            y_pred = torch.tensor([
+                [1, 1, 0, 0, 0],
+                [1, 0, 1, 0, 0],
+                [1, 0, 0, 0, 0],
+                [1, 0, 1, 1, 1],
+                [1, 1, 0, 0, 1],
+            ])
+            state = default_evaluator.run([[y_pred, y_true]])
+            print(state.metrics["label_accuracy"])
+
+        .. testoutput:: 5
+
+            tensor([0.4000, 0.8000, 0.4000, 0.8000, 0.6000])
+
         In binary and multilabel cases, the elements of `y` and `y_pred` should have 0 or 1 values. Thresholding of
         predictions can be done as below:
 
@@ -269,24 +302,33 @@ def thresholded_output_transform(output):
 
     .. versionchanged:: 0.5.1
         ``skip_unrolling`` argument is added.
+
+    .. versionchanged:: 0.6.0
+        ``average`` argument is added.
     """
 
     _state_dict_all_req_keys = ("_num_correct", "_num_examples")
 
     def __init__(
         self,
         output_transform: Callable = lambda x: x,
+        average: bool | None = None,
         is_multilabel: bool = False,
         device: str | torch.device = torch.device("cpu"),
         skip_unrolling: bool = False,
     ):
+        if average is not None and average is not False:
+            raise ValueError("Argument average should be None or False.")
+        if average is False and not is_multilabel:
+            raise ValueError("Argument average=False is only applicable with is_multilabel=True.")
+        self._average = average
         super().__init__(
             output_transform=output_transform, is_multilabel=is_multilabel, device=device, skip_unrolling=skip_unrolling
         )
 
     @reinit__is_reduced
     def reset(self) -> None:
-        self._num_correct = torch.tensor(0, device=self._device)
+        self._num_correct: int | torch.Tensor = 0
         self._num_examples = 0
         super().reset()
 
@@ -307,6 +349,12 @@ def update(self, output: Sequence[torch.Tensor]) -> None:
             last_dim = y_pred.ndimension()
             y_pred = torch.transpose(y_pred, 1, last_dim - 1).reshape(-1, num_classes)
             y = torch.transpose(y, 1, last_dim - 1).reshape(-1, num_classes)
+            if self._average is False:
+                correct = (y == y_pred.type_as(y)).float()
+                self._num_correct += correct.sum(dim=0).to(self._device)
+                self._num_examples += correct.shape[0]
+                return
+            # Default: subset accuracy — entire label vector must match exactly
             correct = torch.all(y == y_pred.type_as(y), dim=-1)
         else:
             raise ValueError(f"Unexpected type: {self._type}")
@@ -315,7 +363,9 @@ def update(self, output: Sequence[torch.Tensor]) -> None:
         self._num_examples += correct.shape[0]
 
     @sync_all_reduce("_num_examples", "_num_correct")
-    def compute(self) -> float:
+    def compute(self) -> float | torch.Tensor:
         if self._num_examples == 0:
             raise NotComputableError("Accuracy must have at least one example before it can be computed.")
-        return self._num_correct.item() / self._num_examples
+        if self._average is False:
+            return self._num_correct / self._num_examples
+        return cast(torch.Tensor, self._num_correct).item() / self._num_examples

tests/ignite/metrics/test_accuracy.py

@@ -154,6 +154,18 @@ def test_multilabel_wrong_inputs():
         acc.update((torch.randint(0, 2, size=(10, 1)), torch.randint(0, 2, size=(10, 1)).long()))
 
 
+def test_average_parameter():
+    with pytest.raises(ValueError, match=r"Argument average should be None or False"):
+        Accuracy(average="macro")
+
+    with pytest.raises(ValueError, match=r"Argument average=False is only applicable with is_multilabel=True"):
+        Accuracy(average=False)
+
+    # both should be fine
+    Accuracy(average=None)
+    Accuracy(average=False, is_multilabel=True)
+
+
 @pytest.mark.parametrize("n_times", range(3))
 def test_multilabel_input(n_times, available_device, test_data_multilabel):
     acc = Accuracy(is_multilabel=True, device=available_device)
@@ -176,6 +188,110 @@ def test_multilabel_input(n_times, available_device, test_data_multilabel):
     assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
 
 
+def test_multilabel_input_average_false():
+    y_true = torch.tensor(
+        [
+            [0, 0, 1, 0, 1],
+            [1, 0, 1, 0, 0],
+            [0, 0, 0, 0, 1],
+            [1, 0, 0, 0, 1],
+            [0, 1, 1, 0, 1],
+        ]
+    )
+    y_pred = torch.tensor(
+        [
+            [1, 1, 0, 0, 0],
+            [1, 0, 1, 0, 0],
+            [1, 0, 0, 0, 0],
+            [1, 0, 1, 1, 1],
+            [1, 1, 0, 0, 1],
+        ]
+    )
+
+    acc = Accuracy(is_multilabel=True, average=False)
+    acc.update((y_pred, y_true))
+    result = acc.compute()
+
+    assert isinstance(result, torch.Tensor)
+    assert result.shape == (5,)
+
+    expected = (y_true == y_pred).float().mean(dim=0)
+    assert torch.allclose(result, expected)
+    assert torch.allclose(result, torch.tensor([0.4, 0.8, 0.4, 0.8, 0.6]))
+
+
+def test_multilabel_input_average_false_all_correct():
+    y_true = torch.randint(0, 2, size=(8, 3))
+    y_pred = y_true.clone()
+
+    acc = Accuracy(is_multilabel=True, average=False)
+    acc.update((y_pred, y_true))
+    assert torch.allclose(acc.compute(), torch.ones(3))
+
+
+def test_multilabel_input_average_false_all_wrong():
+    y_true = torch.tensor([[0, 1, 0]] * 8)
+    y_pred = torch.tensor([[1, 0, 1]] * 8)
+
+    acc = Accuracy(is_multilabel=True, average=False)
+    acc.update((y_pred, y_true))
+    assert torch.allclose(acc.compute(), torch.zeros(3))
+
+
+def test_multilabel_input_average_false_label_with_no_positives():
+    # label at index 1 has no positive samples in y_true
+    y_true = torch.tensor(
+        [
+            [1, 0, 0],
+            [0, 0, 1],
+            [1, 0, 1],
+            [0, 0, 0],
+        ]
+    )
+    y_pred = torch.tensor(
+        [
+            [1, 0, 0],
+            [0, 1, 1],
+            [1, 0, 0],
+            [0, 0, 1],
+        ]
+    )
+
+    acc = Accuracy(is_multilabel=True, average=False)
+    acc.update((y_pred, y_true))
+    result = acc.compute()
+
+    expected = (y_true == y_pred).float().mean(dim=0)
+    assert torch.allclose(result, expected)
+    assert torch.isfinite(result).all()
+
+
+@pytest.mark.parametrize("n_times", range(3))
+def test_multilabel_input_average_false_batched(n_times, available_device, test_data_multilabel):
+    acc = Accuracy(is_multilabel=True, average=False, device=available_device)
+    assert acc._device == torch.device(available_device)
+
+    y_pred, y, batch_size = test_data_multilabel
+    if batch_size > 1:
+        n_iters = y.shape[0] // batch_size + 1
+        for i in range(n_iters):
+            idx = i * batch_size
+            acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
+    else:
+        acc.update((y_pred, y))
+
+    np_y_pred = to_numpy_multilabel(y_pred)
+    np_y = to_numpy_multilabel(y)
+    num_labels = np_y.shape[1]
+
+    result = acc.compute()
+    assert isinstance(result, torch.Tensor)
+    assert result.shape == (num_labels,)
+
+    expected = (np_y == np_y_pred).mean(axis=0)
+    assert result.cpu().numpy().tolist() == pytest.approx(expected.tolist())
+
+
 def test_incorrect_type():
     acc = Accuracy()
 
@@ -192,6 +308,39 @@ def test_incorrect_type():
         acc.update((y_pred, y))
 
 
+@pytest.mark.parametrize("n_epochs", [1, 2])
+def test_engine_integration_multilabel_average_false(n_epochs):
+    # Regression test: reset() must correctly re-initialize the per-label accumulator
+    # at the start of every epoch, not just the first one.
+    n_iters = 10
+    batch_size = 16
+    n_classes = 5
+
+    y_true = torch.randint(0, 2, size=(n_iters * batch_size, n_classes))
+    y_preds = torch.randint(0, 2, size=(n_iters * batch_size, n_classes))
+
+    def update(engine, i):
+        return (
+            y_preds[i * batch_size : (i + 1) * batch_size, :],
+            y_true[i * batch_size : (i + 1) * batch_size, :],
+        )
+
+    engine = Engine(update)
+    acc = Accuracy(is_multilabel=True, average=False)
+    acc.attach(engine, "acc")
+
+    data = list(range(n_iters))
+    engine.run(data=data, max_epochs=n_epochs)
+
+    assert "acc" in engine.state.metrics
+    res = engine.state.metrics["acc"]
+    assert isinstance(res, torch.Tensor)
+    assert res.shape == (n_classes,)
+
+    expected = (y_true == y_preds).float().mean(dim=0)
+    assert torch.allclose(res, expected)
+
+
 @pytest.mark.usefixtures("distributed")
 class TestDistributed:
     def test_multilabel_input_NHW(self):
@@ -400,19 +549,71 @@ def update(engine, i):
 
             assert pytest.approx(res) == true_res
 
-    def test_accumulator_device(self):
+    @pytest.mark.parametrize("n_epochs", [1, 2])
+    def test_integration_multilabel_average_false(self, n_epochs):
+        rank = idist.get_rank()
+        torch.manual_seed(12 + rank)
+
+        n_iters = 80
+        batch_size = 16
+        n_classes = 10
+
         metric_devices = [torch.device("cpu")]
         device = idist.device()
         if device.type != "xla":
             metric_devices.append(device)
 
         for metric_device in metric_devices:
-            acc = Accuracy(device=metric_device)
-            assert acc._device == metric_device
+            metric_device = torch.device(metric_device)
+
+            y_true = torch.randint(0, 2, size=(n_iters * batch_size, n_classes, 8, 10)).to(device)
+            y_preds = torch.randint(0, 2, size=(n_iters * batch_size, n_classes, 8, 10)).to(device)
+
+            def update(engine, i):
+                return (
+                    y_preds[i * batch_size : (i + 1) * batch_size, ...],
+                    y_true[i * batch_size : (i + 1) * batch_size, ...],
+                )
+
+            engine = Engine(update)
+
+            acc = Accuracy(is_multilabel=True, average=False, device=metric_device)
+            acc.attach(engine, "acc")
+
+            data = list(range(n_iters))
+            engine.run(data=data, max_epochs=n_epochs)
+
+            y_true = idist.all_gather(y_true)
+            y_preds = idist.all_gather(y_preds)
+
             assert acc._num_correct.device == metric_device, (
                 f"{type(acc._num_correct.device)}:{acc._num_correct.device} vs {type(metric_device)}:{metric_device}"
             )
 
+            assert "acc" in engine.state.metrics
+            res = engine.state.metrics["acc"]
+            assert isinstance(res, torch.Tensor)
+            assert res.shape == (n_classes,)
+
+            np_y_true = to_numpy_multilabel(y_true)
+            np_y_preds = to_numpy_multilabel(y_preds)
+            expected = (np_y_true == np_y_preds).mean(axis=0)
+
+            assert res.cpu().numpy().tolist() == pytest.approx(expected.tolist())
+
+    def test_accumulator_device(self):
+        metric_devices = [torch.device("cpu")]
+        device = idist.device()
+        if device.type != "xla":
+            metric_devices.append(device)
+
+        for metric_device in metric_devices:
+            acc = Accuracy(device=metric_device)
+            assert acc._device == metric_device
+            # Since the shape of the accumulated amount isn't known before the first update
+            # call, the internal variable isn't a tensor on the right device yet (it's a
+            # plain int 0 right after reset()).
+
             y_pred = torch.randint(0, 2, size=(10,), device=device, dtype=torch.long)
             y = torch.randint(0, 2, size=(10,), device=device, dtype=torch.long)
             acc.update((y_pred, y))

tests/ignite/metrics/test_running_average.py

@@ -121,7 +121,9 @@ def _(engine):
     acc_metric.load_state_dict(metric_state)
     assert acc_metric._value == saved__value
     assert acc_metric.src._num_examples == saved_src__num_examples
-    assert (acc_metric.src._num_correct == saved_src__num_correct).all()
+    # `_num_correct` may be a plain int (right after reset(), before any update()) or a tensor,
+    # depending on whether `src` was updated since its last reset — compare in a type-agnostic way.
+    assert torch.equal(torch.as_tensor(acc_metric.src._num_correct), torch.as_tensor(saved_src__num_correct))
 
     metric_state = avg_output.state_dict()
     saved__value = avg_output._value