feat(early_stopping): add get_default_score_fn and get_default_event_filter helpers (#3411)

ignite/handlers/early_stopping.py

@@ -67,6 +67,7 @@ def score_function(engine):
         Renamed ``min_delta_mode``  to ``threshold_mode``.
         Renamed ``min_delta`` to ``threshold``.
         Renamed ``cumulative_delta`` to ``cumulative``.
+        Added :meth:`get_default_score_fn` and :meth:`get_default_event_filter` static helpers.
     .. versionchanged:: 0.5.4
         Added `mode` parameter to support minimization in addition to maximization.
         Added `min_delta_mode` parameter to support both absolute and relative improvements.
@@ -291,3 +292,90 @@ def load_state_dict(self, state_dict: Mapping) -> None:
         self.counter = state_dict["counter"]
         self.best_score = state_dict["best_score"]
         self.threshold_mode = state_dict.get("threshold_mode", self.threshold_mode)
+
+    @staticmethod
+    def get_default_score_fn(metric_name: str, score_sign: float = 1.0) -> Callable:
+        """Helper method to build a score function from an engine metric name.
+
+        The returned callable reads ``engine.state.metrics[metric_name]`` and multiplies it
+        by ``score_sign``. Use ``score_sign=-1.0`` for error-like metrics (smaller is better)
+        when the handler is configured with ``mode="max"`` so that decreases in the metric
+        register as score improvements.
+
+        Args:
+            metric_name: name of the metric in ``engine.state.metrics``.
+            score_sign: ``1.0`` (default) or ``-1.0``. For error-like metrics where smaller
+                is better, use ``-1.0``.
+
+        Returns:
+            A callable taking an :class:`~ignite.engine.engine.Engine` and returning ``float``.
+
+        Examples:
+            .. code-block:: python
+
+                from ignite.handlers import EarlyStopping
+
+                # Validation accuracy: larger is better, default mode="max"
+                score_fn = EarlyStopping.get_default_score_fn("accuracy")
+                handler = EarlyStopping(patience=5, score_function=score_fn, trainer=trainer)
+
+                # Validation loss: smaller is better, flip the sign so larger is better
+                neg_loss_fn = EarlyStopping.get_default_score_fn("loss", -1.0)
+                handler = EarlyStopping(patience=5, score_function=neg_loss_fn, trainer=trainer)
+
+        .. versionadded:: 0.6.0
+        """
+        if score_sign not in (1.0, -1.0):
+            raise ValueError("Argument score_sign should be 1.0 or -1.0")
+
+        def wrapper(engine: Engine) -> float:
+            return score_sign * engine.state.metrics[metric_name]
+
+        return wrapper
+
+    def get_default_event_filter(self, after: int) -> Callable[[Engine, int], bool]:
+        """Build an event filter that delays early-stopping checks until the trainer
+        has completed at least ``after`` epochs.
+
+        This implements a warmup window for early stopping without coupling the warmup
+        logic to the handler itself, so it composes with any event the handler is
+        attached to (epoch, iteration, custom). The filter consults the trainer's
+        epoch counter (``self.trainer.state.epoch``) rather than the host engine's
+        event count, so the warmup is well-defined even when the handler is attached
+        to an evaluator that re-runs from scratch each epoch.
+
+        Args:
+            after: minimum number of trainer epochs that must have completed before
+                the early-stopping handler is allowed to run. Must be a non-negative
+                integer.
+
+        Returns:
+            A callable matching the ``event_filter`` signature ``(engine, event) -> bool``.
+
+        Examples:
+            .. code-block:: python
+
+                from ignite.engine import Events
+                from ignite.handlers import EarlyStopping
+
+                handler = EarlyStopping(patience=5, score_function=score_fn, trainer=trainer)
+
+                # Skip the first 3 trainer epochs, then enforce early stopping.
+                evaluator.add_event_handler(
+                    Events.COMPLETED(event_filter=handler.get_default_event_filter(after=3)),
+                    handler,
+                )
+
+        .. versionadded:: 0.6.0
+        """
+        if not isinstance(after, int) or after < 0:
+            raise ValueError("Argument after should be a non-negative integer.")
+
+        trainer = self.trainer
+
+        def event_filter(engine: Engine, event: int) -> bool:
+            # The host engine's `event` counter resets per `run()` (e.g. evaluators
+            # called repeatedly), so look at the trainer's epoch counter instead.
+            return trainer.state.epoch > after
+
+        return event_filter

tests/ignite/handlers/test_early_stopping.py

@@ -655,3 +655,139 @@ def score_function(engine):
     with pytest.warns(DeprecationWarning):
         h.cumulative_delta = True
         assert h.cumulative is True
+
+
+def test_get_default_score_fn_positive_sign():
+    score_fn = EarlyStopping.get_default_score_fn("accuracy")
+
+    engine = Engine(do_nothing_update_fn)
+    engine.state.metrics["accuracy"] = 0.9
+
+    assert score_fn(engine) == pytest.approx(0.9)
+
+
+def test_get_default_score_fn_negative_sign():
+    score_fn = EarlyStopping.get_default_score_fn("loss", -1.0)
+
+    engine = Engine(do_nothing_update_fn)
+    engine.state.metrics["loss"] = 0.25
+
+    assert score_fn(engine) == pytest.approx(-0.25)
+
+
+def test_get_default_score_fn_invalid_sign():
+    with pytest.raises(ValueError, match=r"Argument score_sign should be 1.0 or -1.0"):
+        EarlyStopping.get_default_score_fn("acc", score_sign=2.0)
+
+    with pytest.raises(ValueError, match=r"Argument score_sign should be 1.0 or -1.0"):
+        EarlyStopping.get_default_score_fn("acc", score_sign=0.0)
+
+
+def test_get_default_score_fn_with_handler_min_mode():
+    # With mode="min" the score_function should still return the raw metric (sign=1).
+    score_fn = EarlyStopping.get_default_score_fn("loss")
+
+    trainer = Engine(do_nothing_update_fn)
+    losses = iter([1.0, 1.2, 1.3])
+    evaluator = Engine(do_nothing_update_fn)
+
+    @evaluator.on(Events.STARTED)
+    def _set_metric(engine):
+        engine.state.metrics["loss"] = next(losses)
+
+    handler = EarlyStopping(patience=2, score_function=score_fn, trainer=trainer, mode="min")
+
+    evaluator.run([0])
+    handler(evaluator)  # best_score=1.0
+    assert not trainer.should_terminate
+
+    evaluator.run([0])
+    handler(evaluator)  # 1.2 is no improvement (counter=1)
+    assert not trainer.should_terminate
+
+    evaluator.run([0])
+    handler(evaluator)  # 1.3 is no improvement (counter=2 -> stop)
+    assert trainer.should_terminate
+
+
+def test_get_default_event_filter_invalid_after():
+    trainer = Engine(do_nothing_update_fn)
+    handler = EarlyStopping(patience=2, score_function=lambda e: 0.0, trainer=trainer)
+
+    with pytest.raises(ValueError, match=r"Argument after should be a non-negative integer."):
+        handler.get_default_event_filter(after=-1)
+
+    with pytest.raises(ValueError, match=r"Argument after should be a non-negative integer."):
+        handler.get_default_event_filter(after=1.5)  # type: ignore[arg-type]
+
+
+def test_get_default_event_filter_skips_warmup():
+    trainer = Engine(do_nothing_update_fn)
+    handler = EarlyStopping(patience=2, score_function=lambda e: 0.0, trainer=trainer)
+
+    event_filter = handler.get_default_event_filter(after=3)
+
+    # The filter inspects trainer.state.epoch, so vary that to exercise the gate.
+    trainer.state.epoch = 1
+    assert not event_filter(trainer, 1)
+    trainer.state.epoch = 3
+    assert not event_filter(trainer, 1)
+    trainer.state.epoch = 4
+    assert event_filter(trainer, 1)
+    trainer.state.epoch = 10
+    assert event_filter(trainer, 1)
+
+
+def test_get_default_event_filter_zero_after():
+    # after=0 means no warmup; trainer.state.epoch starts at 1 once a run begins.
+    trainer = Engine(do_nothing_update_fn)
+    handler = EarlyStopping(patience=2, score_function=lambda e: 0.0, trainer=trainer)
+
+    event_filter = handler.get_default_event_filter(after=0)
+
+    trainer.state.epoch = 1
+    assert event_filter(trainer, 1)
+    trainer.state.epoch = 5
+    assert event_filter(trainer, 1)
+
+
+def test_get_default_event_filter_with_engine_warmup():
+    # Integration test: handler is attached with the warmup filter and skips the first
+    # N trainer epochs before enforcing patience.
+    scores = iter([0.5, 0.4, 0.39, 0.38, 0.37, 0.36, 0.35])
+
+    def score_function(engine):
+        return next(scores)
+
+    trainer = Engine(do_nothing_update_fn)
+    evaluator = Engine(do_nothing_update_fn)
+
+    handler = EarlyStopping(patience=2, score_function=score_function, trainer=trainer)
+    evaluator.add_event_handler(
+        Events.COMPLETED(event_filter=handler.get_default_event_filter(after=3)),
+        handler,
+    )
+
+    @trainer.on(Events.EPOCH_COMPLETED)
+    def _eval(engine):
+        evaluator.run([0])
+
+    trainer.run([0], max_epochs=10)
+
+    # Trainer epochs 1, 2, 3: filter rejects (no scores consumed).
+    # Epoch 4: score=0.5 -> best_score=0.5, counter=0
+    # Epoch 5: score=0.4 (no improvement, mode=max) counter=1
+    # Epoch 6: score=0.39 (no improvement) counter=2 -> trainer.terminate
+    assert trainer.state.epoch == 6
+
+
+def test_get_default_event_filter_attach_signature():
+    # Sanity: the filter has the right (engine, event) signature accepted by Events(...).
+    trainer = Engine(do_nothing_update_fn)
+    evaluator = Engine(do_nothing_update_fn)
+    handler = EarlyStopping(patience=2, score_function=lambda e: 0.0, trainer=trainer)
+
+    filtered_event = Events.COMPLETED(event_filter=handler.get_default_event_filter(after=2))
+    evaluator.add_event_handler(filtered_event, handler)
+
+    assert evaluator.has_event_handler(handler)