From 9be3375e14638e0c31f8c91b13205d1d381f66fb Mon Sep 17 00:00:00 2001 From: MukundaKatta Date: Sat, 25 Apr 2026 11:15:06 -0700 Subject: [PATCH] feat(early_stopping): add get_default_score_fn and get_default_event_filter helpers Adds two helper methods to EarlyStopping mirroring patterns in Checkpoint and the maintainer's preferred design from the issue thread: - EarlyStopping.get_default_score_fn(metric_name, score_sign=1.0): a static helper that builds a score_function reading engine.state.metrics[metric_name] and applying score_sign (use -1.0 for error-like metrics). - EarlyStopping.get_default_event_filter(after): an instance method that returns a (engine, event) -> bool filter consulting the trainer's epoch counter, so callers can implement a warmup window via Events.COMPLETED( event_filter=handler.get_default_event_filter(after=N)) without coupling the warmup logic to the handler. Uses the trainer epoch (not the host engine's event count) so the warmup is well-defined when the handler is attached to an evaluator that re-runs from scratch each epoch. Both helpers are additive with backwards-compatible defaults and do not change any existing public behavior. Closes part of #3411 --- ignite/handlers/early_stopping.py | 88 ++++++++++++ tests/ignite/handlers/test_early_stopping.py | 136 +++++++++++++++++++ 2 files changed, 224 insertions(+) diff --git a/ignite/handlers/early_stopping.py b/ignite/handlers/early_stopping.py index b969709f4a23..aed4525ad52e 100644 --- a/ignite/handlers/early_stopping.py +++ b/ignite/handlers/early_stopping.py @@ -67,6 +67,7 @@ def score_function(engine): Renamed ``min_delta_mode`` to ``threshold_mode``. Renamed ``min_delta`` to ``threshold``. Renamed ``cumulative_delta`` to ``cumulative``. + Added :meth:`get_default_score_fn` and :meth:`get_default_event_filter` static helpers. .. versionchanged:: 0.5.4 Added `mode` parameter to support minimization in addition to maximization. Added `min_delta_mode` parameter to support both absolute and relative improvements. @@ -291,3 +292,90 @@ def load_state_dict(self, state_dict: Mapping) -> None: self.counter = state_dict["counter"] self.best_score = state_dict["best_score"] self.threshold_mode = state_dict.get("threshold_mode", self.threshold_mode) + + @staticmethod + def get_default_score_fn(metric_name: str, score_sign: float = 1.0) -> Callable: + """Helper method to build a score function from an engine metric name. + + The returned callable reads ``engine.state.metrics[metric_name]`` and multiplies it + by ``score_sign``. Use ``score_sign=-1.0`` for error-like metrics (smaller is better) + when the handler is configured with ``mode="max"`` so that decreases in the metric + register as score improvements. + + Args: + metric_name: name of the metric in ``engine.state.metrics``. + score_sign: ``1.0`` (default) or ``-1.0``. For error-like metrics where smaller + is better, use ``-1.0``. + + Returns: + A callable taking an :class:`~ignite.engine.engine.Engine` and returning ``float``. + + Examples: + .. code-block:: python + + from ignite.handlers import EarlyStopping + + # Validation accuracy: larger is better, default mode="max" + score_fn = EarlyStopping.get_default_score_fn("accuracy") + handler = EarlyStopping(patience=5, score_function=score_fn, trainer=trainer) + + # Validation loss: smaller is better, flip the sign so larger is better + neg_loss_fn = EarlyStopping.get_default_score_fn("loss", -1.0) + handler = EarlyStopping(patience=5, score_function=neg_loss_fn, trainer=trainer) + + .. versionadded:: 0.6.0 + """ + if score_sign not in (1.0, -1.0): + raise ValueError("Argument score_sign should be 1.0 or -1.0") + + def wrapper(engine: Engine) -> float: + return score_sign * engine.state.metrics[metric_name] + + return wrapper + + def get_default_event_filter(self, after: int) -> Callable[[Engine, int], bool]: + """Build an event filter that delays early-stopping checks until the trainer + has completed at least ``after`` epochs. + + This implements a warmup window for early stopping without coupling the warmup + logic to the handler itself, so it composes with any event the handler is + attached to (epoch, iteration, custom). The filter consults the trainer's + epoch counter (``self.trainer.state.epoch``) rather than the host engine's + event count, so the warmup is well-defined even when the handler is attached + to an evaluator that re-runs from scratch each epoch. + + Args: + after: minimum number of trainer epochs that must have completed before + the early-stopping handler is allowed to run. Must be a non-negative + integer. + + Returns: + A callable matching the ``event_filter`` signature ``(engine, event) -> bool``. + + Examples: + .. code-block:: python + + from ignite.engine import Events + from ignite.handlers import EarlyStopping + + handler = EarlyStopping(patience=5, score_function=score_fn, trainer=trainer) + + # Skip the first 3 trainer epochs, then enforce early stopping. + evaluator.add_event_handler( + Events.COMPLETED(event_filter=handler.get_default_event_filter(after=3)), + handler, + ) + + .. versionadded:: 0.6.0 + """ + if not isinstance(after, int) or after < 0: + raise ValueError("Argument after should be a non-negative integer.") + + trainer = self.trainer + + def event_filter(engine: Engine, event: int) -> bool: + # The host engine's `event` counter resets per `run()` (e.g. evaluators + # called repeatedly), so look at the trainer's epoch counter instead. + return trainer.state.epoch > after + + return event_filter diff --git a/tests/ignite/handlers/test_early_stopping.py b/tests/ignite/handlers/test_early_stopping.py index 3b46c60c2648..fae1d420702c 100644 --- a/tests/ignite/handlers/test_early_stopping.py +++ b/tests/ignite/handlers/test_early_stopping.py @@ -655,3 +655,139 @@ def score_function(engine): with pytest.warns(DeprecationWarning): h.cumulative_delta = True assert h.cumulative is True + + +def test_get_default_score_fn_positive_sign(): + score_fn = EarlyStopping.get_default_score_fn("accuracy") + + engine = Engine(do_nothing_update_fn) + engine.state.metrics["accuracy"] = 0.9 + + assert score_fn(engine) == pytest.approx(0.9) + + +def test_get_default_score_fn_negative_sign(): + score_fn = EarlyStopping.get_default_score_fn("loss", -1.0) + + engine = Engine(do_nothing_update_fn) + engine.state.metrics["loss"] = 0.25 + + assert score_fn(engine) == pytest.approx(-0.25) + + +def test_get_default_score_fn_invalid_sign(): + with pytest.raises(ValueError, match=r"Argument score_sign should be 1.0 or -1.0"): + EarlyStopping.get_default_score_fn("acc", score_sign=2.0) + + with pytest.raises(ValueError, match=r"Argument score_sign should be 1.0 or -1.0"): + EarlyStopping.get_default_score_fn("acc", score_sign=0.0) + + +def test_get_default_score_fn_with_handler_min_mode(): + # With mode="min" the score_function should still return the raw metric (sign=1). + score_fn = EarlyStopping.get_default_score_fn("loss") + + trainer = Engine(do_nothing_update_fn) + losses = iter([1.0, 1.2, 1.3]) + evaluator = Engine(do_nothing_update_fn) + + @evaluator.on(Events.STARTED) + def _set_metric(engine): + engine.state.metrics["loss"] = next(losses) + + handler = EarlyStopping(patience=2, score_function=score_fn, trainer=trainer, mode="min") + + evaluator.run([0]) + handler(evaluator) # best_score=1.0 + assert not trainer.should_terminate + + evaluator.run([0]) + handler(evaluator) # 1.2 is no improvement (counter=1) + assert not trainer.should_terminate + + evaluator.run([0]) + handler(evaluator) # 1.3 is no improvement (counter=2 -> stop) + assert trainer.should_terminate + + +def test_get_default_event_filter_invalid_after(): + trainer = Engine(do_nothing_update_fn) + handler = EarlyStopping(patience=2, score_function=lambda e: 0.0, trainer=trainer) + + with pytest.raises(ValueError, match=r"Argument after should be a non-negative integer."): + handler.get_default_event_filter(after=-1) + + with pytest.raises(ValueError, match=r"Argument after should be a non-negative integer."): + handler.get_default_event_filter(after=1.5) # type: ignore[arg-type] + + +def test_get_default_event_filter_skips_warmup(): + trainer = Engine(do_nothing_update_fn) + handler = EarlyStopping(patience=2, score_function=lambda e: 0.0, trainer=trainer) + + event_filter = handler.get_default_event_filter(after=3) + + # The filter inspects trainer.state.epoch, so vary that to exercise the gate. + trainer.state.epoch = 1 + assert not event_filter(trainer, 1) + trainer.state.epoch = 3 + assert not event_filter(trainer, 1) + trainer.state.epoch = 4 + assert event_filter(trainer, 1) + trainer.state.epoch = 10 + assert event_filter(trainer, 1) + + +def test_get_default_event_filter_zero_after(): + # after=0 means no warmup; trainer.state.epoch starts at 1 once a run begins. + trainer = Engine(do_nothing_update_fn) + handler = EarlyStopping(patience=2, score_function=lambda e: 0.0, trainer=trainer) + + event_filter = handler.get_default_event_filter(after=0) + + trainer.state.epoch = 1 + assert event_filter(trainer, 1) + trainer.state.epoch = 5 + assert event_filter(trainer, 1) + + +def test_get_default_event_filter_with_engine_warmup(): + # Integration test: handler is attached with the warmup filter and skips the first + # N trainer epochs before enforcing patience. + scores = iter([0.5, 0.4, 0.39, 0.38, 0.37, 0.36, 0.35]) + + def score_function(engine): + return next(scores) + + trainer = Engine(do_nothing_update_fn) + evaluator = Engine(do_nothing_update_fn) + + handler = EarlyStopping(patience=2, score_function=score_function, trainer=trainer) + evaluator.add_event_handler( + Events.COMPLETED(event_filter=handler.get_default_event_filter(after=3)), + handler, + ) + + @trainer.on(Events.EPOCH_COMPLETED) + def _eval(engine): + evaluator.run([0]) + + trainer.run([0], max_epochs=10) + + # Trainer epochs 1, 2, 3: filter rejects (no scores consumed). + # Epoch 4: score=0.5 -> best_score=0.5, counter=0 + # Epoch 5: score=0.4 (no improvement, mode=max) counter=1 + # Epoch 6: score=0.39 (no improvement) counter=2 -> trainer.terminate + assert trainer.state.epoch == 6 + + +def test_get_default_event_filter_attach_signature(): + # Sanity: the filter has the right (engine, event) signature accepted by Events(...). + trainer = Engine(do_nothing_update_fn) + evaluator = Engine(do_nothing_update_fn) + handler = EarlyStopping(patience=2, score_function=lambda e: 0.0, trainer=trainer) + + filtered_event = Events.COMPLETED(event_filter=handler.get_default_event_filter(after=2)) + evaluator.add_event_handler(filtered_event, handler) + + assert evaluator.has_event_handler(handler)