Guard DETR and Grounding DINO Hungarian matchers against infinite cost matrices

src/transformers/loss/loss_for_object_detection.py

@@ -354,6 +354,10 @@ def forward(self, outputs, targets):
         # Final cost matrix
         cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost
         cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu()
+        # Guard against non-finite costs (e.g. inf/NaN from fp16 overflow under AMP), which would
+        # otherwise make scipy's linear_sum_assignment raise "ValueError: cost matrix is infeasible".
+        max_value = torch.finfo(cost_matrix.dtype).max
+        cost_matrix = torch.nan_to_num(cost_matrix, nan=max_value, posinf=max_value, neginf=max_value)
 
         sizes = [len(v["boxes"]) for v in targets]
         indices = [linear_sum_assignment(c[i]) for i, c in enumerate(cost_matrix.split(sizes, -1))]

src/transformers/loss/loss_grounding_dino.py

@@ -119,6 +119,10 @@ def forward(self, outputs, targets):
         # Final cost matrix
         cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost
         cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu()
+        # Guard against non-finite costs (e.g. inf/NaN from fp16 overflow under AMP), which would
+        # otherwise make scipy's linear_sum_assignment raise "ValueError: cost matrix is infeasible".
+        max_value = torch.finfo(cost_matrix.dtype).max
+        cost_matrix = torch.nan_to_num(cost_matrix, nan=max_value, posinf=max_value, neginf=max_value)
 
         sizes = [len(v["boxes"]) for v in targets]
         indices = [linear_sum_assignment(c[i]) for i, c in enumerate(cost_matrix.split(sizes, -1))]

tests/models/detr/test_modeling_detr.py

@@ -22,7 +22,15 @@
 from parameterized import parameterized
 
 from transformers import DetrConfig, ResNetConfig, is_torch_available, is_vision_available
-from transformers.testing_utils import Expectations, require_timm, require_torch, require_vision, slow, torch_device
+from transformers.testing_utils import (
+    Expectations,
+    require_scipy,
+    require_timm,
+    require_torch,
+    require_vision,
+    slow,
+    torch_device,
+)
 
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import (
@@ -733,3 +741,55 @@ def test_inference_panoptic_segmentation_head(self):
         self.assertEqual(predicted_first_segment["label_id"], expected_first_segment["label_id"])
         self.assertEqual(predicted_first_segment["was_fused"], expected_first_segment["was_fused"])
         self.assertAlmostEqual(predicted_first_segment["score"], expected_first_segment["score"], places=3)
+
+
+@require_torch
+@require_scipy
+class HungarianMatcherInfeasibleCostTest(unittest.TestCase):
+    def test_non_finite_costs_do_not_crash_matcher(self):
+        """
+        Regression test for the object-detection Hungarian matcher. When the cost matrix contains
+        non-finite values (e.g. inf/NaN propagated from the model outputs, as happens with fp16
+        overflow under AMP, see #47000), ``scipy.optimize.linear_sum_assignment`` used to raise
+        ``ValueError: cost matrix is infeasible``. The matcher now replaces non-finite costs with a
+        large finite value before matching.
+        """
+        from transformers.loss.loss_for_object_detection import HungarianMatcher
+
+        torch.manual_seed(0)
+        matcher = HungarianMatcher(class_cost=1, bbox_cost=5, giou_cost=2)
+
+        num_queries, num_classes, num_targets = 5, 4, 3
+        # Boxes in normalized center format (cx, cy, w, h) with strictly positive width/height.
+        targets = [
+            {
+                "class_labels": torch.arange(num_targets),
+                "boxes": torch.rand(num_targets, 4) * 0.5 + 0.25,
+            }
+        ]
+
+        # NaN in the logits poisons the classification cost; +inf on a box width poisons the L1/GIoU
+        # cost. Either leaves a non-finite cost matrix that used to make scipy raise.
+        def base_inputs():
+            return (
+                torch.randn(1, num_queries, num_classes),
+                torch.rand(1, num_queries, 4) * 0.5 + 0.25,
+            )
+
+        cases = []
+        logits, pred_boxes = base_inputs()
+        logits[0, 0, 0] = float("nan")
+        cases.append((logits, pred_boxes))
+        logits, pred_boxes = base_inputs()
+        pred_boxes[0, 0, 2] = float("inf")
+        cases.append((logits, pred_boxes))
+
+        for logits, pred_boxes in cases:
+            outputs = {"logits": logits, "pred_boxes": pred_boxes}
+
+            indices = matcher(outputs, targets)
+
+            self.assertEqual(len(indices), 1)
+            row_indices, col_indices = indices[0]
+            self.assertEqual(len(row_indices), num_targets)
+            self.assertEqual(len(col_indices), num_targets)

tests/models/grounding_dino/test_modeling_grounding_dino.py

@@ -32,6 +32,7 @@
 from transformers.testing_utils import (
     Expectations,
     is_flaky,
+    require_scipy,
     require_timm,
     require_torch,
     require_torch_accelerator,
@@ -869,3 +870,53 @@ def test_grounding_dino_loss(self):
             torch.testing.assert_close(outputs.loss_dict[key], expected_loss_dict[key], rtol=1e-5, atol=1e-3)
 
         self.assertTrue(torch.allclose(outputs.loss, expected_loss, atol=1e-3))
+
+
+@require_torch
+@require_scipy
+class GroundingDinoHungarianMatcherInfeasibleCostTest(unittest.TestCase):
+    def test_non_finite_costs_do_not_crash_matcher(self):
+        """
+        Regression test for the Grounding DINO Hungarian matcher. Its sigmoid focal classification
+        cost can overflow to inf/NaN (e.g. fp16 saturation under AMP, see #47000), which used to make
+        ``scipy.optimize.linear_sum_assignment`` raise ``ValueError: cost matrix is infeasible``. The
+        matcher now replaces non-finite costs with a large finite value before matching.
+        """
+        from transformers.loss.loss_grounding_dino import GroundingDinoHungarianMatcher
+
+        torch.manual_seed(0)
+        matcher = GroundingDinoHungarianMatcher(class_cost=1, bbox_cost=5, giou_cost=2)
+
+        num_queries, hidden_dim, num_classes, num_targets = 5, 6, 4, 3
+        targets = [
+            {
+                "class_labels": torch.arange(num_targets),
+                "boxes": torch.rand(num_targets, 4) * 0.5 + 0.25,
+            }
+        ]
+        # Per-class token maps (one map per batch element), normalized inside the matcher.
+        label_maps = (torch.rand(num_classes, hidden_dim),)
+
+        def base_inputs():
+            return (
+                torch.randn(1, num_queries, hidden_dim),
+                torch.rand(1, num_queries, 4) * 0.5 + 0.25,
+            )
+
+        cases = []
+        logits, pred_boxes = base_inputs()
+        logits[0, 0, 0] = float("nan")
+        cases.append((logits, pred_boxes))
+        logits, pred_boxes = base_inputs()
+        pred_boxes[0, 0, 2] = float("inf")
+        cases.append((logits, pred_boxes))
+
+        for logits, pred_boxes in cases:
+            outputs = {"logits": logits, "pred_boxes": pred_boxes, "label_maps": label_maps}
+
+            indices = matcher(outputs, targets)
+
+            self.assertEqual(len(indices), 1)
+            row_indices, col_indices = indices[0]
+            self.assertEqual(len(row_indices), num_targets)
+            self.assertEqual(len(col_indices), num_targets)