From 16881a964297f77292f81c0a3923e3be28e4ae4e Mon Sep 17 00:00:00 2001 From: MutugiD Date: Sat, 4 Jul 2026 03:13:19 +0300 Subject: [PATCH] Guard DETR and Grounding DINO Hungarian matchers against infinite cost matrices Under AMP/fp16, focal-style class costs can overflow to inf/NaN, which makes scipy's linear_sum_assignment raise "ValueError: cost matrix is infeasible". Replace non-finite costs with torch.finfo(dtype).max before matching, mirroring the RT-DETR fix in #47016. Adds regression tests for both matchers. Related to #47000. --- .../loss/loss_for_object_detection.py | 4 ++ src/transformers/loss/loss_grounding_dino.py | 4 ++ tests/models/detr/test_modeling_detr.py | 62 ++++++++++++++++++- .../test_modeling_grounding_dino.py | 51 +++++++++++++++ 4 files changed, 120 insertions(+), 1 deletion(-) diff --git a/src/transformers/loss/loss_for_object_detection.py b/src/transformers/loss/loss_for_object_detection.py index 52b43f779f35..055f815b5852 100644 --- a/src/transformers/loss/loss_for_object_detection.py +++ b/src/transformers/loss/loss_for_object_detection.py @@ -354,6 +354,10 @@ def forward(self, outputs, targets): # Final cost matrix cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu() + # Guard against non-finite costs (e.g. inf/NaN from fp16 overflow under AMP), which would + # otherwise make scipy's linear_sum_assignment raise "ValueError: cost matrix is infeasible". + max_value = torch.finfo(cost_matrix.dtype).max + cost_matrix = torch.nan_to_num(cost_matrix, nan=max_value, posinf=max_value, neginf=max_value) sizes = [len(v["boxes"]) for v in targets] indices = [linear_sum_assignment(c[i]) for i, c in enumerate(cost_matrix.split(sizes, -1))] diff --git a/src/transformers/loss/loss_grounding_dino.py b/src/transformers/loss/loss_grounding_dino.py index 06ba2b77a607..bb6f91f8fe74 100644 --- a/src/transformers/loss/loss_grounding_dino.py +++ b/src/transformers/loss/loss_grounding_dino.py @@ -119,6 +119,10 @@ def forward(self, outputs, targets): # Final cost matrix cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu() + # Guard against non-finite costs (e.g. inf/NaN from fp16 overflow under AMP), which would + # otherwise make scipy's linear_sum_assignment raise "ValueError: cost matrix is infeasible". + max_value = torch.finfo(cost_matrix.dtype).max + cost_matrix = torch.nan_to_num(cost_matrix, nan=max_value, posinf=max_value, neginf=max_value) sizes = [len(v["boxes"]) for v in targets] indices = [linear_sum_assignment(c[i]) for i, c in enumerate(cost_matrix.split(sizes, -1))] diff --git a/tests/models/detr/test_modeling_detr.py b/tests/models/detr/test_modeling_detr.py index 2943ef755e34..e8d56ce53566 100644 --- a/tests/models/detr/test_modeling_detr.py +++ b/tests/models/detr/test_modeling_detr.py @@ -22,7 +22,15 @@ from parameterized import parameterized from transformers import DetrConfig, ResNetConfig, is_torch_available, is_vision_available -from transformers.testing_utils import Expectations, require_timm, require_torch, require_vision, slow, torch_device +from transformers.testing_utils import ( + Expectations, + require_scipy, + require_timm, + require_torch, + require_vision, + slow, + torch_device, +) from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( @@ -733,3 +741,55 @@ def test_inference_panoptic_segmentation_head(self): self.assertEqual(predicted_first_segment["label_id"], expected_first_segment["label_id"]) self.assertEqual(predicted_first_segment["was_fused"], expected_first_segment["was_fused"]) self.assertAlmostEqual(predicted_first_segment["score"], expected_first_segment["score"], places=3) + + +@require_torch +@require_scipy +class HungarianMatcherInfeasibleCostTest(unittest.TestCase): + def test_non_finite_costs_do_not_crash_matcher(self): + """ + Regression test for the object-detection Hungarian matcher. When the cost matrix contains + non-finite values (e.g. inf/NaN propagated from the model outputs, as happens with fp16 + overflow under AMP, see #47000), ``scipy.optimize.linear_sum_assignment`` used to raise + ``ValueError: cost matrix is infeasible``. The matcher now replaces non-finite costs with a + large finite value before matching. + """ + from transformers.loss.loss_for_object_detection import HungarianMatcher + + torch.manual_seed(0) + matcher = HungarianMatcher(class_cost=1, bbox_cost=5, giou_cost=2) + + num_queries, num_classes, num_targets = 5, 4, 3 + # Boxes in normalized center format (cx, cy, w, h) with strictly positive width/height. + targets = [ + { + "class_labels": torch.arange(num_targets), + "boxes": torch.rand(num_targets, 4) * 0.5 + 0.25, + } + ] + + # NaN in the logits poisons the classification cost; +inf on a box width poisons the L1/GIoU + # cost. Either leaves a non-finite cost matrix that used to make scipy raise. + def base_inputs(): + return ( + torch.randn(1, num_queries, num_classes), + torch.rand(1, num_queries, 4) * 0.5 + 0.25, + ) + + cases = [] + logits, pred_boxes = base_inputs() + logits[0, 0, 0] = float("nan") + cases.append((logits, pred_boxes)) + logits, pred_boxes = base_inputs() + pred_boxes[0, 0, 2] = float("inf") + cases.append((logits, pred_boxes)) + + for logits, pred_boxes in cases: + outputs = {"logits": logits, "pred_boxes": pred_boxes} + + indices = matcher(outputs, targets) + + self.assertEqual(len(indices), 1) + row_indices, col_indices = indices[0] + self.assertEqual(len(row_indices), num_targets) + self.assertEqual(len(col_indices), num_targets) diff --git a/tests/models/grounding_dino/test_modeling_grounding_dino.py b/tests/models/grounding_dino/test_modeling_grounding_dino.py index 1c53845c65b8..408077f6c581 100644 --- a/tests/models/grounding_dino/test_modeling_grounding_dino.py +++ b/tests/models/grounding_dino/test_modeling_grounding_dino.py @@ -32,6 +32,7 @@ from transformers.testing_utils import ( Expectations, is_flaky, + require_scipy, require_timm, require_torch, require_torch_accelerator, @@ -869,3 +870,53 @@ def test_grounding_dino_loss(self): torch.testing.assert_close(outputs.loss_dict[key], expected_loss_dict[key], rtol=1e-5, atol=1e-3) self.assertTrue(torch.allclose(outputs.loss, expected_loss, atol=1e-3)) + + +@require_torch +@require_scipy +class GroundingDinoHungarianMatcherInfeasibleCostTest(unittest.TestCase): + def test_non_finite_costs_do_not_crash_matcher(self): + """ + Regression test for the Grounding DINO Hungarian matcher. Its sigmoid focal classification + cost can overflow to inf/NaN (e.g. fp16 saturation under AMP, see #47000), which used to make + ``scipy.optimize.linear_sum_assignment`` raise ``ValueError: cost matrix is infeasible``. The + matcher now replaces non-finite costs with a large finite value before matching. + """ + from transformers.loss.loss_grounding_dino import GroundingDinoHungarianMatcher + + torch.manual_seed(0) + matcher = GroundingDinoHungarianMatcher(class_cost=1, bbox_cost=5, giou_cost=2) + + num_queries, hidden_dim, num_classes, num_targets = 5, 6, 4, 3 + targets = [ + { + "class_labels": torch.arange(num_targets), + "boxes": torch.rand(num_targets, 4) * 0.5 + 0.25, + } + ] + # Per-class token maps (one map per batch element), normalized inside the matcher. + label_maps = (torch.rand(num_classes, hidden_dim),) + + def base_inputs(): + return ( + torch.randn(1, num_queries, hidden_dim), + torch.rand(1, num_queries, 4) * 0.5 + 0.25, + ) + + cases = [] + logits, pred_boxes = base_inputs() + logits[0, 0, 0] = float("nan") + cases.append((logits, pred_boxes)) + logits, pred_boxes = base_inputs() + pred_boxes[0, 0, 2] = float("inf") + cases.append((logits, pred_boxes)) + + for logits, pred_boxes in cases: + outputs = {"logits": logits, "pred_boxes": pred_boxes, "label_maps": label_maps} + + indices = matcher(outputs, targets) + + self.assertEqual(len(indices), 1) + row_indices, col_indices = indices[0] + self.assertEqual(len(row_indices), num_targets) + self.assertEqual(len(col_indices), num_targets)