Fix crash when scorer LLM response is blocked by content filter

pyrit/exceptions/__init__.py

@@ -12,6 +12,7 @@
     MissingPromptPlaceholderException,
     PyritException,
     RateLimitException,
+    ScorerLLMResponseBlockedException,
     get_retry_max_num_attempts,
     handle_bad_request_exception,
     pyrit_custom_result_retry,
@@ -60,6 +61,7 @@
     "RateLimitException",
     "remove_markdown_json",
     "RetryCollector",
+    "ScorerLLMResponseBlockedException",
     "set_execution_context",
     "set_retry_collector",
     "execution_context",

pyrit/exceptions/exception_classes.py

@@ -205,6 +205,21 @@ def __init__(self, *, status_code: int = 204, message: str = "No Content") -> No
         super().__init__(status_code=status_code, message=message)
 
 
+class ScorerLLMResponseBlockedException(BadRequestException):
+    """Exception raised when a scorer's own LLM response is blocked by content filtering."""
+
+    def __init__(self, *, status_code: int = 400, message: str = "Scorer LLM response blocked") -> None:
+        """
+        Initialize a scorer-response-blocked exception.
+
+        Args:
+            status_code (int): Status code for the error.
+            message (str): Error message.
+
+        """
+        super().__init__(status_code=status_code, message=message)
+
+
 class InvalidJsonException(PyritException):
     """Exception class for blocked content errors."""
 

pyrit/score/float_scale/azure_content_filter_scorer.py

@@ -340,7 +340,9 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: st
             for result in aggregated_results
         ]
 
-    def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]:
+    def _build_fallback_score(
+        self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False
+    ) -> list[Score]:
         """
         Build one neutral ``0.0`` fallback score per configured harm category.
 
@@ -357,6 +359,8 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l
         Args:
             message (Message): The message whose first piece is inspected for status.
             objective (str | None): The objective associated with this scoring call.
+            scorer_response_blocked (bool): When True, the scorer's own LLM response was
+                blocked by content filtering; reflected in the rationale.
 
         Returns:
             list[Score]: One ``0.0`` ``float_scale`` score per configured harm category,
@@ -370,7 +374,10 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l
         if piece_id is None:
             raise ValueError("Cannot create score: message piece has no id or original_prompt_id")
 
-        if first_piece.is_blocked():
+        if scorer_response_blocked:
+            status = "The scorer's own LLM response was blocked by content filtering (raise_if_scorer_blocks is False)"
+            description = "Scorer response blocked; returning 0.0 per configured category."
+        elif first_piece.is_blocked():
             status = (
                 "The request was blocked by the target (score_blocked_content is False or no partial content available)"
             )

pyrit/score/float_scale/float_scale_scorer.py

@@ -49,7 +49,9 @@ def __init__(self, *, validator: ScorerPromptValidator, chat_target: PromptTarge
         """
         super().__init__(validator=validator, chat_target=chat_target)
 
-    def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]:
+    def _build_fallback_score(
+        self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False
+    ) -> list[Score]:
         """
         Build a single-element list containing a neutral ``0.0`` score when no pieces could be scored.
 
@@ -59,6 +61,8 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l
         Args:
             message (Message): The message whose first piece is inspected for status.
             objective (str | None): The objective associated with this scoring call.
+            scorer_response_blocked (bool): When True, the scorer's own LLM response was
+                blocked by content filtering; reflected in the rationale.
 
         Returns:
             list[Score]: A single-element list containing a ``0.0`` ``float_scale`` score
@@ -72,7 +76,13 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l
         if piece_id is None:
             raise ValueError("Cannot create score: message piece has no id or original_prompt_id")
 
-        if first_piece.is_blocked():
+        if scorer_response_blocked:
+            rationale = (
+                "The scorer's own LLM response was blocked by content filtering "
+                "(raise_if_scorer_blocks is False); returning 0.0."
+            )
+            description = "Scorer response blocked; returning 0.0."
+        elif first_piece.is_blocked():
             rationale = (
                 "The request was blocked by the target "
                 "(score_blocked_content is False or no partial content available); returning 0.0."

pyrit/score/llm_scoring.py

@@ -6,7 +6,7 @@
 import uuid
 from typing import TYPE_CHECKING, Any
 
-from pyrit.exceptions import pyrit_json_retry
+from pyrit.exceptions import EmptyResponseException, ScorerLLMResponseBlockedException, pyrit_json_retry
 from pyrit.models import JSON_SCHEMA_METADATA_KEY, Message, MessagePiece
 
 if TYPE_CHECKING:
@@ -75,6 +75,11 @@ async def _run_llm_scoring_async(
             normalized and validated by the caller.
 
     Raises:
+        ScorerLLMResponseBlockedException: If the scorer's LLM response is blocked by
+            content filtering. The transport only surfaces the condition; the calling
+            ``Scorer`` owns the policy for whether to raise or return a default score.
+        EmptyResponseException: If the scorer's LLM response contains no text piece to parse
+            and was not blocked (e.g. an empty or malformed response).
         InvalidJsonException: If the response is not valid JSON, is missing required keys, or
             fails the handler's value validation.
         Exception: For other unexpected errors during scoring.
@@ -129,8 +134,29 @@ async def _run_llm_scoring_async(
     except Exception as ex:
         raise Exception(f"Error scoring prompt with original prompt ID: {scored_prompt_id}") from ex
 
-    # Get the text piece which contains the JSON response containing the score_value and rationale from the LLM
-    text_piece = next(piece for piece in response[0].message_pieces if piece.converted_value_data_type == "text")
+    # Resolve the text piece that holds the JSON response (score_value + rationale). When it's
+    # absent the scorer produced no parseable output. Guard on the actual failure (no text
+    # piece) rather than "all pieces blocked" so every no-text shape is handled instead of
+    # only the fully-blocked one. A content-filter block surfaces as a dedicated exception
+    # (the calling Scorer owns whether to raise or fall back); any other no-text response is a
+    # genuine empty/malformed error. Neither is retried by @pyrit_json_retry.
+    text_piece = next(
+        (piece for piece in response[0].message_pieces if piece.converted_value_data_type == "text"), None
+    )
+    if text_piece is None:
+        if any(piece.is_blocked() for piece in response[0].message_pieces):
+            raise ScorerLLMResponseBlockedException(
+                message=(
+                    f"The scorer's LLM response was blocked by content filtering while scoring "
+                    f"prompt ID: {scored_prompt_id}. Consider using a scorer endpoint with "
+                    f"content filtering disabled for red-teaming workflows."
+                )
+            )
+        raise EmptyResponseException(
+            message=(
+                f"The scorer's LLM response contained no text to parse while scoring prompt ID: {scored_prompt_id}."
+            )
+        )
 
     return response_handler.parse(
         response_text=text_piece.converted_value,

pyrit/score/scorer.py

@@ -14,7 +14,7 @@
     cast,
 )
 
-from pyrit.exceptions import PyritException
+from pyrit.exceptions import PyritException, ScorerLLMResponseBlockedException
 from pyrit.memory import CentralMemory, MemoryInterface
 from pyrit.models import (
     ChatMessageRole,
@@ -81,6 +81,15 @@ class Scorer(Identifiable, abc.ABC):
     #: (Chat Completions API) and ``OpenAIResponseTarget`` (Responses API).
     score_blocked_content: bool = False
 
+    #: Controls what happens when the scorer's *own* LLM response is blocked by content
+    #: filtering (common in red-teaming, since the scorer's rationale quotes harmful content).
+    #: When True (default), scoring raises ``ScorerLLMResponseBlockedException`` — a blocked
+    #: scorer endpoint is treated as a real error. When False, scoring returns the scorer's
+    #: type default instead (False for true/false scorers, 0.0 for float-scale). This is
+    #: distinct from ``score_blocked_content``, which concerns the target-under-test response.
+    #: Set this on scorer instances before use. Defaults to True.
+    raise_if_scorer_blocks: bool = True
+
     def __init_subclass__(cls, **kwargs: Any) -> None:
         """
         Enforce the keyword-only constructor contract on subclasses.
@@ -228,6 +237,8 @@ async def score_async(
             list[Score]: A list of Score objects representing the results.
 
         Raises:
+            ScorerLLMResponseBlockedException: If the scorer's own LLM response is blocked by
+                content filtering and ``raise_if_scorer_blocks`` is True (the default).
             PyritException: If scoring raises a PyRIT exception (re-raised with enhanced context).
             RuntimeError: If scoring raises a non-PyRIT exception (wrapped with scorer context).
         """
@@ -259,6 +270,25 @@ async def score_async(
                 scoring_message,
                 objective=objective,
             )
+        except ScorerLLMResponseBlockedException as e:
+            # The scorer's own LLM response was content-filtered. By default this is a real
+            # error and re-raised; when raise_if_scorer_blocks is False, fall back to the
+            # scorer's type default (False / 0.0) instead. The decision lives here in the
+            # Scorer, not the transport (see doc/code/framework.md).
+            if self.raise_if_scorer_blocks:
+                e.message = f"Error in scorer {self.__class__.__name__}: {e.message}"
+                e.args = (f"Status Code: {e.status_code}, Message: {e.message}",)
+                raise
+            logger.info(
+                "Scorer %s LLM response was blocked by content filtering; "
+                "returning default score (raise_if_scorer_blocks=False).",
+                self.__class__.__name__,
+            )
+            scores = self._build_fallback_score(
+                message=scoring_message,
+                objective=objective,
+                scorer_response_blocked=True,
+            )
         except PyritException as e:
             # Re-raise PyRIT exceptions with enhanced context while preserving type for retry decorators
             e.message = f"Error in scorer {self.__class__.__name__}: {e.message}"
@@ -401,7 +431,9 @@ def _get_supported_pieces(self, message: Message) -> list[MessagePiece]:
         ]
 
     @abstractmethod
-    def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]:
+    def _build_fallback_score(
+        self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False
+    ) -> list[Score]:
         """
         Return neutral fallback ``Score`` objects when ``_score_async`` produced no scores.
 
@@ -420,6 +452,9 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l
         Args:
             message (Message): The (possibly substituted) message that was scored.
             objective (str | None): The objective associated with this scoring call.
+            scorer_response_blocked (bool): When True, the fallback was triggered because the
+                scorer's *own* LLM response was blocked by content filtering (not the
+                target-under-test). Subclasses should reflect this in the rationale.
 
         Returns:
             list[Score]: One or more fallback scores. Must not be empty.

pyrit/score/true_false/true_false_scorer.py

@@ -160,7 +160,9 @@ async def _score_async(self, message: Message, *, objective: str | None = None)
             )
         ]
 
-    def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]:
+    def _build_fallback_score(
+        self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False
+    ) -> list[Score]:
         """
         Build a single-element list containing a ``false`` score when no pieces could be scored.
 
@@ -170,6 +172,8 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l
         Args:
             message (Message): The message whose first piece is inspected for status.
             objective (str | None): The objective associated with this scoring call.
+            scorer_response_blocked (bool): When True, the scorer's own LLM response was
+                blocked by content filtering; reflected in the rationale.
 
         Returns:
             list[Score]: A single-element list containing a ``false`` ``true_false`` score
@@ -183,7 +187,13 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l
         if piece_id is None:
             raise ValueError("Cannot create score: message piece has no id or original_prompt_id")
 
-        if first_piece.is_blocked():
+        if scorer_response_blocked:
+            rationale = (
+                "The scorer's own LLM response was blocked by content filtering "
+                "(raise_if_scorer_blocks is False); returning false."
+            )
+            description = "Scorer response blocked; returning false."
+        elif first_piece.is_blocked():
             rationale = (
                 "The request was blocked by the target "
                 "(score_blocked_content is False or no partial content available); returning false."

tests/unit/score/test_conversation_history_scorer.py

@@ -68,7 +68,9 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: st
     def validate_return_scores(self, scores: list[Score]):
         pass
 
-    def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]:
+    def _build_fallback_score(
+        self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False
+    ) -> list[Score]:
         return [
             Score(
                 score_value="false",