FIX: warn instead of raising on reasoning-model token truncation

pyrit/prompt_target/openai/openai_chat_target.py

@@ -14,6 +14,7 @@
     JsonResponseConfig,
     Message,
     MessagePiece,
+    construct_response_from_request,
 )
 from pyrit.prompt_target.common.chat_completions_message_builder import (
     build_multimodal_chat_messages_async,
@@ -281,7 +282,7 @@ def _extract_partial_content(self, response: Any) -> str | None:
         """
         return extract_partial_content(response)
 
-    def _validate_response(self, response: Any, request: MessagePiece) -> Message | None:
+    def _validate_response(self, response: Any, request: MessagePiece) -> None:
         """
         Validate a Chat Completions API response for errors.
 
@@ -290,19 +291,39 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message |
         - Invalid finish_reason
         - At least one valid response type (text content, audio, or tool_calls)
 
+        A ``finish_reason == "length"`` (token-limit truncation) response is treated as valid, with a
+        warning, so that ``_construct_message_from_response_async`` can preserve any partial content
+        or fall back to a graceful empty response. Genuinely empty responses (no truncation) are
+        raised so the retry logic can attempt to get a complete response. Content filter responses
+        are handled separately by ``_check_content_filter``.
+
         Args:
             response: The ChatCompletion response from OpenAI SDK.
             request: The original request MessagePiece.
 
-        Returns:
-            None if valid, does not return Message for content filter (handled by _check_content_filter).
-
         Raises:
             PyritException: For unexpected response structures or finish reasons.
-            EmptyResponseException: When the API returns an empty response.
+            EmptyResponseException: When the API returns an empty response that was not caused by
+                token-limit truncation.
         """
+        # Token-limit truncation is handled before the shared validator, which would otherwise raise
+        # EmptyResponseException on a validly truncated but empty response. Reasoning models can spend
+        # the whole budget on hidden reasoning before emitting a visible answer, and a low limit may be
+        # deliberate, so warn instead of raising and let construction preserve any partial content or
+        # fall back to a graceful empty response.
+        choice = response.choices[0] if getattr(response, "choices", None) else None
+        if choice is not None and getattr(choice, "finish_reason", None) == "length":
+            logger.warning(
+                "The response was truncated because it reached the token limit (finish_reason='length'). "
+                "Reasoning models consume tokens on hidden reasoning in addition to the visible answer, so a "
+                "low max_completion_tokens can truncate or empty the response. Increase max_completion_tokens "
+                "if you expected complete content."
+            )
+            return
+
+        # Genuinely empty responses (no truncation) raise so the retry logic can attempt to get a
+        # complete response.
         validate_chat_completion_response(response=response)
-        return None
 
     def _detect_response_content(self, message: Any) -> tuple[bool, bool, bool]:
         """
@@ -364,12 +385,25 @@ async def _construct_message_from_response_async(self, response: Any, request: M
             Message: Constructed message with one or more MessagePiece entries.
 
         Raises:
-            EmptyResponseException: If the response contains no content, audio, or tool calls.
+            EmptyResponseException: If a non-truncated response contains no content, audio, or tool
+                calls. A truncated (``finish_reason == "length"``) response with no content instead
+                yields a graceful empty piece so the run continues.
         """
         audio_format = self._audio_response_config.audio_format if self._audio_response_config else "wav"
         pieces = await build_response_pieces_async(response=response, request=request, audio_format=audio_format)
 
         if not pieces:
+            # A truncated (finish_reason == "length") response may legitimately produce no content;
+            # return a graceful empty piece so the run continues. Validation already raised for
+            # genuinely empty (non-truncated) responses.
+            choice = response.choices[0] if getattr(response, "choices", None) else None
+            if choice is not None and getattr(choice, "finish_reason", None) == "length":
+                return construct_response_from_request(
+                    request=request,
+                    response_text_pieces=[""],
+                    response_type="text",
+                    error="empty",
+                )
             raise EmptyResponseException(message="Failed to extract any response content.")
 
         # Capture token usage from the API response and store in the first piece's metadata

pyrit/prompt_target/openai/openai_response_target.py

@@ -27,6 +27,7 @@
     MessagePiece,
     PromptDataType,
     PromptResponseError,
+    construct_response_from_request,
 )
 from pyrit.prompt_target.common.target_capabilities import TargetCapabilities
 from pyrit.prompt_target.common.target_configuration import TargetConfiguration
@@ -505,31 +506,49 @@ def _extract_partial_content(self, response: Any) -> str | None:
         except (AttributeError, IndexError, TypeError):
             return None
 
-    def _validate_response(self, response: Any, request: MessagePiece) -> Message | None:
+    def _validate_response(self, response: Any, request: MessagePiece) -> None:
         """
         Validate a Response API response for errors.
 
         Checks for:
         - Error responses (excluding content filtering which is checked separately)
+        - Truncation at the token limit (``max_output_tokens``), which is warned about, not raised
         - Invalid status
         - Empty output
 
+        Truncation is treated as valid, with a warning, so that
+        ``_construct_message_from_response_async`` can preserve any completed output (reasoning,
+        partial text) or fall back to a graceful empty response. Genuinely empty responses (no
+        truncation) are raised so the retry logic can attempt to get a complete response. Content
+        filter responses are handled separately by ``_check_content_filter``.
+
         Args:
             response: The Response object from the OpenAI SDK.
             request: The original request MessagePiece.
 
-        Returns:
-            None if valid, does not return Message for content filter (handled by _check_content_filter).
-
         Raises:
             PyritException: For unexpected response structures or errors.
-            EmptyResponseException: When the API returns no valid output.
+            EmptyResponseException: When the API returns no valid output (and was not truncated).
         """
         # Check for error response - error is a ResponseError object or None
         # (content_filter is handled by _check_content_filter)
         if response.error is not None and response.error.code != "content_filter":
             raise PyritException(message=f"Response error: {response.error.code} - {response.error.message}")
 
+        # Truncation: the model hit max_output_tokens. Mirroring OpenAIChatTarget's handling of
+        # finish_reason == "length", warn instead of raising so the run continues -- reasoning models
+        # can spend the whole budget on hidden reasoning before emitting a visible answer, and a low
+        # limit may be a deliberate configuration. Construction preserves any completed output
+        # (reasoning, partial text) and falls back to a graceful empty response.
+        if self._is_truncated_response(response):
+            logger.warning(
+                "The response was truncated because it reached the token limit "
+                "(status='incomplete', reason='max_output_tokens'). Reasoning models consume tokens on "
+                "hidden reasoning in addition to the visible answer, so a low max_output_tokens can "
+                "truncate or empty the response. Increase max_output_tokens if you expected complete content."
+            )
+            return
+
         # Check status - should be "completed" for successful responses
         if response.status != "completed":
             raise PyritException(message=f"Unexpected status: {response.status}")
@@ -539,30 +558,72 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message |
             logger.error("The response returned no valid output.")
             raise EmptyResponseException(message="The response returned an empty response.")
 
-        return None
+    def _is_truncated_response(self, response: Any) -> bool:
+        """
+        Return True if the response was cut off by the ``max_output_tokens`` limit.
+
+        The Responses API signals truncation via ``status == "incomplete"`` with
+        ``incomplete_details.reason == "max_output_tokens"`` (``content_filter`` is handled
+        separately by ``_check_content_filter``).
+
+        Args:
+            response: A Response object from the OpenAI SDK.
+
+        Returns:
+            bool: True if the response was truncated at the token limit, False otherwise.
+        """
+        if getattr(response, "status", None) != "incomplete":
+            return False
+        incomplete_details = getattr(response, "incomplete_details", None)
+        reason = getattr(incomplete_details, "reason", None) if incomplete_details else None
+        return reason == "max_output_tokens"
 
     async def _construct_message_from_response_async(self, response: Any, request: MessagePiece) -> Message:
         """
         Construct a Message from a Response API response.
 
+        For a truncated response (see ``_is_truncated_response``), empty output sections are
+        tolerated, partial tool/function calls are skipped so an incomplete call cannot re-enter the
+        agentic loop, and a graceful empty text piece is appended when no visible text was produced.
+        Reasoning and any partial text are always preserved.
+
         Args:
             response: The Response object from OpenAI SDK.
             request: The original request MessagePiece.
 
         Returns:
             Message: Constructed message with extracted content from output sections.
         """
+        truncated = self._is_truncated_response(response)
+
         # Extract and parse message pieces from validated output sections
         extracted_response_pieces: list[MessagePiece] = []
-        for section in response.output:
+        has_text = False
+        for section in getattr(response, "output", None) or []:
             piece = self._parse_response_output_section(
                 section=section,
                 message_piece=request,
                 error=None,  # error is already handled in validation
+                tolerate_empty=truncated,
             )
             if piece is None:
                 continue
+            # On truncation, keep only reasoning/text; a partial tool or function call must not
+            # re-enter the agentic loop.
+            if truncated and piece.original_value_data_type not in ("text", "reasoning"):
+                continue
             extracted_response_pieces.append(piece)
+            if piece.original_value_data_type == "text" and piece.original_value:
+                has_text = True
+
+        if truncated and not has_text:
+            empty_piece = construct_response_from_request(
+                request=request,
+                response_text_pieces=[""],
+                response_type="text",
+                error="empty",
+            ).message_pieces[0]
+            extracted_response_pieces.append(empty_piece)
 
         return Message(message_pieces=extracted_response_pieces)
 
@@ -637,7 +698,12 @@ async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Me
         return responses_to_return
 
     def _parse_response_output_section(
-        self, *, section: Any, message_piece: MessagePiece, error: PromptResponseError | None
+        self,
+        *,
+        section: Any,
+        message_piece: MessagePiece,
+        error: PromptResponseError | None,
+        tolerate_empty: bool = False,
     ) -> MessagePiece | None:
         """
         Parse model output sections, forwarding tool-calls for the agentic loop.
@@ -646,12 +712,15 @@ def _parse_response_output_section(
             section: The section object from OpenAI SDK (Pydantic model).
             message_piece: The original message piece.
             error: Any error information from OpenAI.
+            tolerate_empty: When True, empty sections return None instead of raising
+                EmptyResponseException. Used when constructing a truncated response.
 
         Returns:
             A MessagePiece for this section, or None to skip.
 
         Raises:
-            EmptyResponseException: If the section content is empty or invalid.
+            EmptyResponseException: If the section content is empty or invalid (and tolerate_empty
+                is False).
             ValueError: If the section type is unsupported.
         """
         section_type = section.type
@@ -661,6 +730,8 @@ def _parse_response_output_section(
         if section_type == MessagePieceType.MESSAGE:
             section_content = section.content
             if len(section_content) == 0:
+                if tolerate_empty:
+                    return None
                 raise EmptyResponseException(message="The chat returned an empty message section.")
             piece_value = section_content[0].text
 
@@ -715,6 +786,8 @@ def _parse_response_output_section(
                 raise ValueError(msg)
             piece_value = section.input
             if len(piece_value) == 0:
+                if tolerate_empty:
+                    return None
                 raise EmptyResponseException(message="The chat returned an empty message section.")
 
         else:
@@ -723,6 +796,8 @@ def _parse_response_output_section(
 
         # Handle empty response
         if not piece_value:
+            if tolerate_empty:
+                return None
             raise EmptyResponseException(message="The chat returned an empty response.")
 
         return MessagePiece(

pyrit/prompt_target/openai/openai_target.py

@@ -448,10 +448,8 @@ async def _handle_openai_request_async(
             if self._check_content_filter(response):
                 return self._handle_content_filter_response(response, request_piece)
 
-            # Validate response via subclass implementation
-            error_message = self._validate_response(response, request_piece)
-            if error_message:
-                return error_message
+            # Validate response via subclass implementation (raises on invalid responses)
+            self._validate_response(response, request_piece)
 
             # Construct and return Message from validated response
             return await self._construct_message_from_response_async(response, request_piece)
@@ -604,24 +602,21 @@ def _extract_partial_content(self, response: Any) -> str | None:
         """
         return None
 
-    def _validate_response(self, response: Any, request: MessagePiece) -> Message | None:
+    def _validate_response(self, response: Any, request: MessagePiece) -> None:
         """
-        Validate the response and return error Message if needed.
+        Validate the response, raising if it is invalid.
 
-        Override this method in subclasses that need custom response validation.
-        Default implementation returns None (no validation errors).
+        Override this method in subclasses that need custom response validation. Validation only
+        inspects the response; constructing the resulting Message is the responsibility of
+        ``_construct_message_from_response_async``. The default implementation is a no-op.
 
         Args:
             response: The response object from OpenAI SDK.
             request: The original request MessagePiece.
 
-        Returns:
-            Message | None: Error Message if validation fails, None otherwise.
-
         Raises:
             Various exceptions for validation failures.
         """
-        return None
 
     @abstractmethod
     def _set_openai_env_configuration_vars(self) -> None:

tests/unit/prompt_target/target/test_openai_chat_target.py

@@ -1038,11 +1038,44 @@ def test_validate_response_success_stop(target: OpenAIChatTarget, dummy_text_mes
     assert result is None
 
 
-def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece):
-    """Test _validate_response passes for valid length response."""
+def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog):
+    """Test _validate_response passes for a truncated response that still has content, and warns."""
     mock_response = create_mock_completion(content="Hello", finish_reason="length")
-    result = target._validate_response(mock_response, dummy_text_message_piece)
+    with caplog.at_level(logging.WARNING):
+        result = target._validate_response(mock_response, dummy_text_message_piece)
+    assert result is None
+    assert "finish_reason='length'" in caplog.text
+
+
+def test_validate_response_length_empty_does_not_raise(
+    target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog
+):
+    """Test _validate_response treats a truncated-but-empty response as valid (warns, does not raise)."""
+    mock_response = create_mock_completion(content="", finish_reason="length")
+    with caplog.at_level(logging.WARNING):
+        result = target._validate_response(mock_response, dummy_text_message_piece)
     assert result is None
+    assert "finish_reason='length'" in caplog.text
+
+
+async def test_construct_message_length_empty_returns_graceful_empty(
+    target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece
+):
+    """Test construction returns a graceful empty piece when a truncated response produced no content."""
+    mock_response = create_mock_completion(content="", finish_reason="length")
+    result = await target._construct_message_from_response_async(mock_response, dummy_text_message_piece)
+    assert isinstance(result, Message)
+    assert result.message_pieces[0].original_value == ""
+    assert result.message_pieces[0].response_error == "empty"
+
+
+async def test_construct_message_empty_non_truncated_raises(
+    target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece
+):
+    """Test construction raises for a genuinely empty (non-truncated) response so retries can kick in."""
+    mock_response = create_mock_completion(content="", finish_reason="stop")
+    with pytest.raises(EmptyResponseException, match="Failed to extract any response content"):
+        await target._construct_message_from_response_async(mock_response, dummy_text_message_piece)
 
 
 def test_validate_response_no_choices(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece):