diff --git a/pyrit/prompt_target/openai/openai_chat_target.py b/pyrit/prompt_target/openai/openai_chat_target.py index 4da1ed77c8..aba12bcb28 100644 --- a/pyrit/prompt_target/openai/openai_chat_target.py +++ b/pyrit/prompt_target/openai/openai_chat_target.py @@ -14,6 +14,7 @@ JsonResponseConfig, Message, MessagePiece, + construct_response_from_request, ) from pyrit.prompt_target.common.chat_completions_message_builder import ( build_multimodal_chat_messages_async, @@ -281,7 +282,7 @@ def _extract_partial_content(self, response: Any) -> str | None: """ return extract_partial_content(response) - def _validate_response(self, response: Any, request: MessagePiece) -> Message | None: + def _validate_response(self, response: Any, request: MessagePiece) -> None: """ Validate a Chat Completions API response for errors. @@ -290,19 +291,39 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | - Invalid finish_reason - At least one valid response type (text content, audio, or tool_calls) + A ``finish_reason == "length"`` (token-limit truncation) response is treated as valid, with a + warning, so that ``_construct_message_from_response_async`` can preserve any partial content + or fall back to a graceful empty response. Genuinely empty responses (no truncation) are + raised so the retry logic can attempt to get a complete response. Content filter responses + are handled separately by ``_check_content_filter``. + Args: response: The ChatCompletion response from OpenAI SDK. request: The original request MessagePiece. - Returns: - None if valid, does not return Message for content filter (handled by _check_content_filter). - Raises: PyritException: For unexpected response structures or finish reasons. - EmptyResponseException: When the API returns an empty response. + EmptyResponseException: When the API returns an empty response that was not caused by + token-limit truncation. """ + # Token-limit truncation is handled before the shared validator, which would otherwise raise + # EmptyResponseException on a validly truncated but empty response. Reasoning models can spend + # the whole budget on hidden reasoning before emitting a visible answer, and a low limit may be + # deliberate, so warn instead of raising and let construction preserve any partial content or + # fall back to a graceful empty response. + choice = response.choices[0] if getattr(response, "choices", None) else None + if choice is not None and getattr(choice, "finish_reason", None) == "length": + logger.warning( + "The response was truncated because it reached the token limit (finish_reason='length'). " + "Reasoning models consume tokens on hidden reasoning in addition to the visible answer, so a " + "low max_completion_tokens can truncate or empty the response. Increase max_completion_tokens " + "if you expected complete content." + ) + return + + # Genuinely empty responses (no truncation) raise so the retry logic can attempt to get a + # complete response. validate_chat_completion_response(response=response) - return None def _detect_response_content(self, message: Any) -> tuple[bool, bool, bool]: """ @@ -364,12 +385,25 @@ async def _construct_message_from_response_async(self, response: Any, request: M Message: Constructed message with one or more MessagePiece entries. Raises: - EmptyResponseException: If the response contains no content, audio, or tool calls. + EmptyResponseException: If a non-truncated response contains no content, audio, or tool + calls. A truncated (``finish_reason == "length"``) response with no content instead + yields a graceful empty piece so the run continues. """ audio_format = self._audio_response_config.audio_format if self._audio_response_config else "wav" pieces = await build_response_pieces_async(response=response, request=request, audio_format=audio_format) if not pieces: + # A truncated (finish_reason == "length") response may legitimately produce no content; + # return a graceful empty piece so the run continues. Validation already raised for + # genuinely empty (non-truncated) responses. + choice = response.choices[0] if getattr(response, "choices", None) else None + if choice is not None and getattr(choice, "finish_reason", None) == "length": + return construct_response_from_request( + request=request, + response_text_pieces=[""], + response_type="text", + error="empty", + ) raise EmptyResponseException(message="Failed to extract any response content.") # Capture token usage from the API response and store in the first piece's metadata diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py index 563dfaeb2d..00d011353d 100644 --- a/pyrit/prompt_target/openai/openai_response_target.py +++ b/pyrit/prompt_target/openai/openai_response_target.py @@ -27,6 +27,7 @@ MessagePiece, PromptDataType, PromptResponseError, + construct_response_from_request, ) from pyrit.prompt_target.common.target_capabilities import TargetCapabilities from pyrit.prompt_target.common.target_configuration import TargetConfiguration @@ -505,31 +506,49 @@ def _extract_partial_content(self, response: Any) -> str | None: except (AttributeError, IndexError, TypeError): return None - def _validate_response(self, response: Any, request: MessagePiece) -> Message | None: + def _validate_response(self, response: Any, request: MessagePiece) -> None: """ Validate a Response API response for errors. Checks for: - Error responses (excluding content filtering which is checked separately) + - Truncation at the token limit (``max_output_tokens``), which is warned about, not raised - Invalid status - Empty output + Truncation is treated as valid, with a warning, so that + ``_construct_message_from_response_async`` can preserve any completed output (reasoning, + partial text) or fall back to a graceful empty response. Genuinely empty responses (no + truncation) are raised so the retry logic can attempt to get a complete response. Content + filter responses are handled separately by ``_check_content_filter``. + Args: response: The Response object from the OpenAI SDK. request: The original request MessagePiece. - Returns: - None if valid, does not return Message for content filter (handled by _check_content_filter). - Raises: PyritException: For unexpected response structures or errors. - EmptyResponseException: When the API returns no valid output. + EmptyResponseException: When the API returns no valid output (and was not truncated). """ # Check for error response - error is a ResponseError object or None # (content_filter is handled by _check_content_filter) if response.error is not None and response.error.code != "content_filter": raise PyritException(message=f"Response error: {response.error.code} - {response.error.message}") + # Truncation: the model hit max_output_tokens. Mirroring OpenAIChatTarget's handling of + # finish_reason == "length", warn instead of raising so the run continues -- reasoning models + # can spend the whole budget on hidden reasoning before emitting a visible answer, and a low + # limit may be a deliberate configuration. Construction preserves any completed output + # (reasoning, partial text) and falls back to a graceful empty response. + if self._is_truncated_response(response): + logger.warning( + "The response was truncated because it reached the token limit " + "(status='incomplete', reason='max_output_tokens'). Reasoning models consume tokens on " + "hidden reasoning in addition to the visible answer, so a low max_output_tokens can " + "truncate or empty the response. Increase max_output_tokens if you expected complete content." + ) + return + # Check status - should be "completed" for successful responses if response.status != "completed": raise PyritException(message=f"Unexpected status: {response.status}") @@ -539,12 +558,35 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | logger.error("The response returned no valid output.") raise EmptyResponseException(message="The response returned an empty response.") - return None + def _is_truncated_response(self, response: Any) -> bool: + """ + Return True if the response was cut off by the ``max_output_tokens`` limit. + + The Responses API signals truncation via ``status == "incomplete"`` with + ``incomplete_details.reason == "max_output_tokens"`` (``content_filter`` is handled + separately by ``_check_content_filter``). + + Args: + response: A Response object from the OpenAI SDK. + + Returns: + bool: True if the response was truncated at the token limit, False otherwise. + """ + if getattr(response, "status", None) != "incomplete": + return False + incomplete_details = getattr(response, "incomplete_details", None) + reason = getattr(incomplete_details, "reason", None) if incomplete_details else None + return reason == "max_output_tokens" async def _construct_message_from_response_async(self, response: Any, request: MessagePiece) -> Message: """ Construct a Message from a Response API response. + For a truncated response (see ``_is_truncated_response``), empty output sections are + tolerated, partial tool/function calls are skipped so an incomplete call cannot re-enter the + agentic loop, and a graceful empty text piece is appended when no visible text was produced. + Reasoning and any partial text are always preserved. + Args: response: The Response object from OpenAI SDK. request: The original request MessagePiece. @@ -552,17 +594,36 @@ async def _construct_message_from_response_async(self, response: Any, request: M Returns: Message: Constructed message with extracted content from output sections. """ + truncated = self._is_truncated_response(response) + # Extract and parse message pieces from validated output sections extracted_response_pieces: list[MessagePiece] = [] - for section in response.output: + has_text = False + for section in getattr(response, "output", None) or []: piece = self._parse_response_output_section( section=section, message_piece=request, error=None, # error is already handled in validation + tolerate_empty=truncated, ) if piece is None: continue + # On truncation, keep only reasoning/text; a partial tool or function call must not + # re-enter the agentic loop. + if truncated and piece.original_value_data_type not in ("text", "reasoning"): + continue extracted_response_pieces.append(piece) + if piece.original_value_data_type == "text" and piece.original_value: + has_text = True + + if truncated and not has_text: + empty_piece = construct_response_from_request( + request=request, + response_text_pieces=[""], + response_type="text", + error="empty", + ).message_pieces[0] + extracted_response_pieces.append(empty_piece) return Message(message_pieces=extracted_response_pieces) @@ -637,7 +698,12 @@ async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Me return responses_to_return def _parse_response_output_section( - self, *, section: Any, message_piece: MessagePiece, error: PromptResponseError | None + self, + *, + section: Any, + message_piece: MessagePiece, + error: PromptResponseError | None, + tolerate_empty: bool = False, ) -> MessagePiece | None: """ Parse model output sections, forwarding tool-calls for the agentic loop. @@ -646,12 +712,15 @@ def _parse_response_output_section( section: The section object from OpenAI SDK (Pydantic model). message_piece: The original message piece. error: Any error information from OpenAI. + tolerate_empty: When True, empty sections return None instead of raising + EmptyResponseException. Used when constructing a truncated response. Returns: A MessagePiece for this section, or None to skip. Raises: - EmptyResponseException: If the section content is empty or invalid. + EmptyResponseException: If the section content is empty or invalid (and tolerate_empty + is False). ValueError: If the section type is unsupported. """ section_type = section.type @@ -661,6 +730,8 @@ def _parse_response_output_section( if section_type == MessagePieceType.MESSAGE: section_content = section.content if len(section_content) == 0: + if tolerate_empty: + return None raise EmptyResponseException(message="The chat returned an empty message section.") piece_value = section_content[0].text @@ -715,6 +786,8 @@ def _parse_response_output_section( raise ValueError(msg) piece_value = section.input if len(piece_value) == 0: + if tolerate_empty: + return None raise EmptyResponseException(message="The chat returned an empty message section.") else: @@ -723,6 +796,8 @@ def _parse_response_output_section( # Handle empty response if not piece_value: + if tolerate_empty: + return None raise EmptyResponseException(message="The chat returned an empty response.") return MessagePiece( diff --git a/pyrit/prompt_target/openai/openai_target.py b/pyrit/prompt_target/openai/openai_target.py index 3700ebc370..6117afe425 100644 --- a/pyrit/prompt_target/openai/openai_target.py +++ b/pyrit/prompt_target/openai/openai_target.py @@ -448,10 +448,8 @@ async def _handle_openai_request_async( if self._check_content_filter(response): return self._handle_content_filter_response(response, request_piece) - # Validate response via subclass implementation - error_message = self._validate_response(response, request_piece) - if error_message: - return error_message + # Validate response via subclass implementation (raises on invalid responses) + self._validate_response(response, request_piece) # Construct and return Message from validated response return await self._construct_message_from_response_async(response, request_piece) @@ -604,24 +602,21 @@ def _extract_partial_content(self, response: Any) -> str | None: """ return None - def _validate_response(self, response: Any, request: MessagePiece) -> Message | None: + def _validate_response(self, response: Any, request: MessagePiece) -> None: """ - Validate the response and return error Message if needed. + Validate the response, raising if it is invalid. - Override this method in subclasses that need custom response validation. - Default implementation returns None (no validation errors). + Override this method in subclasses that need custom response validation. Validation only + inspects the response; constructing the resulting Message is the responsibility of + ``_construct_message_from_response_async``. The default implementation is a no-op. Args: response: The response object from OpenAI SDK. request: The original request MessagePiece. - Returns: - Message | None: Error Message if validation fails, None otherwise. - Raises: Various exceptions for validation failures. """ - return None @abstractmethod def _set_openai_env_configuration_vars(self) -> None: diff --git a/tests/unit/prompt_target/target/test_openai_chat_target.py b/tests/unit/prompt_target/target/test_openai_chat_target.py index 71a2ca7d63..51d66a39eb 100644 --- a/tests/unit/prompt_target/target/test_openai_chat_target.py +++ b/tests/unit/prompt_target/target/test_openai_chat_target.py @@ -1038,11 +1038,44 @@ def test_validate_response_success_stop(target: OpenAIChatTarget, dummy_text_mes assert result is None -def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece): - """Test _validate_response passes for valid length response.""" +def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog): + """Test _validate_response passes for a truncated response that still has content, and warns.""" mock_response = create_mock_completion(content="Hello", finish_reason="length") - result = target._validate_response(mock_response, dummy_text_message_piece) + with caplog.at_level(logging.WARNING): + result = target._validate_response(mock_response, dummy_text_message_piece) + assert result is None + assert "finish_reason='length'" in caplog.text + + +def test_validate_response_length_empty_does_not_raise( + target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog +): + """Test _validate_response treats a truncated-but-empty response as valid (warns, does not raise).""" + mock_response = create_mock_completion(content="", finish_reason="length") + with caplog.at_level(logging.WARNING): + result = target._validate_response(mock_response, dummy_text_message_piece) assert result is None + assert "finish_reason='length'" in caplog.text + + +async def test_construct_message_length_empty_returns_graceful_empty( + target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece +): + """Test construction returns a graceful empty piece when a truncated response produced no content.""" + mock_response = create_mock_completion(content="", finish_reason="length") + result = await target._construct_message_from_response_async(mock_response, dummy_text_message_piece) + assert isinstance(result, Message) + assert result.message_pieces[0].original_value == "" + assert result.message_pieces[0].response_error == "empty" + + +async def test_construct_message_empty_non_truncated_raises( + target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece +): + """Test construction raises for a genuinely empty (non-truncated) response so retries can kick in.""" + mock_response = create_mock_completion(content="", finish_reason="stop") + with pytest.raises(EmptyResponseException, match="Failed to extract any response content"): + await target._construct_message_from_response_async(mock_response, dummy_text_message_piece) def test_validate_response_no_choices(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece): diff --git a/tests/unit/prompt_target/target/test_openai_response_target.py b/tests/unit/prompt_target/target/test_openai_response_target.py index 5cd0f1f790..d6a7a6ead8 100644 --- a/tests/unit/prompt_target/target/test_openai_response_target.py +++ b/tests/unit/prompt_target/target/test_openai_response_target.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import json +import logging import os from collections.abc import MutableSequence from tempfile import NamedTemporaryFile @@ -1192,6 +1193,129 @@ def test_validate_response_empty_output(target: OpenAIResponseTarget, dummy_text target._validate_response(mock_response, dummy_text_message_piece) +def _make_reasoning_section() -> MagicMock: + section = MagicMock() + section.type = "reasoning" + section.model_dump.return_value = {"type": "reasoning", "summary": []} + return section + + +def _make_message_section(text: str) -> MagicMock: + section = MagicMock() + section.type = "message" + content_item = MagicMock() + content_item.text = text + section.content = [content_item] + return section + + +def _make_empty_message_section() -> MagicMock: + section = MagicMock() + section.type = "message" + section.content = [] + return section + + +def _make_truncated_response(output: list) -> MagicMock: + mock_response = MagicMock() + mock_response.error = None + mock_response.status = "incomplete" + incomplete_details = MagicMock() + incomplete_details.reason = "max_output_tokens" + mock_response.incomplete_details = incomplete_details + mock_response.output = output + return mock_response + + +def test_is_truncated_response_detects_max_output_tokens(target: OpenAIResponseTarget): + """_is_truncated_response is True only for incomplete status with a max_output_tokens reason.""" + truncated = _make_truncated_response(output=[]) + assert target._is_truncated_response(truncated) is True + + content_filtered = _make_truncated_response(output=[]) + content_filtered.incomplete_details.reason = "content_filter" + assert target._is_truncated_response(content_filtered) is False + + completed = MagicMock() + completed.status = "completed" + assert target._is_truncated_response(completed) is False + + +def test_validate_response_truncated_warns_and_does_not_raise( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece, caplog +): + """Truncation is treated as valid: _validate_response warns and returns None (does not raise).""" + response = _make_truncated_response(output=[_make_reasoning_section(), _make_empty_message_section()]) + + with caplog.at_level(logging.WARNING): + result = target._validate_response(response, dummy_text_message_piece) + + assert result is None + assert "max_output_tokens" in caplog.text + + +async def test_construct_message_truncated_keeps_reasoning_and_empty_text( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece +): + """Truncated response with reasoning but empty text: keep reasoning, add a graceful empty text piece.""" + response = _make_truncated_response(output=[_make_reasoning_section(), _make_empty_message_section()]) + + result = await target._construct_message_from_response_async(response, dummy_text_message_piece) + + reasoning_pieces = [p for p in result.message_pieces if p.original_value_data_type == "reasoning"] + text_pieces = [p for p in result.message_pieces if p.original_value_data_type == "text"] + assert len(reasoning_pieces) == 1 + assert len(text_pieces) == 1 + assert text_pieces[0].original_value == "" + assert text_pieces[0].response_error == "empty" + + +async def test_construct_message_truncated_keeps_partial_text( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece +): + """Truncated response with partial visible text keeps it (error=none), no empty piece added.""" + response = _make_truncated_response(output=[_make_reasoning_section(), _make_message_section("Partial answer")]) + + result = await target._construct_message_from_response_async(response, dummy_text_message_piece) + + text_pieces = [p for p in result.message_pieces if p.original_value_data_type == "text"] + assert len(text_pieces) == 1 + assert text_pieces[0].original_value == "Partial answer" + assert text_pieces[0].response_error == "none" + + +async def test_construct_message_truncated_empty_output_returns_graceful_empty( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece +): + """Truncated response with no output yields a single graceful empty text piece (does not raise).""" + response = _make_truncated_response(output=[]) + + result = await target._construct_message_from_response_async(response, dummy_text_message_piece) + + assert len(result.message_pieces) == 1 + assert result.message_pieces[0].original_value == "" + assert result.message_pieces[0].response_error == "empty" + + +async def test_construct_message_truncated_skips_partial_tool_call( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece +): + """A partial function_call in a truncated response is skipped so it cannot re-enter the agentic loop.""" + func_section = MagicMock() + func_section.type = "function_call" + func_section.call_id = "call_1" + func_section.name = "do_thing" + func_section.arguments = "{}" + response = _make_truncated_response(output=[_make_reasoning_section(), func_section]) + + result = await target._construct_message_from_response_async(response, dummy_text_message_piece) + + data_types = [p.original_value_data_type for p in result.message_pieces] + assert "function_call" not in data_types + assert "reasoning" in data_types + assert any(p.original_value_data_type == "text" and p.response_error == "empty" for p in result.message_pieces) + + async def test_construct_message_from_response(target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece): """Test _construct_message_from_response parses output sections.""" mock_response = MagicMock()