Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
267 changes: 232 additions & 35 deletions doc/code/targets/2_openai_responses_target.ipynb

Large diffs are not rendered by default.

34 changes: 30 additions & 4 deletions doc/code/targets/2_openai_responses_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,32 @@
result = await attack.execute_async(objective="Tell me a joke") # type: ignore
await ConsoleAttackResultPrinter().print_conversation_async(result=result) # type: ignore

# %% [markdown]
# ## Reasoning Configuration
#
# Reasoning models (e.g., o1, o3, o4-mini, GPT-5) support a `reasoning` parameter that controls how much internal reasoning the model performs before responding. You can configure this with two parameters:
#
# - **`reasoning_effort`**: Controls the depth of reasoning. Accepts `"minimal"`, `"low"`, `"medium"`, or `"high"`. Lower effort favors speed and lower cost; higher effort favors thoroughness. The default (when not set) is typically `"medium"`.
# - **`reasoning_summary`**: Controls whether a summary of the model's internal reasoning is included in the response. Accepts `"auto"`, `"concise"`, or `"detailed"`. By default, no summary is included.
#
# For more information, see the [OpenAI reasoning guide](https://developers.openai.com/api/docs/guides/reasoning).

# %%
from pyrit.executor.attack import ConsoleAttackResultPrinter, PromptSendingAttack
from pyrit.prompt_target import OpenAIResponseTarget
from pyrit.setup import IN_MEMORY, initialize_pyrit_async

await initialize_pyrit_async(memory_db_type=IN_MEMORY) # type: ignore

target = OpenAIResponseTarget(
reasoning_effort="high",
reasoning_summary="detailed",
)

attack = PromptSendingAttack(objective_target=target)
result = await attack.execute_async(objective="What are the most dangerous items in a household?") # type: ignore
await ConsoleAttackResultPrinter().print_conversation_async(result=result, include_reasoning_trace=True) # type: ignore

# %% [markdown]
# ## JSON Generation
#
Expand Down Expand Up @@ -170,7 +196,7 @@ async def get_current_weather(args):

for response_msg in response:
for idx, piece in enumerate(response_msg.message_pieces):
print(f"{idx} | {piece.role}: {piece.original_value}")
print(f"{idx} | {piece.api_role}: {piece.original_value}")

# %% [markdown]
# ## Using the Built-in Web Search Tool
Expand Down Expand Up @@ -216,7 +242,7 @@ async def get_current_weather(args):

for response_msg in response:
for idx, piece in enumerate(response_msg.message_pieces):
print(f"{idx} | {piece.role}: {piece.original_value}")
print(f"{idx} | {piece.api_role}: {piece.original_value}")

# %% [markdown]
# ## Grammar-Constrained Generation
Expand Down Expand Up @@ -278,11 +304,11 @@ async def get_current_weather(args):
print("Unconstrained Response:")
for response_msg in unconstrained_result:
for idx, piece in enumerate(response_msg.message_pieces):
print(f"{idx} | {piece.role}: {piece.original_value}")
print(f"{idx} | {piece.api_role}: {piece.original_value}")

print()

print("Constrained Response:")
for response_msg in result:
for idx, piece in enumerate(response_msg.message_pieces):
print(f"{idx} | {piece.role}: {piece.original_value}")
print(f"{idx} | {piece.api_role}: {piece.original_value}")
33 changes: 31 additions & 2 deletions pyrit/executor/attack/printer/console_printer.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

import json
import textwrap
from datetime import datetime
from typing import Any
Expand Down Expand Up @@ -201,8 +202,14 @@ async def print_messages_async(

# Now print all pieces in this message
for piece in message.message_pieces:
# Skip reasoning traces unless explicitly requested
if piece.original_value_data_type == "reasoning" and not include_reasoning_trace:
# Reasoning pieces: show summary when include_reasoning_trace is set
if piece.original_value_data_type == "reasoning":
if include_reasoning_trace:
summary_text = self._extract_reasoning_summary(piece.original_value)
if summary_text:
self._print_colored(f"{self._indent}💭 Reasoning Summary:", Style.DIM, Fore.CYAN)
self._print_wrapped_text(summary_text, Fore.CYAN)
print()
continue

# Handle converted values for user and assistant messages
Expand Down Expand Up @@ -234,6 +241,28 @@ async def print_messages_async(
print()
self._print_colored("─" * self._width, Fore.BLUE)

def _extract_reasoning_summary(self, reasoning_value: str) -> str:
"""
Extract human-readable summary text from a reasoning piece's JSON value.

Args:
reasoning_value (str): The JSON string stored in the reasoning piece.

Returns:
str: The concatenated summary text, or empty string if no summary is present.
"""
try:
data = json.loads(reasoning_value)
except (json.JSONDecodeError, TypeError):
return ""

summary = data.get("summary") if isinstance(data, dict) else None
if not summary or not isinstance(summary, list):
return ""

parts = [item.get("text", "") for item in summary if isinstance(item, dict) and item.get("text")]
return "\n".join(parts)

async def print_summary_async(self, result: AttackResult) -> None:
"""
Print a summary of the attack result with enhanced formatting.
Expand Down
50 changes: 38 additions & 12 deletions pyrit/prompt_target/openai/openai_response_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,14 @@
Callable,
Dict,
List,
Literal,
MutableSequence,
Optional,
cast,
)

from openai.types.shared import ReasoningEffort

from pyrit.common import convert_local_image_to_data_url
from pyrit.exceptions import (
EmptyResponseException,
Expand Down Expand Up @@ -75,6 +78,8 @@ def __init__(
max_output_tokens: Optional[int] = None,
temperature: Optional[float] = None,
top_p: Optional[float] = None,
reasoning_effort: Optional[ReasoningEffort] = None,
reasoning_summary: Optional[Literal["auto", "concise", "detailed"]] = None,
extra_body_parameters: Optional[dict[str, Any]] = None,
fail_on_missing_function: bool = False,
**kwargs: Any,
Expand All @@ -100,6 +105,13 @@ def __init__(
randomness of the response.
top_p (float, Optional): The top-p parameter for controlling the diversity of the
response.
reasoning_effort (ReasoningEffort, Optional): Controls how much reasoning the model
performs. Accepts "minimal", "low", "medium", or "high". Lower effort
favors speed and lower cost; higher effort favors thoroughness. Defaults to None
(uses model default, typically "medium").
reasoning_summary (Literal["auto", "concise", "detailed"], Optional): Controls
whether a summary of the model's reasoning is included in the response.
Defaults to None (no summary).
is_json_supported (bool, Optional): If True, the target will support formatting responses as JSON by
setting the response_format header. Official OpenAI models all support this, but if you are using
this target with different models, is_json_supported should be set correctly to avoid issues when
Expand Down Expand Up @@ -133,9 +145,8 @@ def __init__(
self._top_p = top_p
self._max_output_tokens = max_output_tokens

# Reasoning parameters are not yet supported by PyRIT.
# See https://platform.openai.com/docs/api-reference/responses/create#responses-create-reasoning
# for more information.
self._reasoning_effort = reasoning_effort
self._reasoning_summary = reasoning_summary

self._extra_body_parameters = extra_body_parameters

Expand Down Expand Up @@ -169,6 +180,8 @@ def _build_identifier(self) -> ComponentIdentifier:
"temperature": self._temperature,
"top_p": self._top_p,
"max_output_tokens": self._max_output_tokens,
"reasoning_effort": self._reasoning_effort,
"reasoning_summary": self._reasoning_summary,
},
)

Expand Down Expand Up @@ -360,6 +373,7 @@ async def _construct_request_body(
"input": input_items,
# Correct JSON response format per Responses API
"text": text_format,
"reasoning": self._build_reasoning_config(),
}

if self._extra_body_parameters:
Expand All @@ -368,6 +382,23 @@ async def _construct_request_body(
# Filter out None values
return {k: v for k, v in body_parameters.items() if v is not None}

def _build_reasoning_config(self) -> Optional[Dict[str, Any]]:
"""
Build the reasoning configuration dict for the Responses API.

Returns:
Optional[Dict[str, Any]]: The reasoning config, or None if neither effort nor summary is set.
"""
if self._reasoning_effort is None and self._reasoning_summary is None:
return None

reasoning: Dict[str, Any] = {}
if self._reasoning_effort is not None:
reasoning["effort"] = self._reasoning_effort
if self._reasoning_summary is not None:
reasoning["summary"] = self._reasoning_summary
return reasoning

def _build_text_format(self, json_config: _JsonResponseConfig) -> Optional[Dict[str, Any]]:
if not json_config.enabled:
return None
Expand Down Expand Up @@ -577,13 +608,7 @@ def _parse_response_output_section(
elif section_type == MessagePieceType.REASONING:
# Store reasoning in memory for debugging/logging, but won't be sent back to API
piece_value = json.dumps(
{
"id": section.id,
"type": section.type,
"summary": section.summary,
"content": section.content,
"encrypted_content": section.encrypted_content,
},
section.model_dump(),
separators=(",", ":"),
)
piece_type = "reasoning"
Expand Down Expand Up @@ -691,12 +716,13 @@ def _find_last_pending_tool_call(self, reply: Message) -> Optional[dict[str, Any
The tool-call section dict, or None if not found.
"""
for piece in reversed(reply.message_pieces):
if piece.api_role == "assistant":
# Filter on data_type to skip reasoning/message pieces that also have api_role "assistant".
if piece.api_role == "assistant" and piece.original_value_data_type == "function_call":
try:
section = json.loads(piece.original_value)
except Exception:
continue
if section.get("type") == "function_call":
if isinstance(section, dict) and section.get("type") == "function_call":
# Do NOT skip function_call even if status == "completed" — we still need to emit the output.
return cast(dict[str, Any], section)
return None
Expand Down
51 changes: 51 additions & 0 deletions tests/integration/targets/test_entra_auth_targets.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,57 @@ async def test_openai_responses_target_entra_auth(sqlite_instance, endpoint, mod
assert result.last_response is not None


@pytest.mark.asyncio
@pytest.mark.parametrize(
("endpoint", "model_name"),
[
("OPENAI_RESPONSES_ENDPOINT", "OPENAI_RESPONSES_MODEL"),
("AZURE_OPENAI_GPT5_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT5_MODEL"),
],
)
async def test_openai_responses_target_reasoning_effort_entra_auth(sqlite_instance, endpoint, model_name):
endpoint_value = os.environ[endpoint]
args = {
"endpoint": endpoint_value,
"model_name": os.environ[model_name],
"api_key": get_azure_openai_auth(endpoint_value),
"reasoning_effort": "low",
}

target = OpenAIResponseTarget(**args)

attack = PromptSendingAttack(objective_target=target)
result = await attack.execute_async(objective="What is 2 + 2?")
assert result is not None
assert result.last_response is not None


@pytest.mark.asyncio
@pytest.mark.parametrize(
("endpoint", "model_name"),
[
("OPENAI_RESPONSES_ENDPOINT", "OPENAI_RESPONSES_MODEL"),
("AZURE_OPENAI_GPT5_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT5_MODEL"),
],
)
async def test_openai_responses_target_reasoning_summary_entra_auth(sqlite_instance, endpoint, model_name):
endpoint_value = os.environ[endpoint]
args = {
"endpoint": endpoint_value,
"model_name": os.environ[model_name],
"api_key": get_azure_openai_auth(endpoint_value),
"reasoning_effort": "low",
"reasoning_summary": "auto",
}

target = OpenAIResponseTarget(**args)

attack = PromptSendingAttack(objective_target=target)
result = await attack.execute_async(objective="What is 2 + 2?")
assert result is not None
assert result.last_response is not None


@pytest.mark.asyncio
@pytest.mark.parametrize(
("endpoint", "model_name"),
Expand Down
46 changes: 42 additions & 4 deletions tests/integration/targets/test_openai_responses_gpt5.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ async def test_openai_responses_gpt5(sqlite_instance, gpt5_args):
assert result is not None
assert len(result) == 1
assert len(result[0].message_pieces) == 2
assert result[0].message_pieces[0].role == "assistant"
assert result[0].message_pieces[1].role == "assistant"
assert result[0].message_pieces[0].api_role == "assistant"
assert result[0].message_pieces[1].api_role == "assistant"
# Hope that the model manages to give the correct answer somewhere (GPT-5 really should)
assert "Paris" in result[0].message_pieces[1].converted_value

Expand Down Expand Up @@ -104,7 +104,7 @@ async def test_openai_responses_gpt5_json_schema(sqlite_instance, gpt5_args):
assert len(response) == 1
assert len(response[0].message_pieces) == 2
response_piece = response[0].message_pieces[1]
assert response_piece.role == "assistant"
assert response_piece.api_role == "assistant"
response_json = json.loads(response_piece.converted_value)
jsonschema.validate(instance=response_json, schema=cat_schema)

Expand Down Expand Up @@ -140,6 +140,44 @@ async def test_openai_responses_gpt5_json_object(sqlite_instance, gpt5_args):
assert len(response) == 1
assert len(response[0].message_pieces) == 2
response_piece = response[0].message_pieces[1]
assert response_piece.role == "assistant"
assert response_piece.api_role == "assistant"
_ = json.loads(response_piece.converted_value)
# Can't assert more, since the failure could be due to a bad generation by the model


@pytest.mark.asyncio
async def test_openai_responses_gpt5_reasoning_effort(sqlite_instance, gpt5_args):
target = OpenAIResponseTarget(**gpt5_args, reasoning_effort="low")

conv_id = str(uuid.uuid4())

user_piece = MessagePiece(
role="user",
original_value="What is 2 + 2?",
original_value_data_type="text",
conversation_id=conv_id,
)

result = await target.send_prompt_async(message=user_piece.to_message())
assert result is not None
assert len(result) == 1
assert any(p.converted_value_data_type == "text" for p in result[0].message_pieces)


@pytest.mark.asyncio
async def test_openai_responses_gpt5_reasoning_summary(sqlite_instance, gpt5_args):
target = OpenAIResponseTarget(**gpt5_args, reasoning_effort="low", reasoning_summary="auto")

conv_id = str(uuid.uuid4())

user_piece = MessagePiece(
role="user",
original_value="What is 2 + 2?",
original_value_data_type="text",
conversation_id=conv_id,
)

result = await target.send_prompt_async(message=user_piece.to_message())
assert result is not None
assert len(result) == 1
assert any(p.converted_value_data_type == "text" for p in result[0].message_pieces)
Loading