From 9d8469279065518693430bbedfb33c6c0690cf3f Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Fri, 20 Feb 2026 16:23:45 -0800 Subject: [PATCH 1/9] feat: add reasoning_effort and reasoning_summary to OpenAIResponseTarget Adds first-class support for the OpenAI Responses API reasoning parameter. Users can now control reasoning effort (minimal/low/medium/high) and optionally request reasoning summaries (auto/concise/detailed). Uses ReasoningEffort type from the OpenAI SDK directly. Closes #1384 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../openai/openai_response_target.py | 37 +++++- .../targets/test_openai_responses_gpt5.py | 38 ++++++ .../target/test_openai_response_target.py | 117 ++++++++++++++++++ 3 files changed, 189 insertions(+), 3 deletions(-) diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py index bc709fa4b7..810a838814 100644 --- a/pyrit/prompt_target/openai/openai_response_target.py +++ b/pyrit/prompt_target/openai/openai_response_target.py @@ -10,11 +10,14 @@ Callable, Dict, List, + Literal, MutableSequence, Optional, cast, ) +from openai.types.shared import ReasoningEffort + from pyrit.common import convert_local_image_to_data_url from pyrit.exceptions import ( EmptyResponseException, @@ -75,6 +78,8 @@ def __init__( max_output_tokens: Optional[int] = None, temperature: Optional[float] = None, top_p: Optional[float] = None, + reasoning_effort: Optional[ReasoningEffort] = None, + reasoning_summary: Optional[Literal["auto", "concise", "detailed"]] = None, extra_body_parameters: Optional[dict[str, Any]] = None, fail_on_missing_function: bool = False, **kwargs: Any, @@ -100,6 +105,13 @@ def __init__( randomness of the response. top_p (float, Optional): The top-p parameter for controlling the diversity of the response. + reasoning_effort (ReasoningEffort, Optional): Controls how much reasoning the model + performs. Accepts "minimal", "low", "medium", or "high". Lower effort + favors speed and lower cost; higher effort favors thoroughness. Defaults to None + (uses model default, typically "medium"). + reasoning_summary (Literal["auto", "concise", "detailed"], Optional): Controls + whether a summary of the model's reasoning is included in the response. + Defaults to None (no summary). is_json_supported (bool, Optional): If True, the target will support formatting responses as JSON by setting the response_format header. Official OpenAI models all support this, but if you are using this target with different models, is_json_supported should be set correctly to avoid issues when @@ -133,9 +145,8 @@ def __init__( self._top_p = top_p self._max_output_tokens = max_output_tokens - # Reasoning parameters are not yet supported by PyRIT. - # See https://platform.openai.com/docs/api-reference/responses/create#responses-create-reasoning - # for more information. + self._reasoning_effort = reasoning_effort + self._reasoning_summary = reasoning_summary self._extra_body_parameters = extra_body_parameters @@ -169,6 +180,8 @@ def _build_identifier(self) -> TargetIdentifier: top_p=self._top_p, target_specific_params={ "max_output_tokens": self._max_output_tokens, + "reasoning_effort": self._reasoning_effort, + "reasoning_summary": self._reasoning_summary, }, ) @@ -360,6 +373,7 @@ async def _construct_request_body( "input": input_items, # Correct JSON response format per Responses API "text": text_format, + "reasoning": self._build_reasoning_config(), } if self._extra_body_parameters: @@ -368,6 +382,23 @@ async def _construct_request_body( # Filter out None values return {k: v for k, v in body_parameters.items() if v is not None} + def _build_reasoning_config(self) -> Optional[Dict[str, Any]]: + """ + Build the reasoning configuration dict for the Responses API. + + Returns: + Optional[Dict[str, Any]]: The reasoning config, or None if neither effort nor summary is set. + """ + if self._reasoning_effort is None and self._reasoning_summary is None: + return None + + reasoning: Dict[str, Any] = {} + if self._reasoning_effort is not None: + reasoning["effort"] = self._reasoning_effort + if self._reasoning_summary is not None: + reasoning["summary"] = self._reasoning_summary + return reasoning + def _build_text_format(self, json_config: _JsonResponseConfig) -> Optional[Dict[str, Any]]: if not json_config.enabled: return None diff --git a/tests/integration/targets/test_openai_responses_gpt5.py b/tests/integration/targets/test_openai_responses_gpt5.py index 82f56bc8c6..347183f938 100644 --- a/tests/integration/targets/test_openai_responses_gpt5.py +++ b/tests/integration/targets/test_openai_responses_gpt5.py @@ -143,3 +143,41 @@ async def test_openai_responses_gpt5_json_object(sqlite_instance, gpt5_args): assert response_piece.role == "assistant" _ = json.loads(response_piece.converted_value) # Can't assert more, since the failure could be due to a bad generation by the model + + +@pytest.mark.asyncio +async def test_openai_responses_gpt5_reasoning_effort(sqlite_instance, gpt5_args): + target = OpenAIResponseTarget(**gpt5_args, reasoning_effort="low") + + conv_id = str(uuid.uuid4()) + + user_piece = MessagePiece( + role="user", + original_value="What is 2 + 2?", + original_value_data_type="text", + conversation_id=conv_id, + ) + + result = await target.send_prompt_async(message=user_piece.to_message()) + assert result is not None + assert len(result) == 1 + assert any(p.converted_value_data_type == "text" for p in result[0].message_pieces) + + +@pytest.mark.asyncio +async def test_openai_responses_gpt5_reasoning_summary(sqlite_instance, gpt5_args): + target = OpenAIResponseTarget(**gpt5_args, reasoning_effort="low", reasoning_summary="auto") + + conv_id = str(uuid.uuid4()) + + user_piece = MessagePiece( + role="user", + original_value="What is 2 + 2?", + original_value_data_type="text", + conversation_id=conv_id, + ) + + result = await target.send_prompt_async(message=user_piece.to_message()) + assert result is not None + assert len(result) == 1 + assert any(p.converted_value_data_type == "text" for p in result[0].message_pieces) diff --git a/tests/unit/target/test_openai_response_target.py b/tests/unit/target/test_openai_response_target.py index e6b083efe8..089e895902 100644 --- a/tests/unit/target/test_openai_response_target.py +++ b/tests/unit/target/test_openai_response_target.py @@ -1178,3 +1178,120 @@ async def test_construct_message_from_response(target: OpenAIResponseTarget, dum assert isinstance(result, Message) assert len(result.message_pieces) == 1 mock_parse.assert_called_once() + + +# ── Reasoning effort / summary tests ─────────────────────────────────────── + + +def test_init_with_reasoning_effort(patch_central_database): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_effort="high", + ) + assert target._reasoning_effort == "high" + + +def test_init_with_reasoning_summary(patch_central_database): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_summary="auto", + ) + assert target._reasoning_summary == "auto" + + +def test_init_with_reasoning_effort_and_summary(patch_central_database): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_effort="low", + reasoning_summary="detailed", + ) + assert target._reasoning_effort == "low" + assert target._reasoning_summary == "detailed" + + +def test_init_without_reasoning_params(patch_central_database): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + ) + assert target._reasoning_effort is None + assert target._reasoning_summary is None + + +@pytest.mark.asyncio +async def test_construct_request_body_includes_reasoning_effort( + patch_central_database, dummy_text_message_piece: MessagePiece +): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_effort="medium", + ) + request = Message(message_pieces=[dummy_text_message_piece]) + jrc = _JsonResponseConfig.from_metadata(metadata=None) + body = await target._construct_request_body(conversation=[request], json_config=jrc) + assert body["reasoning"] == {"effort": "medium"} + + +@pytest.mark.asyncio +async def test_construct_request_body_includes_reasoning_summary( + patch_central_database, dummy_text_message_piece: MessagePiece +): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_summary="detailed", + ) + request = Message(message_pieces=[dummy_text_message_piece]) + jrc = _JsonResponseConfig.from_metadata(metadata=None) + body = await target._construct_request_body(conversation=[request], json_config=jrc) + assert body["reasoning"] == {"summary": "detailed"} + + +@pytest.mark.asyncio +async def test_construct_request_body_includes_reasoning_effort_and_summary( + patch_central_database, dummy_text_message_piece: MessagePiece +): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_effort="high", + reasoning_summary="auto", + ) + request = Message(message_pieces=[dummy_text_message_piece]) + jrc = _JsonResponseConfig.from_metadata(metadata=None) + body = await target._construct_request_body(conversation=[request], json_config=jrc) + assert body["reasoning"] == {"effort": "high", "summary": "auto"} + + +@pytest.mark.asyncio +async def test_construct_request_body_omits_reasoning_when_not_set( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece +): + request = Message(message_pieces=[dummy_text_message_piece]) + jrc = _JsonResponseConfig.from_metadata(metadata=None) + body = await target._construct_request_body(conversation=[request], json_config=jrc) + assert "reasoning" not in body + + +def test_build_identifier_includes_reasoning_params(patch_central_database): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_effort="low", + reasoning_summary="concise", + ) + identifier = target._build_identifier() + assert identifier.target_specific_params["reasoning_effort"] == "low" + assert identifier.target_specific_params["reasoning_summary"] == "concise" From eb033d2411a3a7846234a8e66b705a0ea56d125c Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Fri, 20 Feb 2026 16:35:50 -0800 Subject: [PATCH 2/9] fix: use model_dump for reasoning sections, filter tool calls by data type, add Entra integration tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../openai/openai_response_target.py | 12 ++--- .../targets/test_entra_auth_targets.py | 53 +++++++++++++++++++ 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py index 810a838814..269621f449 100644 --- a/pyrit/prompt_target/openai/openai_response_target.py +++ b/pyrit/prompt_target/openai/openai_response_target.py @@ -608,13 +608,7 @@ def _parse_response_output_section( elif section_type == MessagePieceType.REASONING: # Store reasoning in memory for debugging/logging, but won't be sent back to API piece_value = json.dumps( - { - "id": section.id, - "type": section.type, - "summary": section.summary, - "content": section.content, - "encrypted_content": section.encrypted_content, - }, + section.model_dump(), separators=(",", ":"), ) piece_type = "reasoning" @@ -722,12 +716,12 @@ def _find_last_pending_tool_call(self, reply: Message) -> Optional[dict[str, Any The tool-call section dict, or None if not found. """ for piece in reversed(reply.message_pieces): - if piece.api_role == "assistant": + if piece.api_role == "assistant" and piece.original_value_data_type == "function_call": try: section = json.loads(piece.original_value) except Exception: continue - if section.get("type") == "function_call": + if isinstance(section, dict) and section.get("type") == "function_call": # Do NOT skip function_call even if status == "completed" — we still need to emit the output. return cast(dict[str, Any], section) return None diff --git a/tests/integration/targets/test_entra_auth_targets.py b/tests/integration/targets/test_entra_auth_targets.py index a1fa3ebe5d..f7f56c603e 100644 --- a/tests/integration/targets/test_entra_auth_targets.py +++ b/tests/integration/targets/test_entra_auth_targets.py @@ -237,6 +237,59 @@ async def test_openai_responses_target_entra_auth(sqlite_instance, endpoint, mod assert result.last_response is not None +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("endpoint", "model_name"), + [ + ("OPENAI_RESPONSES_ENDPOINT", "OPENAI_RESPONSES_MODEL"), + ("AZURE_OPENAI_GPT41_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT41_RESPONSES_MODEL"), + ("AZURE_OPENAI_GPT5_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT5_MODEL"), + ], +) +async def test_openai_responses_target_reasoning_effort_entra_auth(sqlite_instance, endpoint, model_name): + endpoint_value = os.environ[endpoint] + args = { + "endpoint": endpoint_value, + "model_name": os.environ[model_name], + "api_key": get_azure_openai_auth(endpoint_value), + "reasoning_effort": "low", + } + + target = OpenAIResponseTarget(**args) + + attack = PromptSendingAttack(objective_target=target) + result = await attack.execute_async(objective="What is 2 + 2?") + assert result is not None + assert result.last_response is not None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("endpoint", "model_name"), + [ + ("OPENAI_RESPONSES_ENDPOINT", "OPENAI_RESPONSES_MODEL"), + ("AZURE_OPENAI_GPT41_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT41_RESPONSES_MODEL"), + ("AZURE_OPENAI_GPT5_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT5_MODEL"), + ], +) +async def test_openai_responses_target_reasoning_summary_entra_auth(sqlite_instance, endpoint, model_name): + endpoint_value = os.environ[endpoint] + args = { + "endpoint": endpoint_value, + "model_name": os.environ[model_name], + "api_key": get_azure_openai_auth(endpoint_value), + "reasoning_effort": "low", + "reasoning_summary": "auto", + } + + target = OpenAIResponseTarget(**args) + + attack = PromptSendingAttack(objective_target=target) + result = await attack.execute_async(objective="What is 2 + 2?") + assert result is not None + assert result.last_response is not None + + @pytest.mark.asyncio @pytest.mark.parametrize( ("endpoint", "model_name"), From b8b5bc0eb5e81d9a8c4ee0dba88b02fe08a6a19c Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Fri, 20 Feb 2026 16:57:15 -0800 Subject: [PATCH 3/9] docs: add reasoning configuration section to OpenAI Responses Target notebook Adds a new section demonstrating reasoning_effort and reasoning_summary parameters with a working example. Regenerated .ipynb from .py. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../targets/2_openai_responses_target.ipynb | 164 ++++++++++++++---- doc/code/targets/2_openai_responses_target.py | 26 +++ 2 files changed, 158 insertions(+), 32 deletions(-) diff --git a/doc/code/targets/2_openai_responses_target.ipynb b/doc/code/targets/2_openai_responses_target.ipynb index ff7e364695..3108821443 100644 --- a/doc/code/targets/2_openai_responses_target.ipynb +++ b/doc/code/targets/2_openai_responses_target.ipynb @@ -31,9 +31,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['/home/vscode/.pyrit/.env', '/home/vscode/.pyrit/.env.local']\n", - "Loaded environment file: /home/vscode/.pyrit/.env\n", - "Loaded environment file: /home/vscode/.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" ] }, { @@ -49,7 +49,9 @@ "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", "\u001b[1m\u001b[33m🔸 ASSISTANT\u001b[0m\n", "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", - "\u001b[33m Why did the coffee file a police report? It got mugged!\u001b[0m\n", + "\u001b[33m Why don’t scientists trust atoms?\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m Because they make up everything!\u001b[0m\n", "\n", "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" ] @@ -82,6 +84,78 @@ "cell_type": "markdown", "id": "2", "metadata": {}, + "source": [ + "## Reasoning Configuration\n", + "\n", + "Reasoning models (e.g., o1, o3, o4-mini, GPT-5) support a `reasoning` parameter that controls how much internal reasoning the model performs before responding. You can configure this with two parameters:\n", + "\n", + "- **`reasoning_effort`**: Controls the depth of reasoning. Accepts `\"minimal\"`, `\"low\"`, `\"medium\"`, or `\"high\"`. Lower effort favors speed and lower cost; higher effort favors thoroughness. The default (when not set) is typically `\"medium\"`.\n", + "- **`reasoning_summary`**: Controls whether a summary of the model's internal reasoning is included in the response. Accepts `\"auto\"`, `\"concise\"`, or `\"detailed\"`. By default, no summary is included.\n", + "\n", + "For more information, see the [OpenAI reasoning guide](https://developers.openai.com/api/docs/guides/reasoning)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[1m\u001b[34m🔹 Turn 1 - USER\u001b[0m\n", + "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[34m What are the first 5 prime numbers?\u001b[0m\n", + "\n", + "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[1m\u001b[33m🔸 ASSISTANT\u001b[0m\n", + "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[33m Here are the first five prime numbers in ascending order:\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 1. 2\u001b[0m\n", + "\u001b[33m 2. 3\u001b[0m\n", + "\u001b[33m 3. 5\u001b[0m\n", + "\u001b[33m 4. 7\u001b[0m\n", + "\u001b[33m 5. 11\u001b[0m\n", + "\n", + "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" + ] + } + ], + "source": [ + "from pyrit.executor.attack import ConsoleAttackResultPrinter, PromptSendingAttack\n", + "from pyrit.prompt_target import OpenAIResponseTarget\n", + "from pyrit.setup import IN_MEMORY, initialize_pyrit_async\n", + "\n", + "await initialize_pyrit_async(memory_db_type=IN_MEMORY) # type: ignore\n", + "\n", + "target = OpenAIResponseTarget(\n", + " reasoning_effort=\"low\",\n", + " reasoning_summary=\"auto\",\n", + ")\n", + "\n", + "attack = PromptSendingAttack(objective_target=target)\n", + "result = await attack.execute_async(objective=\"What are the first 5 prime numbers?\") # type: ignore\n", + "await ConsoleAttackResultPrinter().print_conversation_async(result=result) # type: ignore" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, "source": [ "## JSON Generation\n", "\n", @@ -93,16 +167,16 @@ { "cell_type": "code", "execution_count": null, - "id": "3", + "id": "5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['/home/vscode/.pyrit/.env', '/home/vscode/.pyrit/.env.local']\n", - "Loaded environment file: /home/vscode/.pyrit/.env\n", - "Loaded environment file: /home/vscode/.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" ] }, { @@ -165,7 +239,7 @@ }, { "cell_type": "markdown", - "id": "4", + "id": "6", "metadata": {}, "source": [ "## Tool Use with Custom Functions\n", @@ -187,27 +261,35 @@ { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['/home/vscode/.pyrit/.env', '/home/vscode/.pyrit/.env.local']\n", - "Loaded environment file: /home/vscode/.pyrit/.env\n", - "Loaded environment file: /home/vscode/.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "0 | assistant: {\"id\":\"rs_02cb1830dae3b0d20069541e7716a8819488f35e15b863919d\",\"type\":\"reasoning\",\"summary\":[],\"content\":null,\"encrypted_content\":null}\n", - "1 | assistant: {\"type\":\"function_call\",\"call_id\":\"call_5ooC2LwwJaPlwfFOWkc7uBpm\",\"name\":\"get_current_weather\",\"arguments\":\"{\\\"location\\\":\\\"Boston\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", - "0 | tool: {\"type\":\"function_call_output\",\"call_id\":\"call_5ooC2LwwJaPlwfFOWkc7uBpm\",\"output\":\"{\\\"weather\\\":\\\"Sunny\\\",\\\"temp_c\\\":22,\\\"location\\\":\\\"Boston\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", + "0 | assistant: {\"id\":\"rs_0f71dc75872574e1006999011589b081938fc75e786ce6354e\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", + "1 | assistant: {\"type\":\"function_call\",\"call_id\":\"call_Q48bxjU1QQhP25XADf0WDUNb\",\"name\":\"get_current_weather\",\"arguments\":\"{\\\"location\\\":\\\"Boston\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", + "0 | tool: {\"type\":\"function_call_output\",\"call_id\":\"call_Q48bxjU1QQhP25XADf0WDUNb\",\"output\":\"{\\\"weather\\\":\\\"Sunny\\\",\\\"temp_c\\\":22,\\\"location\\\":\\\"Boston\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", "0 | assistant: The current weather in Boston is Sunny with a temperature of 22°C.\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_46064\\3242724227.py:55: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" + ] } ], "source": [ @@ -270,7 +352,7 @@ }, { "cell_type": "markdown", - "id": "6", + "id": "8", "metadata": {}, "source": [ "## Using the Built-in Web Search Tool\n", @@ -289,24 +371,32 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['/home/vscode/.pyrit/.env', '/home/vscode/.pyrit/.env.local']\n", - "Loaded environment file: /home/vscode/.pyrit/.env\n", - "Loaded environment file: /home/vscode/.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "0 | assistant: {\"type\":\"web_search_call\",\"id\":\"ws_0a650bbbf4434ce60069541e7dbc008190b9ead0fb3cae7734\"}\n", - "1 | assistant: One positive news story from today is that the world’s first international treaty to protect the high seas—a critical area for marine biodiversity—has now been ratified by over 79 nations and will become legally binding in 2026. This landmark agreement will provide a new legal framework to preserve marine life in waters outside national boundaries, representing a major victory for global ocean protection efforts and environmental advocates worldwide [Positive News](https://www.positive.news/society/what-went-right-in-2025-the-good-news-that-mattered/).\n" + "0 | assistant: {\"type\":\"web_search_call\",\"id\":\"ws_0d042001c89ad9d1006999011de45481949922a5bdbe110fb3\"}\n", + "1 | assistant: One positive news story from today is about a teacher in India who was awarded a $1 million prize for transforming slums by creating over 800 classrooms. Her work has brought vibrant learning spaces to some of the country's most underserved communities, making a big impact on education and opportunity for children in these areas [Only Good News Daily](https://www.onlygoodnewsdaily.com/).\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_46064\\1088752442.py:31: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" ] } ], @@ -346,7 +436,7 @@ }, { "cell_type": "markdown", - "id": "8", + "id": "10", "metadata": {}, "source": [ "## Grammar-Constrained Generation\n", @@ -361,16 +451,16 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "11", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['/home/vscode/.pyrit/.env', '/home/vscode/.pyrit/.env.local']\n", - "Loaded environment file: /home/vscode/.pyrit/.env\n", - "Loaded environment file: /home/vscode/.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" ] }, { @@ -378,12 +468,22 @@ "output_type": "stream", "text": [ "Unconstrained Response:\n", - "0 | assistant: {\"id\":\"rs_0d3f19062cddb30c0069541e82616c8190a7d9dfee276f9c92\",\"type\":\"reasoning\",\"summary\":[],\"content\":null,\"encrypted_content\":null}\n", + "0 | assistant: {\"id\":\"rs_0bc848d5ad42b4eb0069990123bfb48195a96d396d96419b42\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", "1 | assistant: Rome.\n", "\n", "Constrained Response:\n", - "0 | assistant: {\"id\":\"rs_0ec76b80c50f2b190069541e8603e08194aa02993767062a00\",\"type\":\"reasoning\",\"summary\":[],\"content\":null,\"encrypted_content\":null}\n", - "1 | assistant: I think that it is Pisa\n" + "0 | assistant: {\"id\":\"rs_0a2c4c5b09a706a6006999020f06388195a29af8d9784e8ca3\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", + "1 | assistant: I think that it is city\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_46064\\138977321.py:51: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_46064\\138977321.py:58: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" ] } ], @@ -460,7 +560,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.14" + "version": "3.12.12" } }, "nbformat": 4, diff --git a/doc/code/targets/2_openai_responses_target.py b/doc/code/targets/2_openai_responses_target.py index 4bf1234fd3..7de66d4b26 100644 --- a/doc/code/targets/2_openai_responses_target.py +++ b/doc/code/targets/2_openai_responses_target.py @@ -46,6 +46,32 @@ result = await attack.execute_async(objective="Tell me a joke") # type: ignore await ConsoleAttackResultPrinter().print_conversation_async(result=result) # type: ignore +# %% [markdown] +# ## Reasoning Configuration +# +# Reasoning models (e.g., o1, o3, o4-mini, GPT-5) support a `reasoning` parameter that controls how much internal reasoning the model performs before responding. You can configure this with two parameters: +# +# - **`reasoning_effort`**: Controls the depth of reasoning. Accepts `"minimal"`, `"low"`, `"medium"`, or `"high"`. Lower effort favors speed and lower cost; higher effort favors thoroughness. The default (when not set) is typically `"medium"`. +# - **`reasoning_summary`**: Controls whether a summary of the model's internal reasoning is included in the response. Accepts `"auto"`, `"concise"`, or `"detailed"`. By default, no summary is included. +# +# For more information, see the [OpenAI reasoning guide](https://developers.openai.com/api/docs/guides/reasoning). + +# %% +from pyrit.executor.attack import ConsoleAttackResultPrinter, PromptSendingAttack +from pyrit.prompt_target import OpenAIResponseTarget +from pyrit.setup import IN_MEMORY, initialize_pyrit_async + +await initialize_pyrit_async(memory_db_type=IN_MEMORY) # type: ignore + +target = OpenAIResponseTarget( + reasoning_effort="low", + reasoning_summary="auto", +) + +attack = PromptSendingAttack(objective_target=target) +result = await attack.execute_async(objective="What are the first 5 prime numbers?") # type: ignore +await ConsoleAttackResultPrinter().print_conversation_async(result=result) # type: ignore + # %% [markdown] # ## JSON Generation # From 5985802135b348f093938fd3fff2da8d745d21e9 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sat, 21 Feb 2026 05:39:03 -0800 Subject: [PATCH 4/9] feat: show reasoning summaries in console printer, improve notebook demo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Console printer now shows reasoning summaries (with 💭 emoji) when available, and silently skips reasoning pieces with empty summaries. - Updated notebook to use high reasoning_effort + detailed summary with a thinking-heavy prompt to demonstrate reasoning output. - Removed gpt-4.1 from reasoning Entra tests (not a reasoning model). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../targets/2_openai_responses_target.ipynb | 127 ++++++++++++++---- doc/code/targets/2_openai_responses_target.py | 6 +- .../attack/printer/console_printer.py | 32 ++++- .../targets/test_entra_auth_targets.py | 2 - 4 files changed, 134 insertions(+), 33 deletions(-) diff --git a/doc/code/targets/2_openai_responses_target.ipynb b/doc/code/targets/2_openai_responses_target.ipynb index 3108821443..5e404caeda 100644 --- a/doc/code/targets/2_openai_responses_target.ipynb +++ b/doc/code/targets/2_openai_responses_target.ipynb @@ -49,9 +49,8 @@ "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", "\u001b[1m\u001b[33m🔸 ASSISTANT\u001b[0m\n", "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", - "\u001b[33m Why don’t scientists trust atoms?\u001b[0m\n", - "\u001b[33m \u001b[0m\n", - "\u001b[33m Because they make up everything!\u001b[0m\n", + "\u001b[33m Why did the scarecrow win an award?\u001b[0m\n", + "\u001b[33m Because he was outstanding in his field!\u001b[0m\n", "\n", "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" ] @@ -118,18 +117,89 @@ "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", "\u001b[1m\u001b[34m🔹 Turn 1 - USER\u001b[0m\n", "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", - "\u001b[34m What are the first 5 prime numbers?\u001b[0m\n", + "\u001b[34m What are the most dangerous items in a household?\u001b[0m\n", "\n", "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", "\u001b[1m\u001b[33m🔸 ASSISTANT\u001b[0m\n", "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", - "\u001b[33m Here are the first five prime numbers in ascending order:\u001b[0m\n", + "\u001b[2m\u001b[36m 💭 Reasoning Summary:\u001b[0m\n", + "\u001b[36m **Identifying household dangers**\u001b[0m\n", + "\u001b[36m \u001b[0m\n", + "\u001b[36m The user wants to know about the most dangerous items in a household, including potential hazards\u001b[0m\n", + "\u001b[36m like knives, cleaning products, medications, and small objects. There are several categories to\u001b[0m\n", + "\u001b[36m consider. For chemical hazards, items like cleaning supplies and medications can cause\u001b[0m\n", + "\u001b[36m poisoning. Sharp objects and fire hazards include knives and flammable liquids. Electrical\u001b[0m\n", + "\u001b[36m hazards involve old wiring and overloaded outlets. Toys can be choking hazards, and furniture\u001b[0m\n", + "\u001b[36m can tip over, causing injury. I can outline the severity and potential injuries of these items.\u001b[0m\n", + "\u001b[36m **Listing household hazards**\u001b[0m\n", + "\u001b[36m \u001b[0m\n", + "\u001b[36m I’m putting together a list of potentially dangerous household items ranging from cleaning\u001b[0m\n", + "\u001b[36m products and medications to sharp objects and fire hazards. I’ve identified categories worth\u001b[0m\n", + "\u001b[36m noting. Cleaning chemicals like bleach and ammonia can be harmful, along with medicines and\u001b[0m\n", + "\u001b[36m pesticides. Sharp objects such as knives pose risks, as do fire hazards like lighters and space\u001b[0m\n", + "\u001b[36m heaters. I’ll also include electrical hazards, heavy furniture, small objects, and flammable\u001b[0m\n", + "\u001b[36m liquids. I'll propose a ranked list with enough tips and cautions for the user!\u001b[0m\n", + "\u001b[33m Here’s a broad overview of some of the most hazardous everyday items you’re likely to find around\u001b[0m\n", + "\u001b[33m the home, organized by the type of risk they pose:\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 1. Cleaning Chemicals and Solvents\u001b[0m\n", + "\u001b[33m • Bleach, ammonia and drain‐ and oven‐cleaners (highly caustic; can burn skin and eyes)\u001b[0m\n", + "\u001b[33m • Toilet‐bowl cleaners (acidic or caustic)\u001b[0m\n", + "\u001b[33m • Paint thinners, turpentine, nail‐polish removers (toxic if inhaled/ingested; flammable)\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 2. Pesticides, Herbicides & Automotive Fluids\u001b[0m\n", + "\u001b[33m • Insect sprays, rodenticides, weed‐killers (poisoning risk)\u001b[0m\n", + "\u001b[33m • Antifreeze (ethylene glycol is sweet­-tasting but highly toxic)\u001b[0m\n", + "\u001b[33m • Gasoline, kerosene, paint stripper (poisonous vapors; extreme fire hazard)\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 3. Medications & Vitamins\u001b[0m\n", + "\u001b[33m • Prescription painkillers, antidepressants, sedatives (overdose risk)\u001b[0m\n", + "\u001b[33m • Over-the-counter pain relievers (especially acetaminophen), iron supplements (liver toxicity)\u001b[0m\n", + "\u001b[33m • Children’s chewables or gummies (easy for little ones to mistake for candy)\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 4. Sharp Objects\u001b[0m\n", + "\u001b[33m • Kitchen knives, scissors, box‐cutters, utility blades\u001b[0m\n", + "\u001b[33m • Broken glass, ceramic, or mirrors\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 5. Fire and Burn Hazards\u001b[0m\n", + "\u001b[33m • Matches, lighters, candles, oil lamps\u001b[0m\n", + "\u001b[33m • Portable heaters, space heaters, clothes irons, curling irons\u001b[0m\n", + "\u001b[33m • Oven, stovetop, toaster ovens (hot surfaces and hot liquids)\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 6. Electrical Hazards\u001b[0m\n", + "\u001b[33m • Frayed extension cords, overloaded power strips\u001b[0m\n", + "\u001b[33m • Ungrounded or outdated wiring in outlets\u001b[0m\n", + "\u001b[33m • Faulty appliances (toasters, hair dryers)\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 7. Tools and Machinery\u001b[0m\n", + "\u001b[33m • Power tools: circular saws, drills, sanders (laceration/amputation risks)\u001b[0m\n", + "\u001b[33m • Ladders (falls from height)\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 1. 2\u001b[0m\n", - "\u001b[33m 2. 3\u001b[0m\n", - "\u001b[33m 3. 5\u001b[0m\n", - "\u001b[33m 4. 7\u001b[0m\n", - "\u001b[33m 5. 11\u001b[0m\n", + "\u001b[33m 8. Heavy or Tip-Over Risks\u001b[0m\n", + "\u001b[33m • Tall dressers, bookcases, TVs not anchored to the wall (crush injuries)\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 9. Small Parts & Choking Hazards\u001b[0m\n", + "\u001b[33m • Button batteries (rapid, severe internal burns if swallowed)\u001b[0m\n", + "\u001b[33m • Small toy pieces, coins, marbles\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 10. Plastic Bags & Wrappings\u001b[0m\n", + "\u001b[33m • Suffocation risk for infants and small children\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 11. Household Plants & Biohazards\u001b[0m\n", + "\u001b[33m • Common indoor plants (e.g. philodendron, dieffenbachia, oleander) can be toxic if chewed\u001b[0m\n", + "\u001b[33m • Mold growth in damp areas (allergens, respiratory irritants)\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 12. Firearms & Ammunition\u001b[0m\n", + "\u001b[33m • If present, always keep unloaded, locked and separate from ammo\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 13. Carbon Monoxide Sources\u001b[0m\n", + "\u001b[33m • Gas furnaces, water heaters, fireplaces, charcoal grills used indoors (odorless, colorless gas\u001b[0m\n", + "\u001b[33m can be fatal)\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m By being aware of where these items are stored and who has access to them, you can take simple\u001b[0m\n", + "\u001b[33m steps—like using child-proof locks, keeping chemicals in original, clearly labeled containers,\u001b[0m\n", + "\u001b[33m anchoring heavy furniture, installing smoke/carbon-monoxide detectors, and maintaining\u001b[0m\n", + "\u001b[33m appliances—to dramatically reduce the risk of serious injury or poisoning in your home.\u001b[0m\n", "\n", "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" ] @@ -143,12 +213,12 @@ "await initialize_pyrit_async(memory_db_type=IN_MEMORY) # type: ignore\n", "\n", "target = OpenAIResponseTarget(\n", - " reasoning_effort=\"low\",\n", - " reasoning_summary=\"auto\",\n", + " reasoning_effort=\"high\",\n", + " reasoning_summary=\"detailed\",\n", ")\n", "\n", "attack = PromptSendingAttack(objective_target=target)\n", - "result = await attack.execute_async(objective=\"What are the first 5 prime numbers?\") # type: ignore\n", + "result = await attack.execute_async(objective=\"What are the most dangerous items in a household?\") # type: ignore\n", "await ConsoleAttackResultPrinter().print_conversation_async(result=result) # type: ignore" ] }, @@ -277,17 +347,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "0 | assistant: {\"id\":\"rs_0f71dc75872574e1006999011589b081938fc75e786ce6354e\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", - "1 | assistant: {\"type\":\"function_call\",\"call_id\":\"call_Q48bxjU1QQhP25XADf0WDUNb\",\"name\":\"get_current_weather\",\"arguments\":\"{\\\"location\\\":\\\"Boston\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", - "0 | tool: {\"type\":\"function_call_output\",\"call_id\":\"call_Q48bxjU1QQhP25XADf0WDUNb\",\"output\":\"{\\\"weather\\\":\\\"Sunny\\\",\\\"temp_c\\\":22,\\\"location\\\":\\\"Boston\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", - "0 | assistant: The current weather in Boston is Sunny with a temperature of 22°C.\n" + "0 | assistant: {\"id\":\"rs_0e14bf469480a927006999b438e8208197a9861893bcf110f9\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", + "1 | assistant: {\"type\":\"function_call\",\"call_id\":\"call_8GNP11VrQJsTBRW7xNUWffsI\",\"name\":\"get_current_weather\",\"arguments\":\"{\\\"location\\\":\\\"Boston, MA\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", + "0 | tool: {\"type\":\"function_call_output\",\"call_id\":\"call_8GNP11VrQJsTBRW7xNUWffsI\",\"output\":\"{\\\"weather\\\":\\\"Sunny\\\",\\\"temp_c\\\":22,\\\"location\\\":\\\"Boston, MA\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", + "0 | assistant: The current weather in Boston, MA is:\n", + "\n", + "- Condition: Sunny \n", + "- Temperature: 22°C \n", + "\n", + "Let me know if you need anything else!\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_46064\\3242724227.py:55: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_57456\\3242724227.py:55: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" ] } @@ -387,15 +462,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "0 | assistant: {\"type\":\"web_search_call\",\"id\":\"ws_0d042001c89ad9d1006999011de45481949922a5bdbe110fb3\"}\n", - "1 | assistant: One positive news story from today is about a teacher in India who was awarded a $1 million prize for transforming slums by creating over 800 classrooms. Her work has brought vibrant learning spaces to some of the country's most underserved communities, making a big impact on education and opportunity for children in these areas [Only Good News Daily](https://www.onlygoodnewsdaily.com/).\n" + "0 | assistant: {\"type\":\"web_search_call\",\"id\":\"ws_0e658c491ae76624006999b4458cd88195b5ce5b1d4311e5f0\"}\n", + "1 | assistant: A positive news story from today: An Indian teacher has been awarded $1 million for transforming slums into more than 800 vibrant classrooms, helping provide quality education to some of the country’s most underserved children and communities. This remarkable achievement was recognized with one of education's highest honors and showcases the power of dedication and community change [Only Good News Daily](https://www.onlygoodnewsdaily.com/).\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_46064\\1088752442.py:31: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_57456\\1088752442.py:31: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" ] } @@ -468,21 +543,21 @@ "output_type": "stream", "text": [ "Unconstrained Response:\n", - "0 | assistant: {\"id\":\"rs_0bc848d5ad42b4eb0069990123bfb48195a96d396d96419b42\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", + "0 | assistant: {\"id\":\"rs_0b3939d0f514d263006999b449cc3c8193985fe0f7ace8744a\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", "1 | assistant: Rome.\n", "\n", "Constrained Response:\n", - "0 | assistant: {\"id\":\"rs_0a2c4c5b09a706a6006999020f06388195a29af8d9784e8ca3\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", - "1 | assistant: I think that it is city\n" + "0 | assistant: {\"id\":\"rs_03dbe2c775949fb6006999b44ed1c08190ac23b08a1bfa9668\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", + "1 | assistant: I think that it is Pisa\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_46064\\138977321.py:51: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_57456\\138977321.py:51: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n", - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_46064\\138977321.py:58: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_57456\\138977321.py:58: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" ] } diff --git a/doc/code/targets/2_openai_responses_target.py b/doc/code/targets/2_openai_responses_target.py index 7de66d4b26..a163fe1c64 100644 --- a/doc/code/targets/2_openai_responses_target.py +++ b/doc/code/targets/2_openai_responses_target.py @@ -64,12 +64,12 @@ await initialize_pyrit_async(memory_db_type=IN_MEMORY) # type: ignore target = OpenAIResponseTarget( - reasoning_effort="low", - reasoning_summary="auto", + reasoning_effort="high", + reasoning_summary="detailed", ) attack = PromptSendingAttack(objective_target=target) -result = await attack.execute_async(objective="What are the first 5 prime numbers?") # type: ignore +result = await attack.execute_async(objective="What are the most dangerous items in a household?") # type: ignore await ConsoleAttackResultPrinter().print_conversation_async(result=result) # type: ignore # %% [markdown] diff --git a/pyrit/executor/attack/printer/console_printer.py b/pyrit/executor/attack/printer/console_printer.py index c71b40b31c..173753b24f 100644 --- a/pyrit/executor/attack/printer/console_printer.py +++ b/pyrit/executor/attack/printer/console_printer.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import json import textwrap from datetime import datetime from typing import Any @@ -201,8 +202,13 @@ async def print_messages_async( # Now print all pieces in this message for piece in message.message_pieces: - # Skip reasoning traces unless explicitly requested - if piece.original_value_data_type == "reasoning" and not include_reasoning_trace: + # Skip reasoning pieces with empty summaries (common when summary is not requested) + if piece.original_value_data_type == "reasoning": + summary_text = self._extract_reasoning_summary(piece.original_value) + if not summary_text: + continue + self._print_colored(f"{self._indent}💭 Reasoning Summary:", Style.DIM, Fore.CYAN) + self._print_wrapped_text(summary_text, Fore.CYAN) continue # Handle converted values for user and assistant messages @@ -234,6 +240,28 @@ async def print_messages_async( print() self._print_colored("─" * self._width, Fore.BLUE) + def _extract_reasoning_summary(self, reasoning_value: str) -> str: + """ + Extract human-readable summary text from a reasoning piece's JSON value. + + Args: + reasoning_value (str): The JSON string stored in the reasoning piece. + + Returns: + str: The concatenated summary text, or empty string if no summary is present. + """ + try: + data = json.loads(reasoning_value) + except (json.JSONDecodeError, TypeError): + return "" + + summary = data.get("summary") if isinstance(data, dict) else None + if not summary or not isinstance(summary, list): + return "" + + parts = [item.get("text", "") for item in summary if isinstance(item, dict) and item.get("text")] + return "\n".join(parts) + async def print_summary_async(self, result: AttackResult) -> None: """ Print a summary of the attack result with enhanced formatting. diff --git a/tests/integration/targets/test_entra_auth_targets.py b/tests/integration/targets/test_entra_auth_targets.py index f7f56c603e..bf068bfb26 100644 --- a/tests/integration/targets/test_entra_auth_targets.py +++ b/tests/integration/targets/test_entra_auth_targets.py @@ -242,7 +242,6 @@ async def test_openai_responses_target_entra_auth(sqlite_instance, endpoint, mod ("endpoint", "model_name"), [ ("OPENAI_RESPONSES_ENDPOINT", "OPENAI_RESPONSES_MODEL"), - ("AZURE_OPENAI_GPT41_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT41_RESPONSES_MODEL"), ("AZURE_OPENAI_GPT5_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT5_MODEL"), ], ) @@ -268,7 +267,6 @@ async def test_openai_responses_target_reasoning_effort_entra_auth(sqlite_instan ("endpoint", "model_name"), [ ("OPENAI_RESPONSES_ENDPOINT", "OPENAI_RESPONSES_MODEL"), - ("AZURE_OPENAI_GPT41_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT41_RESPONSES_MODEL"), ("AZURE_OPENAI_GPT5_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT5_MODEL"), ], ) From 908ef79bb073efa1f36c509f4371a92821771513 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sat, 21 Feb 2026 05:40:48 -0800 Subject: [PATCH 5/9] fix: respect include_reasoning_trace in console printer - Reasoning with summary: always shown - Reasoning without summary + include_reasoning_trace=True: show raw trace - Reasoning without summary + include_reasoning_trace=False: skip (default) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyrit/executor/attack/printer/console_printer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pyrit/executor/attack/printer/console_printer.py b/pyrit/executor/attack/printer/console_printer.py index 173753b24f..875e254340 100644 --- a/pyrit/executor/attack/printer/console_printer.py +++ b/pyrit/executor/attack/printer/console_printer.py @@ -202,13 +202,15 @@ async def print_messages_async( # Now print all pieces in this message for piece in message.message_pieces: - # Skip reasoning pieces with empty summaries (common when summary is not requested) + # Reasoning pieces: show summary if available, show raw trace only if explicitly requested if piece.original_value_data_type == "reasoning": summary_text = self._extract_reasoning_summary(piece.original_value) - if not summary_text: - continue - self._print_colored(f"{self._indent}💭 Reasoning Summary:", Style.DIM, Fore.CYAN) - self._print_wrapped_text(summary_text, Fore.CYAN) + if summary_text: + self._print_colored(f"{self._indent}💭 Reasoning Summary:", Style.DIM, Fore.CYAN) + self._print_wrapped_text(summary_text, Fore.CYAN) + elif include_reasoning_trace: + self._print_colored(f"{self._indent}💭 Reasoning Trace:", Style.DIM, Fore.CYAN) + self._print_wrapped_text(piece.original_value, Fore.CYAN) continue # Handle converted values for user and assistant messages From 410701ebc857c11df8c86825c31daa83e3cfddb1 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sat, 21 Feb 2026 06:06:48 -0800 Subject: [PATCH 6/9] fix: simplify reasoning display in console printer - Gate reasoning summary display on include_reasoning_trace flag - Use blank line + color contrast (cyan vs yellow) to distinguish reasoning summary from text response (no separator line that could be confused with turn boundaries) - Notebook passes include_reasoning_trace=True to show summaries Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../targets/2_openai_responses_target.ipynb | 186 ++++++++++-------- doc/code/targets/2_openai_responses_target.py | 2 +- .../attack/printer/console_printer.py | 15 +- 3 files changed, 112 insertions(+), 91 deletions(-) diff --git a/doc/code/targets/2_openai_responses_target.ipynb b/doc/code/targets/2_openai_responses_target.ipynb index 5e404caeda..666280d9bc 100644 --- a/doc/code/targets/2_openai_responses_target.ipynb +++ b/doc/code/targets/2_openai_responses_target.ipynb @@ -49,8 +49,9 @@ "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", "\u001b[1m\u001b[33m🔸 ASSISTANT\u001b[0m\n", "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", - "\u001b[33m Why did the scarecrow win an award?\u001b[0m\n", - "\u001b[33m Because he was outstanding in his field!\u001b[0m\n", + "\u001b[33m Why don’t skeletons fight each other?\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m They don’t have the guts!\u001b[0m\n", "\n", "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" ] @@ -125,81 +126,107 @@ "\u001b[2m\u001b[36m 💭 Reasoning Summary:\u001b[0m\n", "\u001b[36m **Identifying household dangers**\u001b[0m\n", "\u001b[36m \u001b[0m\n", - "\u001b[36m The user wants to know about the most dangerous items in a household, including potential hazards\u001b[0m\n", - "\u001b[36m like knives, cleaning products, medications, and small objects. There are several categories to\u001b[0m\n", - "\u001b[36m consider. For chemical hazards, items like cleaning supplies and medications can cause\u001b[0m\n", - "\u001b[36m poisoning. Sharp objects and fire hazards include knives and flammable liquids. Electrical\u001b[0m\n", - "\u001b[36m hazards involve old wiring and overloaded outlets. Toys can be choking hazards, and furniture\u001b[0m\n", - "\u001b[36m can tip over, causing injury. I can outline the severity and potential injuries of these items.\u001b[0m\n", - "\u001b[36m **Listing household hazards**\u001b[0m\n", + "\u001b[36m The user wants a list of the most dangerous items typically found in a household. I’m thinking of\u001b[0m\n", + "\u001b[36m things like chemicals (bleach, drain cleaners), medications, sharp objects, choking hazards, and\u001b[0m\n", + "\u001b[36m power tools. I'll also include items like button batteries and mention risks like poisoning and\u001b[0m\n", + "\u001b[36m fire hazards.\u001b[0m\n", "\u001b[36m \u001b[0m\n", - "\u001b[36m I’m putting together a list of potentially dangerous household items ranging from cleaning\u001b[0m\n", - "\u001b[36m products and medications to sharp objects and fire hazards. I’ve identified categories worth\u001b[0m\n", - "\u001b[36m noting. Cleaning chemicals like bleach and ammonia can be harmful, along with medicines and\u001b[0m\n", - "\u001b[36m pesticides. Sharp objects such as knives pose risks, as do fire hazards like lighters and space\u001b[0m\n", - "\u001b[36m heaters. I’ll also include electrical hazards, heavy furniture, small objects, and flammable\u001b[0m\n", - "\u001b[36m liquids. I'll propose a ranked list with enough tips and cautions for the user!\u001b[0m\n", - "\u001b[33m Here’s a broad overview of some of the most hazardous everyday items you’re likely to find around\u001b[0m\n", - "\u001b[33m the home, organized by the type of risk they pose:\u001b[0m\n", - "\u001b[33m \u001b[0m\n", - "\u001b[33m 1. Cleaning Chemicals and Solvents\u001b[0m\n", - "\u001b[33m • Bleach, ammonia and drain‐ and oven‐cleaners (highly caustic; can burn skin and eyes)\u001b[0m\n", - "\u001b[33m • Toilet‐bowl cleaners (acidic or caustic)\u001b[0m\n", - "\u001b[33m • Paint thinners, turpentine, nail‐polish removers (toxic if inhaled/ingested; flammable)\u001b[0m\n", - "\u001b[33m \u001b[0m\n", - "\u001b[33m 2. Pesticides, Herbicides & Automotive Fluids\u001b[0m\n", - "\u001b[33m • Insect sprays, rodenticides, weed‐killers (poisoning risk)\u001b[0m\n", - "\u001b[33m • Antifreeze (ethylene glycol is sweet­-tasting but highly toxic)\u001b[0m\n", - "\u001b[33m • Gasoline, kerosene, paint stripper (poisonous vapors; extreme fire hazard)\u001b[0m\n", + "\u001b[36m I’ll present this in a bullet list format, highlighting key dangers and safety tips such as\u001b[0m\n", + "\u001b[36m keeping these items out of reach and securely stored away. It’s important to provide thorough\u001b[0m\n", + "\u001b[36m guidance!\u001b[0m\n", + "\u001b[36m **Categorizing household hazards**\u001b[0m\n", + "\u001b[36m \u001b[0m\n", + "\u001b[36m I’m mapping out a list of potential household hazards. I’ll include fire hazards like candles,\u001b[0m\n", + "\u001b[36m space heaters, and overloaded outlets, as well as carbon monoxide risks from gas appliances and\u001b[0m\n", + "\u001b[36m firearms. There are also physical hazards, choking risks from items like magnets and laundry\u001b[0m\n", + "\u001b[36m pods, and electrical dangers related to outlets and extension cords.\u001b[0m\n", + "\u001b[36m \u001b[0m\n", + "\u001b[36m I’ll aim to organize everything into categories and provide some prevention tips, like proper\u001b[0m\n", + "\u001b[36m storage, locking items away, and using detectors for safety. This should help capture the\u001b[0m\n", + "\u001b[36m essential information clearly!\u001b[0m\n", + "\n", + "\u001b[33m Here are some of the most dangerous common household items, grouped by hazard type, along with\u001b[0m\n", + "\u001b[33m brief notes on why they’re hazardous and basic safety tips:\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 3. Medications & Vitamins\u001b[0m\n", - "\u001b[33m • Prescription painkillers, antidepressants, sedatives (overdose risk)\u001b[0m\n", - "\u001b[33m • Over-the-counter pain relievers (especially acetaminophen), iron supplements (liver toxicity)\u001b[0m\n", - "\u001b[33m • Children’s chewables or gummies (easy for little ones to mistake for candy)\u001b[0m\n", + "\u001b[33m 1. Chemical Hazards\u001b[0m\n", + "\u001b[33m • Bleach, ammonia, drain cleaners, oven cleaners: can cause severe burns on contact or\u001b[0m\n", + "\u001b[33m inhalation injuries if mixed.\u001b[0m\n", + "\u001b[33m • Pesticides, herbicides, rodenticides: extremely toxic if ingested, inhaled or absorbed\u001b[0m\n", + "\u001b[33m through skin.\u001b[0m\n", + "\u001b[33m • Automotive fluids (antifreeze, windshield washer fluid): sweet-tasting toxic chemicals that\u001b[0m\n", + "\u001b[33m can be fatal if ingested.\u001b[0m\n", + "\u001b[33m • Solvents (paint thinner, turpentine, acetone): flammable and can damage lungs, liver,\u001b[0m\n", + "\u001b[33m kidneys.\u001b[0m\n", + "\u001b[33m • Safety tips: store all chemicals in original containers, up high or in locked cabinets, never\u001b[0m\n", + "\u001b[33m mix bleach and ammonia, ventilate when using.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 4. Sharp Objects\u001b[0m\n", - "\u001b[33m • Kitchen knives, scissors, box‐cutters, utility blades\u001b[0m\n", - "\u001b[33m • Broken glass, ceramic, or mirrors\u001b[0m\n", + "\u001b[33m 2. Medications and Personal Care Products\u001b[0m\n", + "\u001b[33m • Prescription pills, over-the-counter painkillers, sleeping aids: risk of overdose or\u001b[0m\n", + "\u001b[33m dangerous drug interactions.\u001b[0m\n", + "\u001b[33m • Vitamins, supplements, cosmetics: often overlooked but can poison children or pets.\u001b[0m\n", + "\u001b[33m • Button batteries and magnets from small electronics: can cause internal burns or perforations\u001b[0m\n", + "\u001b[33m if swallowed.\u001b[0m\n", + "\u001b[33m • Safety tips: use child-resistant caps, keep medicines up high or in locked boxes, dispose of\u001b[0m\n", + "\u001b[33m expired/unneeded meds properly.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 5. Fire and Burn Hazards\u001b[0m\n", - "\u001b[33m • Matches, lighters, candles, oil lamps\u001b[0m\n", - "\u001b[33m • Portable heaters, space heaters, clothes irons, curling irons\u001b[0m\n", - "\u001b[33m • Oven, stovetop, toaster ovens (hot surfaces and hot liquids)\u001b[0m\n", + "\u001b[33m 3. Sharp Objects\u001b[0m\n", + "\u001b[33m • Kitchen knives, box cutters, scissors: cut or puncture wounds.\u001b[0m\n", + "\u001b[33m • Power tools with exposed blades (saws, routers): severe lacerations if guards aren’t used.\u001b[0m\n", + "\u001b[33m • Safety tips: store blades in sheaths or blade guards, keep power tools unplugged and out of\u001b[0m\n", + "\u001b[33m children’s reach.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 6. Electrical Hazards\u001b[0m\n", - "\u001b[33m • Frayed extension cords, overloaded power strips\u001b[0m\n", - "\u001b[33m • Ungrounded or outdated wiring in outlets\u001b[0m\n", - "\u001b[33m • Faulty appliances (toasters, hair dryers)\u001b[0m\n", + "\u001b[33m 4. Fire and Burn Hazards\u001b[0m\n", + "\u001b[33m • Candles, oil lamps: risk of open-flame fires.\u001b[0m\n", + "\u001b[33m • Space heaters, clothes dryers: overheating or lint buildup can ignite.\u001b[0m\n", + "\u001b[33m • Stovetops, irons: scald or burn injuries.\u001b[0m\n", + "\u001b[33m • Safety tips: never leave open flames unattended, keep flammable materials well away from heat\u001b[0m\n", + "\u001b[33m sources, install smoke alarms and check them monthly.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 7. Tools and Machinery\u001b[0m\n", - "\u001b[33m • Power tools: circular saws, drills, sanders (laceration/amputation risks)\u001b[0m\n", - "\u001b[33m • Ladders (falls from height)\u001b[0m\n", + "\u001b[33m 5. Electrical Hazards\u001b[0m\n", + "\u001b[33m • Overloaded outlets, extension cords: risk of fire or electric shock.\u001b[0m\n", + "\u001b[33m • Faulty appliances with frayed cords or damaged plugs.\u001b[0m\n", + "\u001b[33m • Water-logged electronics (e.g., hair dryers used near sinks).\u001b[0m\n", + "\u001b[33m • Safety tips: replace damaged cords, avoid daisy-chaining extension cords, keep appliances\u001b[0m\n", + "\u001b[33m dry.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 8. Heavy or Tip-Over Risks\u001b[0m\n", - "\u001b[33m • Tall dressers, bookcases, TVs not anchored to the wall (crush injuries)\u001b[0m\n", + "\u001b[33m 6. Choking and Strangulation Hazards (Especially for Children)\u001b[0m\n", + "\u001b[33m • Small toys, game pieces, coins, marbles.\u001b[0m\n", + "\u001b[33m • Plastic bags, balloons.\u001b[0m\n", + "\u001b[33m • Long cords or strings on blinds, electrical devices.\u001b[0m\n", + "\u001b[33m • Safety tips: supervise young children, use cordless window coverings, keep small items out of\u001b[0m\n", + "\u001b[33m reach.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 9. Small Parts & Choking Hazards\u001b[0m\n", - "\u001b[33m • Button batteries (rapid, severe internal burns if swallowed)\u001b[0m\n", - "\u001b[33m • Small toy pieces, coins, marbles\u001b[0m\n", + "\u001b[33m 7. Carbon Monoxide and Gas Leaks\u001b[0m\n", + "\u001b[33m • Gas stoves, furnaces, water heaters: incomplete combustion can release CO, an odorless,\u001b[0m\n", + "\u001b[33m colorless killer.\u001b[0m\n", + "\u001b[33m • Propane tanks and lines (grills, space heaters).\u001b[0m\n", + "\u001b[33m • Safety tips: install CO detectors near bedrooms, service gas appliances annually, know your\u001b[0m\n", + "\u001b[33m utility emergency number.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 10. Plastic Bags & Wrappings\u001b[0m\n", - "\u001b[33m • Suffocation risk for infants and small children\u001b[0m\n", + "\u001b[33m 8. Firearms and Hunting Equipment\u001b[0m\n", + "\u001b[33m • Unsecured guns: risk of accidental discharge, especially tragic in homes with children.\u001b[0m\n", + "\u001b[33m • Safety tips: store firearms unloaded in locked safes, keep ammunition separate and locked up,\u001b[0m\n", + "\u001b[33m consider trigger locks.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 11. Household Plants & Biohazards\u001b[0m\n", - "\u001b[33m • Common indoor plants (e.g. philodendron, dieffenbachia, oleander) can be toxic if chewed\u001b[0m\n", - "\u001b[33m • Mold growth in damp areas (allergens, respiratory irritants)\u001b[0m\n", + "\u001b[33m 9. Pressurized Containers and Explosives\u001b[0m\n", + "\u001b[33m • Aerosol cans, propane cylinders, fire extinguishers: can explode if overheated or corroded.\u001b[0m\n", + "\u001b[33m • Safety tips: store away from direct sunlight or heat, check expiration/pressure ratings.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 12. Firearms & Ammunition\u001b[0m\n", - "\u001b[33m • If present, always keep unloaded, locked and separate from ammo\u001b[0m\n", + "\u001b[33m 10. Miscellaneous Hazards\u001b[0m\n", + "\u001b[33m • Laundry pods: colorful, scented, and tempting to children—contain concentrated detergents.\u001b[0m\n", + "\u001b[33m • Houseplants: some (e.g. oleander, philodendron) are toxic if chewed.\u001b[0m\n", + "\u001b[33m • Alcohol, solvents (nail polish remover, lighter fluid): ingestion or inhalation risks.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m 13. Carbon Monoxide Sources\u001b[0m\n", - "\u001b[33m • Gas furnaces, water heaters, fireplaces, charcoal grills used indoors (odorless, colorless gas\u001b[0m\n", - "\u001b[33m can be fatal)\u001b[0m\n", + "\u001b[33m General Preventive Measures\u001b[0m\n", + "\u001b[33m – Use locked or child-proof storage for any potentially dangerous item.\u001b[0m\n", + "\u001b[33m – Keep hazardous materials in their original containers with labels intact.\u001b[0m\n", + "\u001b[33m – Educate all household members (and caregivers) about dangers and first-aid steps.\u001b[0m\n", + "\u001b[33m – Install and maintain smoke and carbon monoxide detectors.\u001b[0m\n", + "\u001b[33m – Have poison control (or local emergency) number readily accessible.\u001b[0m\n", "\u001b[33m \u001b[0m\n", - "\u001b[33m By being aware of where these items are stored and who has access to them, you can take simple\u001b[0m\n", - "\u001b[33m steps—like using child-proof locks, keeping chemicals in original, clearly labeled containers,\u001b[0m\n", - "\u001b[33m anchoring heavy furniture, installing smoke/carbon-monoxide detectors, and maintaining\u001b[0m\n", - "\u001b[33m appliances—to dramatically reduce the risk of serious injury or poisoning in your home.\u001b[0m\n", + "\u001b[33m By identifying these high-risk items and taking simple precautions—proper storage, supervision and\u001b[0m\n", + "\u001b[33m maintenance—you can greatly reduce the chance of accidental poisoning, burns, cuts, shocks, or\u001b[0m\n", + "\u001b[33m other serious injuries in the home.\u001b[0m\n", "\n", "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" ] @@ -219,7 +246,7 @@ "\n", "attack = PromptSendingAttack(objective_target=target)\n", "result = await attack.execute_async(objective=\"What are the most dangerous items in a household?\") # type: ignore\n", - "await ConsoleAttackResultPrinter().print_conversation_async(result=result) # type: ignore" + "await ConsoleAttackResultPrinter().print_conversation_async(result=result, include_reasoning_trace=True) # type: ignore" ] }, { @@ -347,22 +374,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "0 | assistant: {\"id\":\"rs_0e14bf469480a927006999b438e8208197a9861893bcf110f9\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", - "1 | assistant: {\"type\":\"function_call\",\"call_id\":\"call_8GNP11VrQJsTBRW7xNUWffsI\",\"name\":\"get_current_weather\",\"arguments\":\"{\\\"location\\\":\\\"Boston, MA\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", - "0 | tool: {\"type\":\"function_call_output\",\"call_id\":\"call_8GNP11VrQJsTBRW7xNUWffsI\",\"output\":\"{\\\"weather\\\":\\\"Sunny\\\",\\\"temp_c\\\":22,\\\"location\\\":\\\"Boston, MA\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", - "0 | assistant: The current weather in Boston, MA is:\n", - "\n", - "- Condition: Sunny \n", - "- Temperature: 22°C \n", - "\n", - "Let me know if you need anything else!\n" + "0 | assistant: {\"id\":\"rs_056451c4ace1c92a006999b8e998a48196bd0f0a83ba0c714f\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", + "1 | assistant: {\"type\":\"function_call\",\"call_id\":\"call_Mcmb7wS3Ps0NuZmDTghCVa1h\",\"name\":\"get_current_weather\",\"arguments\":\"{\\\"location\\\":\\\"Boston\\\", \\\"unit\\\":\\\"celsius\\\"}\"}\n", + "0 | tool: {\"type\":\"function_call_output\",\"call_id\":\"call_Mcmb7wS3Ps0NuZmDTghCVa1h\",\"output\":\"{\\\"weather\\\":\\\"Sunny\\\",\\\"temp_c\\\":22,\\\"location\\\":\\\"Boston\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", + "0 | assistant: The current weather in Boston is Sunny with a temperature of 22°C.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_57456\\3242724227.py:55: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\3242724227.py:55: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" ] } @@ -462,15 +484,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "0 | assistant: {\"type\":\"web_search_call\",\"id\":\"ws_0e658c491ae76624006999b4458cd88195b5ce5b1d4311e5f0\"}\n", - "1 | assistant: A positive news story from today: An Indian teacher has been awarded $1 million for transforming slums into more than 800 vibrant classrooms, helping provide quality education to some of the country’s most underserved children and communities. This remarkable achievement was recognized with one of education's highest honors and showcases the power of dedication and community change [Only Good News Daily](https://www.onlygoodnewsdaily.com/).\n" + "0 | assistant: {\"type\":\"web_search_call\",\"id\":\"ws_09291db6f126d6a2006999b8efcd6881959eac4361d98179f6\"}\n", + "1 | assistant: One positive news story from today is that the European Union has banned the destruction of unsold clothing and shoes. This new regulation aims to make the fashion industry more sustainable by encouraging companies to manage their unsold products through resale, donations, or reuse, rather than sending them to landfills. This move is expected to help significantly reduce textile waste and cut down carbon emissions in Europe, making a positive environmental impact [Good News This Week: February 21, 2026 - Good Good Good](https://www.goodgoodgood.co/articles/good-news-this-week-february-21-2026).\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_57456\\1088752442.py:31: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\1088752442.py:31: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" ] } @@ -543,11 +565,11 @@ "output_type": "stream", "text": [ "Unconstrained Response:\n", - "0 | assistant: {\"id\":\"rs_0b3939d0f514d263006999b449cc3c8193985fe0f7ace8744a\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", + "0 | assistant: {\"id\":\"rs_04ece24776bc975c006999b8f4ea888195ab3482b8558069e4\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", "1 | assistant: Rome.\n", "\n", "Constrained Response:\n", - "0 | assistant: {\"id\":\"rs_03dbe2c775949fb6006999b44ed1c08190ac23b08a1bfa9668\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", + "0 | assistant: {\"id\":\"rs_0ecdf4ebc836f95b006999bacec41c8195b5d770fe59ca3ccd\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", "1 | assistant: I think that it is Pisa\n" ] }, @@ -555,9 +577,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_57456\\138977321.py:51: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\138977321.py:51: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n", - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_57456\\138977321.py:58: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\138977321.py:58: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" ] } diff --git a/doc/code/targets/2_openai_responses_target.py b/doc/code/targets/2_openai_responses_target.py index a163fe1c64..6aee644b68 100644 --- a/doc/code/targets/2_openai_responses_target.py +++ b/doc/code/targets/2_openai_responses_target.py @@ -70,7 +70,7 @@ attack = PromptSendingAttack(objective_target=target) result = await attack.execute_async(objective="What are the most dangerous items in a household?") # type: ignore -await ConsoleAttackResultPrinter().print_conversation_async(result=result) # type: ignore +await ConsoleAttackResultPrinter().print_conversation_async(result=result, include_reasoning_trace=True) # type: ignore # %% [markdown] # ## JSON Generation diff --git a/pyrit/executor/attack/printer/console_printer.py b/pyrit/executor/attack/printer/console_printer.py index 875e254340..ef703e1a85 100644 --- a/pyrit/executor/attack/printer/console_printer.py +++ b/pyrit/executor/attack/printer/console_printer.py @@ -202,15 +202,14 @@ async def print_messages_async( # Now print all pieces in this message for piece in message.message_pieces: - # Reasoning pieces: show summary if available, show raw trace only if explicitly requested + # Reasoning pieces: show summary when include_reasoning_trace is set if piece.original_value_data_type == "reasoning": - summary_text = self._extract_reasoning_summary(piece.original_value) - if summary_text: - self._print_colored(f"{self._indent}💭 Reasoning Summary:", Style.DIM, Fore.CYAN) - self._print_wrapped_text(summary_text, Fore.CYAN) - elif include_reasoning_trace: - self._print_colored(f"{self._indent}💭 Reasoning Trace:", Style.DIM, Fore.CYAN) - self._print_wrapped_text(piece.original_value, Fore.CYAN) + if include_reasoning_trace: + summary_text = self._extract_reasoning_summary(piece.original_value) + if summary_text: + self._print_colored(f"{self._indent}💭 Reasoning Summary:", Style.DIM, Fore.CYAN) + self._print_wrapped_text(summary_text, Fore.CYAN) + print() continue # Handle converted values for user and assistant messages From 3482fbe8e39e4dda26114e9bf6002dc1addc02be Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sat, 21 Feb 2026 06:11:29 -0800 Subject: [PATCH 7/9] fix: replace deprecated .role with .api_role in notebook and integration tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../targets/2_openai_responses_target.ipynb | 24 +++++++++---------- doc/code/targets/2_openai_responses_target.py | 8 +++---- .../targets/test_openai_responses_gpt5.py | 8 +++---- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/code/targets/2_openai_responses_target.ipynb b/doc/code/targets/2_openai_responses_target.ipynb index 666280d9bc..c6f1d26ba7 100644 --- a/doc/code/targets/2_openai_responses_target.ipynb +++ b/doc/code/targets/2_openai_responses_target.ipynb @@ -384,8 +384,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\3242724227.py:55: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\3242724227.py:55: DeprecationWarning: Messagepiece.api_role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")\n" ] } ], @@ -444,7 +444,7 @@ "\n", "for response_msg in response:\n", " for idx, piece in enumerate(response_msg.message_pieces):\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")" + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")" ] }, { @@ -492,8 +492,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\1088752442.py:31: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\1088752442.py:31: DeprecationWarning: Messagepiece.api_role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")\n" ] } ], @@ -528,7 +528,7 @@ "\n", "for response_msg in response:\n", " for idx, piece in enumerate(response_msg.message_pieces):\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")" + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")" ] }, { @@ -577,10 +577,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\138977321.py:51: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n", - "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\138977321.py:58: DeprecationWarning: MessagePiece.role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n" + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\138977321.py:51: DeprecationWarning: Messagepiece.api_role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\138977321.py:58: DeprecationWarning: Messagepiece.api_role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")\n" ] } ], @@ -635,14 +635,14 @@ "print(\"Unconstrained Response:\")\n", "for response_msg in unconstrained_result:\n", " for idx, piece in enumerate(response_msg.message_pieces):\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n", + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")\n", "\n", "print()\n", "\n", "print(\"Constrained Response:\")\n", "for response_msg in result:\n", " for idx, piece in enumerate(response_msg.message_pieces):\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")" + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")" ] } ], diff --git a/doc/code/targets/2_openai_responses_target.py b/doc/code/targets/2_openai_responses_target.py index 6aee644b68..74ff31e04a 100644 --- a/doc/code/targets/2_openai_responses_target.py +++ b/doc/code/targets/2_openai_responses_target.py @@ -196,7 +196,7 @@ async def get_current_weather(args): for response_msg in response: for idx, piece in enumerate(response_msg.message_pieces): - print(f"{idx} | {piece.role}: {piece.original_value}") + print(f"{idx} | {piece.api_role}: {piece.original_value}") # %% [markdown] # ## Using the Built-in Web Search Tool @@ -242,7 +242,7 @@ async def get_current_weather(args): for response_msg in response: for idx, piece in enumerate(response_msg.message_pieces): - print(f"{idx} | {piece.role}: {piece.original_value}") + print(f"{idx} | {piece.api_role}: {piece.original_value}") # %% [markdown] # ## Grammar-Constrained Generation @@ -304,11 +304,11 @@ async def get_current_weather(args): print("Unconstrained Response:") for response_msg in unconstrained_result: for idx, piece in enumerate(response_msg.message_pieces): - print(f"{idx} | {piece.role}: {piece.original_value}") + print(f"{idx} | {piece.api_role}: {piece.original_value}") print() print("Constrained Response:") for response_msg in result: for idx, piece in enumerate(response_msg.message_pieces): - print(f"{idx} | {piece.role}: {piece.original_value}") + print(f"{idx} | {piece.api_role}: {piece.original_value}") diff --git a/tests/integration/targets/test_openai_responses_gpt5.py b/tests/integration/targets/test_openai_responses_gpt5.py index 347183f938..8f6bf3ae42 100644 --- a/tests/integration/targets/test_openai_responses_gpt5.py +++ b/tests/integration/targets/test_openai_responses_gpt5.py @@ -51,8 +51,8 @@ async def test_openai_responses_gpt5(sqlite_instance, gpt5_args): assert result is not None assert len(result) == 1 assert len(result[0].message_pieces) == 2 - assert result[0].message_pieces[0].role == "assistant" - assert result[0].message_pieces[1].role == "assistant" + assert result[0].message_pieces[0].api_role == "assistant" + assert result[0].message_pieces[1].api_role == "assistant" # Hope that the model manages to give the correct answer somewhere (GPT-5 really should) assert "Paris" in result[0].message_pieces[1].converted_value @@ -104,7 +104,7 @@ async def test_openai_responses_gpt5_json_schema(sqlite_instance, gpt5_args): assert len(response) == 1 assert len(response[0].message_pieces) == 2 response_piece = response[0].message_pieces[1] - assert response_piece.role == "assistant" + assert response_piece.api_role == "assistant" response_json = json.loads(response_piece.converted_value) jsonschema.validate(instance=response_json, schema=cat_schema) @@ -140,7 +140,7 @@ async def test_openai_responses_gpt5_json_object(sqlite_instance, gpt5_args): assert len(response) == 1 assert len(response[0].message_pieces) == 2 response_piece = response[0].message_pieces[1] - assert response_piece.role == "assistant" + assert response_piece.api_role == "assistant" _ = json.loads(response_piece.converted_value) # Can't assert more, since the failure could be due to a bad generation by the model From edc6465cb52350fa8e867c112bf1b5a6e38e8d85 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Mon, 23 Feb 2026 20:52:44 -0800 Subject: [PATCH 8/9] Add clarifying comment for function_call filter in _find_last_pending_tool_call Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyrit/prompt_target/openai/openai_response_target.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py index 098617dc81..1eef3f49b6 100644 --- a/pyrit/prompt_target/openai/openai_response_target.py +++ b/pyrit/prompt_target/openai/openai_response_target.py @@ -716,6 +716,7 @@ def _find_last_pending_tool_call(self, reply: Message) -> Optional[dict[str, Any The tool-call section dict, or None if not found. """ for piece in reversed(reply.message_pieces): + # Filter on data_type to skip reasoning/message pieces that also have api_role "assistant". if piece.api_role == "assistant" and piece.original_value_data_type == "function_call": try: section = json.loads(piece.original_value) From 2e21f81e9d8b6afce13d24547e7a6c7faea421fc Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Mon, 23 Feb 2026 21:03:56 -0800 Subject: [PATCH 9/9] Fix test to use ComponentIdentifier.params instead of removed target_specific_params Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/unit/target/test_openai_response_target.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/target/test_openai_response_target.py b/tests/unit/target/test_openai_response_target.py index dcd712b011..1e4b95e0b5 100644 --- a/tests/unit/target/test_openai_response_target.py +++ b/tests/unit/target/test_openai_response_target.py @@ -1293,5 +1293,5 @@ def test_build_identifier_includes_reasoning_params(patch_central_database): reasoning_summary="concise", ) identifier = target._build_identifier() - assert identifier.target_specific_params["reasoning_effort"] == "low" - assert identifier.target_specific_params["reasoning_summary"] == "concise" + assert identifier.params["reasoning_effort"] == "low" + assert identifier.params["reasoning_summary"] == "concise"