diff --git a/doc/code/targets/2_openai_responses_target.ipynb b/doc/code/targets/2_openai_responses_target.ipynb index ff7e36469..c6f1d26ba 100644 --- a/doc/code/targets/2_openai_responses_target.ipynb +++ b/doc/code/targets/2_openai_responses_target.ipynb @@ -31,9 +31,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['/home/vscode/.pyrit/.env', '/home/vscode/.pyrit/.env.local']\n", - "Loaded environment file: /home/vscode/.pyrit/.env\n", - "Loaded environment file: /home/vscode/.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" ] }, { @@ -49,7 +49,9 @@ "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", "\u001b[1m\u001b[33m🔸 ASSISTANT\u001b[0m\n", "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", - "\u001b[33m Why did the coffee file a police report? It got mugged!\u001b[0m\n", + "\u001b[33m Why don’t skeletons fight each other?\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m They don’t have the guts!\u001b[0m\n", "\n", "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" ] @@ -82,6 +84,175 @@ "cell_type": "markdown", "id": "2", "metadata": {}, + "source": [ + "## Reasoning Configuration\n", + "\n", + "Reasoning models (e.g., o1, o3, o4-mini, GPT-5) support a `reasoning` parameter that controls how much internal reasoning the model performs before responding. You can configure this with two parameters:\n", + "\n", + "- **`reasoning_effort`**: Controls the depth of reasoning. Accepts `\"minimal\"`, `\"low\"`, `\"medium\"`, or `\"high\"`. Lower effort favors speed and lower cost; higher effort favors thoroughness. The default (when not set) is typically `\"medium\"`.\n", + "- **`reasoning_summary`**: Controls whether a summary of the model's internal reasoning is included in the response. Accepts `\"auto\"`, `\"concise\"`, or `\"detailed\"`. By default, no summary is included.\n", + "\n", + "For more information, see the [OpenAI reasoning guide](https://developers.openai.com/api/docs/guides/reasoning)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[1m\u001b[34m🔹 Turn 1 - USER\u001b[0m\n", + "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[34m What are the most dangerous items in a household?\u001b[0m\n", + "\n", + "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[1m\u001b[33m🔸 ASSISTANT\u001b[0m\n", + "\u001b[33m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[2m\u001b[36m 💭 Reasoning Summary:\u001b[0m\n", + "\u001b[36m **Identifying household dangers**\u001b[0m\n", + "\u001b[36m \u001b[0m\n", + "\u001b[36m The user wants a list of the most dangerous items typically found in a household. I’m thinking of\u001b[0m\n", + "\u001b[36m things like chemicals (bleach, drain cleaners), medications, sharp objects, choking hazards, and\u001b[0m\n", + "\u001b[36m power tools. I'll also include items like button batteries and mention risks like poisoning and\u001b[0m\n", + "\u001b[36m fire hazards.\u001b[0m\n", + "\u001b[36m \u001b[0m\n", + "\u001b[36m I’ll present this in a bullet list format, highlighting key dangers and safety tips such as\u001b[0m\n", + "\u001b[36m keeping these items out of reach and securely stored away. It’s important to provide thorough\u001b[0m\n", + "\u001b[36m guidance!\u001b[0m\n", + "\u001b[36m **Categorizing household hazards**\u001b[0m\n", + "\u001b[36m \u001b[0m\n", + "\u001b[36m I’m mapping out a list of potential household hazards. I’ll include fire hazards like candles,\u001b[0m\n", + "\u001b[36m space heaters, and overloaded outlets, as well as carbon monoxide risks from gas appliances and\u001b[0m\n", + "\u001b[36m firearms. There are also physical hazards, choking risks from items like magnets and laundry\u001b[0m\n", + "\u001b[36m pods, and electrical dangers related to outlets and extension cords.\u001b[0m\n", + "\u001b[36m \u001b[0m\n", + "\u001b[36m I’ll aim to organize everything into categories and provide some prevention tips, like proper\u001b[0m\n", + "\u001b[36m storage, locking items away, and using detectors for safety. This should help capture the\u001b[0m\n", + "\u001b[36m essential information clearly!\u001b[0m\n", + "\n", + "\u001b[33m Here are some of the most dangerous common household items, grouped by hazard type, along with\u001b[0m\n", + "\u001b[33m brief notes on why they’re hazardous and basic safety tips:\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 1. Chemical Hazards\u001b[0m\n", + "\u001b[33m • Bleach, ammonia, drain cleaners, oven cleaners: can cause severe burns on contact or\u001b[0m\n", + "\u001b[33m inhalation injuries if mixed.\u001b[0m\n", + "\u001b[33m • Pesticides, herbicides, rodenticides: extremely toxic if ingested, inhaled or absorbed\u001b[0m\n", + "\u001b[33m through skin.\u001b[0m\n", + "\u001b[33m • Automotive fluids (antifreeze, windshield washer fluid): sweet-tasting toxic chemicals that\u001b[0m\n", + "\u001b[33m can be fatal if ingested.\u001b[0m\n", + "\u001b[33m • Solvents (paint thinner, turpentine, acetone): flammable and can damage lungs, liver,\u001b[0m\n", + "\u001b[33m kidneys.\u001b[0m\n", + "\u001b[33m • Safety tips: store all chemicals in original containers, up high or in locked cabinets, never\u001b[0m\n", + "\u001b[33m mix bleach and ammonia, ventilate when using.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 2. Medications and Personal Care Products\u001b[0m\n", + "\u001b[33m • Prescription pills, over-the-counter painkillers, sleeping aids: risk of overdose or\u001b[0m\n", + "\u001b[33m dangerous drug interactions.\u001b[0m\n", + "\u001b[33m • Vitamins, supplements, cosmetics: often overlooked but can poison children or pets.\u001b[0m\n", + "\u001b[33m • Button batteries and magnets from small electronics: can cause internal burns or perforations\u001b[0m\n", + "\u001b[33m if swallowed.\u001b[0m\n", + "\u001b[33m • Safety tips: use child-resistant caps, keep medicines up high or in locked boxes, dispose of\u001b[0m\n", + "\u001b[33m expired/unneeded meds properly.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 3. Sharp Objects\u001b[0m\n", + "\u001b[33m • Kitchen knives, box cutters, scissors: cut or puncture wounds.\u001b[0m\n", + "\u001b[33m • Power tools with exposed blades (saws, routers): severe lacerations if guards aren’t used.\u001b[0m\n", + "\u001b[33m • Safety tips: store blades in sheaths or blade guards, keep power tools unplugged and out of\u001b[0m\n", + "\u001b[33m children’s reach.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 4. Fire and Burn Hazards\u001b[0m\n", + "\u001b[33m • Candles, oil lamps: risk of open-flame fires.\u001b[0m\n", + "\u001b[33m • Space heaters, clothes dryers: overheating or lint buildup can ignite.\u001b[0m\n", + "\u001b[33m • Stovetops, irons: scald or burn injuries.\u001b[0m\n", + "\u001b[33m • Safety tips: never leave open flames unattended, keep flammable materials well away from heat\u001b[0m\n", + "\u001b[33m sources, install smoke alarms and check them monthly.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 5. Electrical Hazards\u001b[0m\n", + "\u001b[33m • Overloaded outlets, extension cords: risk of fire or electric shock.\u001b[0m\n", + "\u001b[33m • Faulty appliances with frayed cords or damaged plugs.\u001b[0m\n", + "\u001b[33m • Water-logged electronics (e.g., hair dryers used near sinks).\u001b[0m\n", + "\u001b[33m • Safety tips: replace damaged cords, avoid daisy-chaining extension cords, keep appliances\u001b[0m\n", + "\u001b[33m dry.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 6. Choking and Strangulation Hazards (Especially for Children)\u001b[0m\n", + "\u001b[33m • Small toys, game pieces, coins, marbles.\u001b[0m\n", + "\u001b[33m • Plastic bags, balloons.\u001b[0m\n", + "\u001b[33m • Long cords or strings on blinds, electrical devices.\u001b[0m\n", + "\u001b[33m • Safety tips: supervise young children, use cordless window coverings, keep small items out of\u001b[0m\n", + "\u001b[33m reach.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 7. Carbon Monoxide and Gas Leaks\u001b[0m\n", + "\u001b[33m • Gas stoves, furnaces, water heaters: incomplete combustion can release CO, an odorless,\u001b[0m\n", + "\u001b[33m colorless killer.\u001b[0m\n", + "\u001b[33m • Propane tanks and lines (grills, space heaters).\u001b[0m\n", + "\u001b[33m • Safety tips: install CO detectors near bedrooms, service gas appliances annually, know your\u001b[0m\n", + "\u001b[33m utility emergency number.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 8. Firearms and Hunting Equipment\u001b[0m\n", + "\u001b[33m • Unsecured guns: risk of accidental discharge, especially tragic in homes with children.\u001b[0m\n", + "\u001b[33m • Safety tips: store firearms unloaded in locked safes, keep ammunition separate and locked up,\u001b[0m\n", + "\u001b[33m consider trigger locks.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 9. Pressurized Containers and Explosives\u001b[0m\n", + "\u001b[33m • Aerosol cans, propane cylinders, fire extinguishers: can explode if overheated or corroded.\u001b[0m\n", + "\u001b[33m • Safety tips: store away from direct sunlight or heat, check expiration/pressure ratings.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m 10. Miscellaneous Hazards\u001b[0m\n", + "\u001b[33m • Laundry pods: colorful, scented, and tempting to children—contain concentrated detergents.\u001b[0m\n", + "\u001b[33m • Houseplants: some (e.g. oleander, philodendron) are toxic if chewed.\u001b[0m\n", + "\u001b[33m • Alcohol, solvents (nail polish remover, lighter fluid): ingestion or inhalation risks.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m General Preventive Measures\u001b[0m\n", + "\u001b[33m – Use locked or child-proof storage for any potentially dangerous item.\u001b[0m\n", + "\u001b[33m – Keep hazardous materials in their original containers with labels intact.\u001b[0m\n", + "\u001b[33m – Educate all household members (and caregivers) about dangers and first-aid steps.\u001b[0m\n", + "\u001b[33m – Install and maintain smoke and carbon monoxide detectors.\u001b[0m\n", + "\u001b[33m – Have poison control (or local emergency) number readily accessible.\u001b[0m\n", + "\u001b[33m \u001b[0m\n", + "\u001b[33m By identifying these high-risk items and taking simple precautions—proper storage, supervision and\u001b[0m\n", + "\u001b[33m maintenance—you can greatly reduce the chance of accidental poisoning, burns, cuts, shocks, or\u001b[0m\n", + "\u001b[33m other serious injuries in the home.\u001b[0m\n", + "\n", + "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" + ] + } + ], + "source": [ + "from pyrit.executor.attack import ConsoleAttackResultPrinter, PromptSendingAttack\n", + "from pyrit.prompt_target import OpenAIResponseTarget\n", + "from pyrit.setup import IN_MEMORY, initialize_pyrit_async\n", + "\n", + "await initialize_pyrit_async(memory_db_type=IN_MEMORY) # type: ignore\n", + "\n", + "target = OpenAIResponseTarget(\n", + " reasoning_effort=\"high\",\n", + " reasoning_summary=\"detailed\",\n", + ")\n", + "\n", + "attack = PromptSendingAttack(objective_target=target)\n", + "result = await attack.execute_async(objective=\"What are the most dangerous items in a household?\") # type: ignore\n", + "await ConsoleAttackResultPrinter().print_conversation_async(result=result, include_reasoning_trace=True) # type: ignore" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, "source": [ "## JSON Generation\n", "\n", @@ -93,16 +264,16 @@ { "cell_type": "code", "execution_count": null, - "id": "3", + "id": "5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['/home/vscode/.pyrit/.env', '/home/vscode/.pyrit/.env.local']\n", - "Loaded environment file: /home/vscode/.pyrit/.env\n", - "Loaded environment file: /home/vscode/.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" ] }, { @@ -165,7 +336,7 @@ }, { "cell_type": "markdown", - "id": "4", + "id": "6", "metadata": {}, "source": [ "## Tool Use with Custom Functions\n", @@ -187,27 +358,35 @@ { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['/home/vscode/.pyrit/.env', '/home/vscode/.pyrit/.env.local']\n", - "Loaded environment file: /home/vscode/.pyrit/.env\n", - "Loaded environment file: /home/vscode/.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "0 | assistant: {\"id\":\"rs_02cb1830dae3b0d20069541e7716a8819488f35e15b863919d\",\"type\":\"reasoning\",\"summary\":[],\"content\":null,\"encrypted_content\":null}\n", - "1 | assistant: {\"type\":\"function_call\",\"call_id\":\"call_5ooC2LwwJaPlwfFOWkc7uBpm\",\"name\":\"get_current_weather\",\"arguments\":\"{\\\"location\\\":\\\"Boston\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", - "0 | tool: {\"type\":\"function_call_output\",\"call_id\":\"call_5ooC2LwwJaPlwfFOWkc7uBpm\",\"output\":\"{\\\"weather\\\":\\\"Sunny\\\",\\\"temp_c\\\":22,\\\"location\\\":\\\"Boston\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", + "0 | assistant: {\"id\":\"rs_056451c4ace1c92a006999b8e998a48196bd0f0a83ba0c714f\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", + "1 | assistant: {\"type\":\"function_call\",\"call_id\":\"call_Mcmb7wS3Ps0NuZmDTghCVa1h\",\"name\":\"get_current_weather\",\"arguments\":\"{\\\"location\\\":\\\"Boston\\\", \\\"unit\\\":\\\"celsius\\\"}\"}\n", + "0 | tool: {\"type\":\"function_call_output\",\"call_id\":\"call_Mcmb7wS3Ps0NuZmDTghCVa1h\",\"output\":\"{\\\"weather\\\":\\\"Sunny\\\",\\\"temp_c\\\":22,\\\"location\\\":\\\"Boston\\\",\\\"unit\\\":\\\"celsius\\\"}\"}\n", "0 | assistant: The current weather in Boston is Sunny with a temperature of 22°C.\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\3242724227.py:55: DeprecationWarning: Messagepiece.api_role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")\n" + ] } ], "source": [ @@ -265,12 +444,12 @@ "\n", "for response_msg in response:\n", " for idx, piece in enumerate(response_msg.message_pieces):\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")" + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")" ] }, { "cell_type": "markdown", - "id": "6", + "id": "8", "metadata": {}, "source": [ "## Using the Built-in Web Search Tool\n", @@ -289,24 +468,32 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['/home/vscode/.pyrit/.env', '/home/vscode/.pyrit/.env.local']\n", - "Loaded environment file: /home/vscode/.pyrit/.env\n", - "Loaded environment file: /home/vscode/.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "0 | assistant: {\"type\":\"web_search_call\",\"id\":\"ws_0a650bbbf4434ce60069541e7dbc008190b9ead0fb3cae7734\"}\n", - "1 | assistant: One positive news story from today is that the world’s first international treaty to protect the high seas—a critical area for marine biodiversity—has now been ratified by over 79 nations and will become legally binding in 2026. This landmark agreement will provide a new legal framework to preserve marine life in waters outside national boundaries, representing a major victory for global ocean protection efforts and environmental advocates worldwide [Positive News](https://www.positive.news/society/what-went-right-in-2025-the-good-news-that-mattered/).\n" + "0 | assistant: {\"type\":\"web_search_call\",\"id\":\"ws_09291db6f126d6a2006999b8efcd6881959eac4361d98179f6\"}\n", + "1 | assistant: One positive news story from today is that the European Union has banned the destruction of unsold clothing and shoes. This new regulation aims to make the fashion industry more sustainable by encouraging companies to manage their unsold products through resale, donations, or reuse, rather than sending them to landfills. This move is expected to help significantly reduce textile waste and cut down carbon emissions in Europe, making a positive environmental impact [Good News This Week: February 21, 2026 - Good Good Good](https://www.goodgoodgood.co/articles/good-news-this-week-february-21-2026).\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\1088752442.py:31: DeprecationWarning: Messagepiece.api_role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")\n" ] } ], @@ -341,12 +528,12 @@ "\n", "for response_msg in response:\n", " for idx, piece in enumerate(response_msg.message_pieces):\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")" + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")" ] }, { "cell_type": "markdown", - "id": "8", + "id": "10", "metadata": {}, "source": [ "## Grammar-Constrained Generation\n", @@ -361,16 +548,16 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "11", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['/home/vscode/.pyrit/.env', '/home/vscode/.pyrit/.env.local']\n", - "Loaded environment file: /home/vscode/.pyrit/.env\n", - "Loaded environment file: /home/vscode/.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\romanlutz\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\romanlutz\\.pyrit\\.env.local\n" ] }, { @@ -378,13 +565,23 @@ "output_type": "stream", "text": [ "Unconstrained Response:\n", - "0 | assistant: {\"id\":\"rs_0d3f19062cddb30c0069541e82616c8190a7d9dfee276f9c92\",\"type\":\"reasoning\",\"summary\":[],\"content\":null,\"encrypted_content\":null}\n", + "0 | assistant: {\"id\":\"rs_04ece24776bc975c006999b8f4ea888195ab3482b8558069e4\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", "1 | assistant: Rome.\n", "\n", "Constrained Response:\n", - "0 | assistant: {\"id\":\"rs_0ec76b80c50f2b190069541e8603e08194aa02993767062a00\",\"type\":\"reasoning\",\"summary\":[],\"content\":null,\"encrypted_content\":null}\n", + "0 | assistant: {\"id\":\"rs_0ecdf4ebc836f95b006999bacec41c8195b5d770fe59ca3ccd\",\"summary\":[],\"type\":\"reasoning\",\"content\":null,\"encrypted_content\":null,\"status\":null}\n", "1 | assistant: I think that it is Pisa\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\138977321.py:51: DeprecationWarning: Messagepiece.api_role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")\n", + "C:\\Users\\romanlutz\\AppData\\Local\\Temp\\ipykernel_55020\\138977321.py:58: DeprecationWarning: Messagepiece.api_role getter is deprecated. Use api_role for comparisons. This property will be removed in 0.13.0.\n", + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")\n" + ] } ], "source": [ @@ -438,14 +635,14 @@ "print(\"Unconstrained Response:\")\n", "for response_msg in unconstrained_result:\n", " for idx, piece in enumerate(response_msg.message_pieces):\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")\n", + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")\n", "\n", "print()\n", "\n", "print(\"Constrained Response:\")\n", "for response_msg in result:\n", " for idx, piece in enumerate(response_msg.message_pieces):\n", - " print(f\"{idx} | {piece.role}: {piece.original_value}\")" + " print(f\"{idx} | {piece.api_role}: {piece.original_value}\")" ] } ], @@ -460,7 +657,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.14" + "version": "3.12.12" } }, "nbformat": 4, diff --git a/doc/code/targets/2_openai_responses_target.py b/doc/code/targets/2_openai_responses_target.py index 4bf1234fd..74ff31e04 100644 --- a/doc/code/targets/2_openai_responses_target.py +++ b/doc/code/targets/2_openai_responses_target.py @@ -46,6 +46,32 @@ result = await attack.execute_async(objective="Tell me a joke") # type: ignore await ConsoleAttackResultPrinter().print_conversation_async(result=result) # type: ignore +# %% [markdown] +# ## Reasoning Configuration +# +# Reasoning models (e.g., o1, o3, o4-mini, GPT-5) support a `reasoning` parameter that controls how much internal reasoning the model performs before responding. You can configure this with two parameters: +# +# - **`reasoning_effort`**: Controls the depth of reasoning. Accepts `"minimal"`, `"low"`, `"medium"`, or `"high"`. Lower effort favors speed and lower cost; higher effort favors thoroughness. The default (when not set) is typically `"medium"`. +# - **`reasoning_summary`**: Controls whether a summary of the model's internal reasoning is included in the response. Accepts `"auto"`, `"concise"`, or `"detailed"`. By default, no summary is included. +# +# For more information, see the [OpenAI reasoning guide](https://developers.openai.com/api/docs/guides/reasoning). + +# %% +from pyrit.executor.attack import ConsoleAttackResultPrinter, PromptSendingAttack +from pyrit.prompt_target import OpenAIResponseTarget +from pyrit.setup import IN_MEMORY, initialize_pyrit_async + +await initialize_pyrit_async(memory_db_type=IN_MEMORY) # type: ignore + +target = OpenAIResponseTarget( + reasoning_effort="high", + reasoning_summary="detailed", +) + +attack = PromptSendingAttack(objective_target=target) +result = await attack.execute_async(objective="What are the most dangerous items in a household?") # type: ignore +await ConsoleAttackResultPrinter().print_conversation_async(result=result, include_reasoning_trace=True) # type: ignore + # %% [markdown] # ## JSON Generation # @@ -170,7 +196,7 @@ async def get_current_weather(args): for response_msg in response: for idx, piece in enumerate(response_msg.message_pieces): - print(f"{idx} | {piece.role}: {piece.original_value}") + print(f"{idx} | {piece.api_role}: {piece.original_value}") # %% [markdown] # ## Using the Built-in Web Search Tool @@ -216,7 +242,7 @@ async def get_current_weather(args): for response_msg in response: for idx, piece in enumerate(response_msg.message_pieces): - print(f"{idx} | {piece.role}: {piece.original_value}") + print(f"{idx} | {piece.api_role}: {piece.original_value}") # %% [markdown] # ## Grammar-Constrained Generation @@ -278,11 +304,11 @@ async def get_current_weather(args): print("Unconstrained Response:") for response_msg in unconstrained_result: for idx, piece in enumerate(response_msg.message_pieces): - print(f"{idx} | {piece.role}: {piece.original_value}") + print(f"{idx} | {piece.api_role}: {piece.original_value}") print() print("Constrained Response:") for response_msg in result: for idx, piece in enumerate(response_msg.message_pieces): - print(f"{idx} | {piece.role}: {piece.original_value}") + print(f"{idx} | {piece.api_role}: {piece.original_value}") diff --git a/pyrit/executor/attack/printer/console_printer.py b/pyrit/executor/attack/printer/console_printer.py index c71b40b31..ef703e1a8 100644 --- a/pyrit/executor/attack/printer/console_printer.py +++ b/pyrit/executor/attack/printer/console_printer.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import json import textwrap from datetime import datetime from typing import Any @@ -201,8 +202,14 @@ async def print_messages_async( # Now print all pieces in this message for piece in message.message_pieces: - # Skip reasoning traces unless explicitly requested - if piece.original_value_data_type == "reasoning" and not include_reasoning_trace: + # Reasoning pieces: show summary when include_reasoning_trace is set + if piece.original_value_data_type == "reasoning": + if include_reasoning_trace: + summary_text = self._extract_reasoning_summary(piece.original_value) + if summary_text: + self._print_colored(f"{self._indent}💭 Reasoning Summary:", Style.DIM, Fore.CYAN) + self._print_wrapped_text(summary_text, Fore.CYAN) + print() continue # Handle converted values for user and assistant messages @@ -234,6 +241,28 @@ async def print_messages_async( print() self._print_colored("─" * self._width, Fore.BLUE) + def _extract_reasoning_summary(self, reasoning_value: str) -> str: + """ + Extract human-readable summary text from a reasoning piece's JSON value. + + Args: + reasoning_value (str): The JSON string stored in the reasoning piece. + + Returns: + str: The concatenated summary text, or empty string if no summary is present. + """ + try: + data = json.loads(reasoning_value) + except (json.JSONDecodeError, TypeError): + return "" + + summary = data.get("summary") if isinstance(data, dict) else None + if not summary or not isinstance(summary, list): + return "" + + parts = [item.get("text", "") for item in summary if isinstance(item, dict) and item.get("text")] + return "\n".join(parts) + async def print_summary_async(self, result: AttackResult) -> None: """ Print a summary of the attack result with enhanced formatting. diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py index eda38a139..1eef3f49b 100644 --- a/pyrit/prompt_target/openai/openai_response_target.py +++ b/pyrit/prompt_target/openai/openai_response_target.py @@ -10,11 +10,14 @@ Callable, Dict, List, + Literal, MutableSequence, Optional, cast, ) +from openai.types.shared import ReasoningEffort + from pyrit.common import convert_local_image_to_data_url from pyrit.exceptions import ( EmptyResponseException, @@ -75,6 +78,8 @@ def __init__( max_output_tokens: Optional[int] = None, temperature: Optional[float] = None, top_p: Optional[float] = None, + reasoning_effort: Optional[ReasoningEffort] = None, + reasoning_summary: Optional[Literal["auto", "concise", "detailed"]] = None, extra_body_parameters: Optional[dict[str, Any]] = None, fail_on_missing_function: bool = False, **kwargs: Any, @@ -100,6 +105,13 @@ def __init__( randomness of the response. top_p (float, Optional): The top-p parameter for controlling the diversity of the response. + reasoning_effort (ReasoningEffort, Optional): Controls how much reasoning the model + performs. Accepts "minimal", "low", "medium", or "high". Lower effort + favors speed and lower cost; higher effort favors thoroughness. Defaults to None + (uses model default, typically "medium"). + reasoning_summary (Literal["auto", "concise", "detailed"], Optional): Controls + whether a summary of the model's reasoning is included in the response. + Defaults to None (no summary). is_json_supported (bool, Optional): If True, the target will support formatting responses as JSON by setting the response_format header. Official OpenAI models all support this, but if you are using this target with different models, is_json_supported should be set correctly to avoid issues when @@ -133,9 +145,8 @@ def __init__( self._top_p = top_p self._max_output_tokens = max_output_tokens - # Reasoning parameters are not yet supported by PyRIT. - # See https://platform.openai.com/docs/api-reference/responses/create#responses-create-reasoning - # for more information. + self._reasoning_effort = reasoning_effort + self._reasoning_summary = reasoning_summary self._extra_body_parameters = extra_body_parameters @@ -169,6 +180,8 @@ def _build_identifier(self) -> ComponentIdentifier: "temperature": self._temperature, "top_p": self._top_p, "max_output_tokens": self._max_output_tokens, + "reasoning_effort": self._reasoning_effort, + "reasoning_summary": self._reasoning_summary, }, ) @@ -360,6 +373,7 @@ async def _construct_request_body( "input": input_items, # Correct JSON response format per Responses API "text": text_format, + "reasoning": self._build_reasoning_config(), } if self._extra_body_parameters: @@ -368,6 +382,23 @@ async def _construct_request_body( # Filter out None values return {k: v for k, v in body_parameters.items() if v is not None} + def _build_reasoning_config(self) -> Optional[Dict[str, Any]]: + """ + Build the reasoning configuration dict for the Responses API. + + Returns: + Optional[Dict[str, Any]]: The reasoning config, or None if neither effort nor summary is set. + """ + if self._reasoning_effort is None and self._reasoning_summary is None: + return None + + reasoning: Dict[str, Any] = {} + if self._reasoning_effort is not None: + reasoning["effort"] = self._reasoning_effort + if self._reasoning_summary is not None: + reasoning["summary"] = self._reasoning_summary + return reasoning + def _build_text_format(self, json_config: _JsonResponseConfig) -> Optional[Dict[str, Any]]: if not json_config.enabled: return None @@ -577,13 +608,7 @@ def _parse_response_output_section( elif section_type == MessagePieceType.REASONING: # Store reasoning in memory for debugging/logging, but won't be sent back to API piece_value = json.dumps( - { - "id": section.id, - "type": section.type, - "summary": section.summary, - "content": section.content, - "encrypted_content": section.encrypted_content, - }, + section.model_dump(), separators=(",", ":"), ) piece_type = "reasoning" @@ -691,12 +716,13 @@ def _find_last_pending_tool_call(self, reply: Message) -> Optional[dict[str, Any The tool-call section dict, or None if not found. """ for piece in reversed(reply.message_pieces): - if piece.api_role == "assistant": + # Filter on data_type to skip reasoning/message pieces that also have api_role "assistant". + if piece.api_role == "assistant" and piece.original_value_data_type == "function_call": try: section = json.loads(piece.original_value) except Exception: continue - if section.get("type") == "function_call": + if isinstance(section, dict) and section.get("type") == "function_call": # Do NOT skip function_call even if status == "completed" — we still need to emit the output. return cast(dict[str, Any], section) return None diff --git a/tests/integration/targets/test_entra_auth_targets.py b/tests/integration/targets/test_entra_auth_targets.py index a1fa3ebe5..bf068bfb2 100644 --- a/tests/integration/targets/test_entra_auth_targets.py +++ b/tests/integration/targets/test_entra_auth_targets.py @@ -237,6 +237,57 @@ async def test_openai_responses_target_entra_auth(sqlite_instance, endpoint, mod assert result.last_response is not None +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("endpoint", "model_name"), + [ + ("OPENAI_RESPONSES_ENDPOINT", "OPENAI_RESPONSES_MODEL"), + ("AZURE_OPENAI_GPT5_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT5_MODEL"), + ], +) +async def test_openai_responses_target_reasoning_effort_entra_auth(sqlite_instance, endpoint, model_name): + endpoint_value = os.environ[endpoint] + args = { + "endpoint": endpoint_value, + "model_name": os.environ[model_name], + "api_key": get_azure_openai_auth(endpoint_value), + "reasoning_effort": "low", + } + + target = OpenAIResponseTarget(**args) + + attack = PromptSendingAttack(objective_target=target) + result = await attack.execute_async(objective="What is 2 + 2?") + assert result is not None + assert result.last_response is not None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("endpoint", "model_name"), + [ + ("OPENAI_RESPONSES_ENDPOINT", "OPENAI_RESPONSES_MODEL"), + ("AZURE_OPENAI_GPT5_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT5_MODEL"), + ], +) +async def test_openai_responses_target_reasoning_summary_entra_auth(sqlite_instance, endpoint, model_name): + endpoint_value = os.environ[endpoint] + args = { + "endpoint": endpoint_value, + "model_name": os.environ[model_name], + "api_key": get_azure_openai_auth(endpoint_value), + "reasoning_effort": "low", + "reasoning_summary": "auto", + } + + target = OpenAIResponseTarget(**args) + + attack = PromptSendingAttack(objective_target=target) + result = await attack.execute_async(objective="What is 2 + 2?") + assert result is not None + assert result.last_response is not None + + @pytest.mark.asyncio @pytest.mark.parametrize( ("endpoint", "model_name"), diff --git a/tests/integration/targets/test_openai_responses_gpt5.py b/tests/integration/targets/test_openai_responses_gpt5.py index 82f56bc8c..8f6bf3ae4 100644 --- a/tests/integration/targets/test_openai_responses_gpt5.py +++ b/tests/integration/targets/test_openai_responses_gpt5.py @@ -51,8 +51,8 @@ async def test_openai_responses_gpt5(sqlite_instance, gpt5_args): assert result is not None assert len(result) == 1 assert len(result[0].message_pieces) == 2 - assert result[0].message_pieces[0].role == "assistant" - assert result[0].message_pieces[1].role == "assistant" + assert result[0].message_pieces[0].api_role == "assistant" + assert result[0].message_pieces[1].api_role == "assistant" # Hope that the model manages to give the correct answer somewhere (GPT-5 really should) assert "Paris" in result[0].message_pieces[1].converted_value @@ -104,7 +104,7 @@ async def test_openai_responses_gpt5_json_schema(sqlite_instance, gpt5_args): assert len(response) == 1 assert len(response[0].message_pieces) == 2 response_piece = response[0].message_pieces[1] - assert response_piece.role == "assistant" + assert response_piece.api_role == "assistant" response_json = json.loads(response_piece.converted_value) jsonschema.validate(instance=response_json, schema=cat_schema) @@ -140,6 +140,44 @@ async def test_openai_responses_gpt5_json_object(sqlite_instance, gpt5_args): assert len(response) == 1 assert len(response[0].message_pieces) == 2 response_piece = response[0].message_pieces[1] - assert response_piece.role == "assistant" + assert response_piece.api_role == "assistant" _ = json.loads(response_piece.converted_value) # Can't assert more, since the failure could be due to a bad generation by the model + + +@pytest.mark.asyncio +async def test_openai_responses_gpt5_reasoning_effort(sqlite_instance, gpt5_args): + target = OpenAIResponseTarget(**gpt5_args, reasoning_effort="low") + + conv_id = str(uuid.uuid4()) + + user_piece = MessagePiece( + role="user", + original_value="What is 2 + 2?", + original_value_data_type="text", + conversation_id=conv_id, + ) + + result = await target.send_prompt_async(message=user_piece.to_message()) + assert result is not None + assert len(result) == 1 + assert any(p.converted_value_data_type == "text" for p in result[0].message_pieces) + + +@pytest.mark.asyncio +async def test_openai_responses_gpt5_reasoning_summary(sqlite_instance, gpt5_args): + target = OpenAIResponseTarget(**gpt5_args, reasoning_effort="low", reasoning_summary="auto") + + conv_id = str(uuid.uuid4()) + + user_piece = MessagePiece( + role="user", + original_value="What is 2 + 2?", + original_value_data_type="text", + conversation_id=conv_id, + ) + + result = await target.send_prompt_async(message=user_piece.to_message()) + assert result is not None + assert len(result) == 1 + assert any(p.converted_value_data_type == "text" for p in result[0].message_pieces) diff --git a/tests/unit/target/test_openai_response_target.py b/tests/unit/target/test_openai_response_target.py index 581431cd6..1e4b95e0b 100644 --- a/tests/unit/target/test_openai_response_target.py +++ b/tests/unit/target/test_openai_response_target.py @@ -1178,3 +1178,120 @@ async def test_construct_message_from_response(target: OpenAIResponseTarget, dum assert isinstance(result, Message) assert len(result.message_pieces) == 1 mock_parse.assert_called_once() + + +# ── Reasoning effort / summary tests ─────────────────────────────────────── + + +def test_init_with_reasoning_effort(patch_central_database): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_effort="high", + ) + assert target._reasoning_effort == "high" + + +def test_init_with_reasoning_summary(patch_central_database): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_summary="auto", + ) + assert target._reasoning_summary == "auto" + + +def test_init_with_reasoning_effort_and_summary(patch_central_database): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_effort="low", + reasoning_summary="detailed", + ) + assert target._reasoning_effort == "low" + assert target._reasoning_summary == "detailed" + + +def test_init_without_reasoning_params(patch_central_database): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + ) + assert target._reasoning_effort is None + assert target._reasoning_summary is None + + +@pytest.mark.asyncio +async def test_construct_request_body_includes_reasoning_effort( + patch_central_database, dummy_text_message_piece: MessagePiece +): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_effort="medium", + ) + request = Message(message_pieces=[dummy_text_message_piece]) + jrc = _JsonResponseConfig.from_metadata(metadata=None) + body = await target._construct_request_body(conversation=[request], json_config=jrc) + assert body["reasoning"] == {"effort": "medium"} + + +@pytest.mark.asyncio +async def test_construct_request_body_includes_reasoning_summary( + patch_central_database, dummy_text_message_piece: MessagePiece +): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_summary="detailed", + ) + request = Message(message_pieces=[dummy_text_message_piece]) + jrc = _JsonResponseConfig.from_metadata(metadata=None) + body = await target._construct_request_body(conversation=[request], json_config=jrc) + assert body["reasoning"] == {"summary": "detailed"} + + +@pytest.mark.asyncio +async def test_construct_request_body_includes_reasoning_effort_and_summary( + patch_central_database, dummy_text_message_piece: MessagePiece +): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_effort="high", + reasoning_summary="auto", + ) + request = Message(message_pieces=[dummy_text_message_piece]) + jrc = _JsonResponseConfig.from_metadata(metadata=None) + body = await target._construct_request_body(conversation=[request], json_config=jrc) + assert body["reasoning"] == {"effort": "high", "summary": "auto"} + + +@pytest.mark.asyncio +async def test_construct_request_body_omits_reasoning_when_not_set( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece +): + request = Message(message_pieces=[dummy_text_message_piece]) + jrc = _JsonResponseConfig.from_metadata(metadata=None) + body = await target._construct_request_body(conversation=[request], json_config=jrc) + assert "reasoning" not in body + + +def test_build_identifier_includes_reasoning_params(patch_central_database): + target = OpenAIResponseTarget( + model_name="gpt-5", + endpoint="https://mock.azure.com/", + api_key="mock-api-key", + reasoning_effort="low", + reasoning_summary="concise", + ) + identifier = target._build_identifier() + assert identifier.params["reasoning_effort"] == "low" + assert identifier.params["reasoning_summary"] == "concise"