From 78130f429029a2c06b3e208b701b07326e5a84d3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 22:11:54 +0000 Subject: [PATCH 1/7] Initial plan From e1c86b203499eea0202f979839159b8a6a33be8e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 22:15:33 +0000 Subject: [PATCH 2/7] Standardize version to VERSION and replace assert with proper error checks in scenarios Co-authored-by: ValbuenaVC <50061128+ValbuenaVC@users.noreply.github.com> --- pyrit/scenario/scenarios/airt/content_harms.py | 9 ++++++--- pyrit/scenario/scenarios/airt/cyber.py | 12 ++++++++---- pyrit/scenario/scenarios/airt/jailbreak.py | 9 ++++++--- pyrit/scenario/scenarios/airt/leakage_scenario.py | 9 ++++++--- pyrit/scenario/scenarios/airt/scam.py | 9 ++++++--- pyrit/scenario/scenarios/foundry/red_team_agent.py | 4 ++-- pyrit/scenario/scenarios/garak/encoding.py | 9 ++++++--- 7 files changed, 40 insertions(+), 21 deletions(-) diff --git a/pyrit/scenario/scenarios/airt/content_harms.py b/pyrit/scenario/scenarios/airt/content_harms.py index d2fb4055f6..446dcb1b2e 100644 --- a/pyrit/scenario/scenarios/airt/content_harms.py +++ b/pyrit/scenario/scenarios/airt/content_harms.py @@ -104,7 +104,7 @@ class ContentHarms(Scenario): with respect to certain harm categories. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> Type[ScenarioStrategy]: @@ -179,7 +179,7 @@ def __init__( super().__init__( name="Content Harms", - version=self.version, + version=self.VERSION, objective_scorer=self._objective_scorer, strategy_class=ContentHarmsStrategy, scenario_result_id=scenario_result_id, @@ -241,7 +241,10 @@ def _get_strategy_attacks( List[AtomicAttack]: The constructed AtomicAttack instances for each attack type. """ # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if not self._objective_target: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) prompt_sending_attack = PromptSendingAttack( objective_target=self._objective_target, diff --git a/pyrit/scenario/scenarios/airt/cyber.py b/pyrit/scenario/scenarios/airt/cyber.py index 220fac3f74..ce0fc8a495 100644 --- a/pyrit/scenario/scenarios/airt/cyber.py +++ b/pyrit/scenario/scenarios/airt/cyber.py @@ -60,7 +60,7 @@ class Cyber(Scenario): techniques. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -141,7 +141,7 @@ def __init__( super().__init__( name="Cyber", - version=self.version, + version=self.VERSION, strategy_class=CyberStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, @@ -244,7 +244,10 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: ValueError: if an unknown CyberStrategy is passed. """ # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if not self._objective_target: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) attack_strategy: Optional[AttackStrategy[Any, Any]] = None if strategy == "single_turn": attack_strategy = PromptSendingAttack( @@ -261,7 +264,8 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: raise ValueError(f"Unknown CyberStrategy: {strategy}") # _seed_groups is guaranteed to be set by _get_atomic_attacks_async before this method is called - assert self._seed_groups is not None, "_seed_groups must be resolved before creating atomic attacks" + if not self._seed_groups: + raise ValueError("_seed_groups must be resolved before creating atomic attacks") return AtomicAttack( atomic_attack_name=f"cyber_{strategy}", diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index e28676db72..c5041b5e71 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -49,7 +49,7 @@ class Jailbreak(Scenario): scored to determine if the jailbreak was successful. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -114,7 +114,7 @@ def __init__( super().__init__( name="Jailbreak", - version=self.version, + version=self.VERSION, strategy_class=JailbreakStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, @@ -184,7 +184,10 @@ async def _get_atomic_attack_from_jailbreak_async(self, *, jailbreak_template_na AtomicAttack: An atomic attack using the specified jailbreak template. """ # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if not self._objective_target: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) # Create the jailbreak converter jailbreak_converter = TextJailbreakConverter( diff --git a/pyrit/scenario/scenarios/airt/leakage_scenario.py b/pyrit/scenario/scenarios/airt/leakage_scenario.py index 5137ef2eb7..9a440dd59e 100644 --- a/pyrit/scenario/scenarios/airt/leakage_scenario.py +++ b/pyrit/scenario/scenarios/airt/leakage_scenario.py @@ -89,7 +89,7 @@ class LeakageScenario(Scenario): attack variations designed to extract sensitive information from models. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -161,7 +161,7 @@ def __init__( super().__init__( name="Leakage Scenario", - version=self.version, + version=self.VERSION, strategy_class=LeakageStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, @@ -264,7 +264,10 @@ async def _get_atomic_attack_from_strategy_async(self, strategy: str) -> AtomicA ValueError: If an unknown LeakageStrategy is passed. """ # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if not self._objective_target: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) strategy_factories = { "first_letter": self._create_first_letter_attack, diff --git a/pyrit/scenario/scenarios/airt/scam.py b/pyrit/scenario/scenarios/airt/scam.py index e29204a715..7c762f2c27 100644 --- a/pyrit/scenario/scenarios/airt/scam.py +++ b/pyrit/scenario/scenarios/airt/scam.py @@ -89,7 +89,7 @@ class Scam(Scenario): (e.g., phishing emails, fraudulent messages) with primarily persuasion-oriented techniques. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -167,7 +167,7 @@ def __init__( super().__init__( name="Scam", - version=self.version, + version=self.VERSION, strategy_class=ScamStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, @@ -269,7 +269,10 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: ValueError: If an unknown ScamStrategy is provided. """ # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if not self._objective_target: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) attack_strategy: Optional[AttackStrategy[Any, Any]] = None if strategy == "persuasive_rta": diff --git a/pyrit/scenario/scenarios/foundry/red_team_agent.py b/pyrit/scenario/scenarios/foundry/red_team_agent.py index 04ba25272d..b85242bcd1 100644 --- a/pyrit/scenario/scenarios/foundry/red_team_agent.py +++ b/pyrit/scenario/scenarios/foundry/red_team_agent.py @@ -214,7 +214,7 @@ class RedTeamAgent(Scenario): providing a consistent PyRIT contract for their integration. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> Type[ScenarioStrategy]: @@ -297,7 +297,7 @@ def __init__( # Call super().__init__() first to initialize self._memory super().__init__( name="RedTeamAgent", - version=self.version, + version=self.VERSION, strategy_class=FoundryStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, diff --git a/pyrit/scenario/scenarios/garak/encoding.py b/pyrit/scenario/scenarios/garak/encoding.py index 712d2efa6c..378bc1141c 100644 --- a/pyrit/scenario/scenarios/garak/encoding.py +++ b/pyrit/scenario/scenarios/garak/encoding.py @@ -133,7 +133,7 @@ class Encoding(Scenario): By default, this uses the same dataset as Garak: slur terms and web XSS payloads. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -209,7 +209,7 @@ def __init__( super().__init__( name="Encoding", - version=self.version, + version=self.VERSION, strategy_class=EncodingStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, @@ -351,7 +351,10 @@ def _get_prompt_attacks(self, *, converters: list[PromptConverter], encoding_nam atomic_attacks = [] for attack_converter_config in converter_configs: # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if not self._objective_target: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) attack = PromptSendingAttack( objective_target=self._objective_target, attack_converter_config=attack_converter_config, From ac78b46a414541c1146cf09c1787f8110a67a7e2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 22:24:30 +0000 Subject: [PATCH 3/7] Add version alias for backward compatibility with tests Co-authored-by: ValbuenaVC <50061128+ValbuenaVC@users.noreply.github.com> --- pyrit/scenario/scenarios/airt/content_harms.py | 1 + pyrit/scenario/scenarios/airt/cyber.py | 1 + pyrit/scenario/scenarios/airt/jailbreak.py | 1 + pyrit/scenario/scenarios/airt/leakage_scenario.py | 1 + pyrit/scenario/scenarios/airt/scam.py | 1 + pyrit/scenario/scenarios/foundry/red_team_agent.py | 1 + pyrit/scenario/scenarios/garak/encoding.py | 1 + 7 files changed, 7 insertions(+) diff --git a/pyrit/scenario/scenarios/airt/content_harms.py b/pyrit/scenario/scenarios/airt/content_harms.py index 446dcb1b2e..f5804b3751 100644 --- a/pyrit/scenario/scenarios/airt/content_harms.py +++ b/pyrit/scenario/scenarios/airt/content_harms.py @@ -105,6 +105,7 @@ class ContentHarms(Scenario): """ VERSION: int = 1 + version: int = 1 # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> Type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/airt/cyber.py b/pyrit/scenario/scenarios/airt/cyber.py index ce0fc8a495..9cee8226a7 100644 --- a/pyrit/scenario/scenarios/airt/cyber.py +++ b/pyrit/scenario/scenarios/airt/cyber.py @@ -61,6 +61,7 @@ class Cyber(Scenario): """ VERSION: int = 1 + version: int = 1 # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index c5041b5e71..b4b116a61c 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -50,6 +50,7 @@ class Jailbreak(Scenario): """ VERSION: int = 1 + version: int = 1 # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/airt/leakage_scenario.py b/pyrit/scenario/scenarios/airt/leakage_scenario.py index 9a440dd59e..b0e13e6b62 100644 --- a/pyrit/scenario/scenarios/airt/leakage_scenario.py +++ b/pyrit/scenario/scenarios/airt/leakage_scenario.py @@ -90,6 +90,7 @@ class LeakageScenario(Scenario): """ VERSION: int = 1 + version: int = 1 # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/airt/scam.py b/pyrit/scenario/scenarios/airt/scam.py index 7c762f2c27..99b258963c 100644 --- a/pyrit/scenario/scenarios/airt/scam.py +++ b/pyrit/scenario/scenarios/airt/scam.py @@ -90,6 +90,7 @@ class Scam(Scenario): """ VERSION: int = 1 + version: int = 1 # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/foundry/red_team_agent.py b/pyrit/scenario/scenarios/foundry/red_team_agent.py index b85242bcd1..a4ec6d32ad 100644 --- a/pyrit/scenario/scenarios/foundry/red_team_agent.py +++ b/pyrit/scenario/scenarios/foundry/red_team_agent.py @@ -215,6 +215,7 @@ class RedTeamAgent(Scenario): """ VERSION: int = 1 + version: int = 1 # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> Type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/garak/encoding.py b/pyrit/scenario/scenarios/garak/encoding.py index 378bc1141c..1e56c19b22 100644 --- a/pyrit/scenario/scenarios/garak/encoding.py +++ b/pyrit/scenario/scenarios/garak/encoding.py @@ -134,6 +134,7 @@ class Encoding(Scenario): """ VERSION: int = 1 + version: int = 1 # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: From 3cd2d55d56f9689fb49e9fb97c3806293a12703c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 22:26:53 +0000 Subject: [PATCH 4/7] Use explicit None checks instead of falsy checks for better type safety Co-authored-by: ValbuenaVC <50061128+ValbuenaVC@users.noreply.github.com> --- pyrit/scenario/scenarios/airt/content_harms.py | 2 +- pyrit/scenario/scenarios/airt/cyber.py | 4 ++-- pyrit/scenario/scenarios/airt/jailbreak.py | 2 +- pyrit/scenario/scenarios/airt/leakage_scenario.py | 2 +- pyrit/scenario/scenarios/airt/scam.py | 2 +- pyrit/scenario/scenarios/garak/encoding.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyrit/scenario/scenarios/airt/content_harms.py b/pyrit/scenario/scenarios/airt/content_harms.py index f5804b3751..2c34f4bb68 100644 --- a/pyrit/scenario/scenarios/airt/content_harms.py +++ b/pyrit/scenario/scenarios/airt/content_harms.py @@ -242,7 +242,7 @@ def _get_strategy_attacks( List[AtomicAttack]: The constructed AtomicAttack instances for each attack type. """ # objective_target is guaranteed to be non-None by parent class validation - if not self._objective_target: + if self._objective_target is None: raise ValueError( "Scenario not properly initialized. Call await scenario.initialize_async() before running." ) diff --git a/pyrit/scenario/scenarios/airt/cyber.py b/pyrit/scenario/scenarios/airt/cyber.py index 9cee8226a7..ece8bf1b91 100644 --- a/pyrit/scenario/scenarios/airt/cyber.py +++ b/pyrit/scenario/scenarios/airt/cyber.py @@ -245,7 +245,7 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: ValueError: if an unknown CyberStrategy is passed. """ # objective_target is guaranteed to be non-None by parent class validation - if not self._objective_target: + if self._objective_target is None: raise ValueError( "Scenario not properly initialized. Call await scenario.initialize_async() before running." ) @@ -265,7 +265,7 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: raise ValueError(f"Unknown CyberStrategy: {strategy}") # _seed_groups is guaranteed to be set by _get_atomic_attacks_async before this method is called - if not self._seed_groups: + if self._seed_groups is None: raise ValueError("_seed_groups must be resolved before creating atomic attacks") return AtomicAttack( diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index b4b116a61c..58c6c99eb0 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -185,7 +185,7 @@ async def _get_atomic_attack_from_jailbreak_async(self, *, jailbreak_template_na AtomicAttack: An atomic attack using the specified jailbreak template. """ # objective_target is guaranteed to be non-None by parent class validation - if not self._objective_target: + if self._objective_target is None: raise ValueError( "Scenario not properly initialized. Call await scenario.initialize_async() before running." ) diff --git a/pyrit/scenario/scenarios/airt/leakage_scenario.py b/pyrit/scenario/scenarios/airt/leakage_scenario.py index b0e13e6b62..e58b58c11b 100644 --- a/pyrit/scenario/scenarios/airt/leakage_scenario.py +++ b/pyrit/scenario/scenarios/airt/leakage_scenario.py @@ -265,7 +265,7 @@ async def _get_atomic_attack_from_strategy_async(self, strategy: str) -> AtomicA ValueError: If an unknown LeakageStrategy is passed. """ # objective_target is guaranteed to be non-None by parent class validation - if not self._objective_target: + if self._objective_target is None: raise ValueError( "Scenario not properly initialized. Call await scenario.initialize_async() before running." ) diff --git a/pyrit/scenario/scenarios/airt/scam.py b/pyrit/scenario/scenarios/airt/scam.py index 99b258963c..df3531ffd5 100644 --- a/pyrit/scenario/scenarios/airt/scam.py +++ b/pyrit/scenario/scenarios/airt/scam.py @@ -270,7 +270,7 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: ValueError: If an unknown ScamStrategy is provided. """ # objective_target is guaranteed to be non-None by parent class validation - if not self._objective_target: + if self._objective_target is None: raise ValueError( "Scenario not properly initialized. Call await scenario.initialize_async() before running." ) diff --git a/pyrit/scenario/scenarios/garak/encoding.py b/pyrit/scenario/scenarios/garak/encoding.py index 1e56c19b22..3ce929136f 100644 --- a/pyrit/scenario/scenarios/garak/encoding.py +++ b/pyrit/scenario/scenarios/garak/encoding.py @@ -352,7 +352,7 @@ def _get_prompt_attacks(self, *, converters: list[PromptConverter], encoding_nam atomic_attacks = [] for attack_converter_config in converter_configs: # objective_target is guaranteed to be non-None by parent class validation - if not self._objective_target: + if self._objective_target is None: raise ValueError( "Scenario not properly initialized. Call await scenario.initialize_async() before running." ) From 4290950c9c00712e03069ede804429dc2d003256 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 23:00:37 +0000 Subject: [PATCH 5/7] Use VERSION as reference for version alias and add ValueError to docstrings Co-authored-by: ValbuenaVC <50061128+ValbuenaVC@users.noreply.github.com> --- pyrit/scenario/scenarios/airt/content_harms.py | 5 ++++- pyrit/scenario/scenarios/airt/cyber.py | 4 ++-- pyrit/scenario/scenarios/airt/jailbreak.py | 5 ++++- pyrit/scenario/scenarios/airt/leakage_scenario.py | 4 ++-- pyrit/scenario/scenarios/airt/scam.py | 4 ++-- pyrit/scenario/scenarios/foundry/red_team_agent.py | 2 +- pyrit/scenario/scenarios/garak/encoding.py | 5 ++++- 7 files changed, 19 insertions(+), 10 deletions(-) diff --git a/pyrit/scenario/scenarios/airt/content_harms.py b/pyrit/scenario/scenarios/airt/content_harms.py index 2c34f4bb68..ab26409490 100644 --- a/pyrit/scenario/scenarios/airt/content_harms.py +++ b/pyrit/scenario/scenarios/airt/content_harms.py @@ -105,7 +105,7 @@ class ContentHarms(Scenario): """ VERSION: int = 1 - version: int = 1 # Alias for backward compatibility + version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> Type[ScenarioStrategy]: @@ -240,6 +240,9 @@ def _get_strategy_attacks( Returns: List[AtomicAttack]: The constructed AtomicAttack instances for each attack type. + + Raises: + ValueError: If scenario is not properly initialized. """ # objective_target is guaranteed to be non-None by parent class validation if self._objective_target is None: diff --git a/pyrit/scenario/scenarios/airt/cyber.py b/pyrit/scenario/scenarios/airt/cyber.py index ece8bf1b91..e69b95373c 100644 --- a/pyrit/scenario/scenarios/airt/cyber.py +++ b/pyrit/scenario/scenarios/airt/cyber.py @@ -61,7 +61,7 @@ class Cyber(Scenario): """ VERSION: int = 1 - version: int = 1 # Alias for backward compatibility + version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -242,7 +242,7 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: AtomicAttack: configured for the specified strategy. Raises: - ValueError: if an unknown CyberStrategy is passed. + ValueError: If scenario is not properly initialized or an unknown CyberStrategy is passed. """ # objective_target is guaranteed to be non-None by parent class validation if self._objective_target is None: diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index 58c6c99eb0..e9092831e9 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -50,7 +50,7 @@ class Jailbreak(Scenario): """ VERSION: int = 1 - version: int = 1 # Alias for backward compatibility + version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -183,6 +183,9 @@ async def _get_atomic_attack_from_jailbreak_async(self, *, jailbreak_template_na Returns: AtomicAttack: An atomic attack using the specified jailbreak template. + + Raises: + ValueError: If scenario is not properly initialized. """ # objective_target is guaranteed to be non-None by parent class validation if self._objective_target is None: diff --git a/pyrit/scenario/scenarios/airt/leakage_scenario.py b/pyrit/scenario/scenarios/airt/leakage_scenario.py index e58b58c11b..aaed955dd3 100644 --- a/pyrit/scenario/scenarios/airt/leakage_scenario.py +++ b/pyrit/scenario/scenarios/airt/leakage_scenario.py @@ -90,7 +90,7 @@ class LeakageScenario(Scenario): """ VERSION: int = 1 - version: int = 1 # Alias for backward compatibility + version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -262,7 +262,7 @@ async def _get_atomic_attack_from_strategy_async(self, strategy: str) -> AtomicA AtomicAttack: Configured for the specified strategy. Raises: - ValueError: If an unknown LeakageStrategy is passed. + ValueError: If scenario is not properly initialized or an unknown LeakageStrategy is passed. """ # objective_target is guaranteed to be non-None by parent class validation if self._objective_target is None: diff --git a/pyrit/scenario/scenarios/airt/scam.py b/pyrit/scenario/scenarios/airt/scam.py index df3531ffd5..39d5d3b1dd 100644 --- a/pyrit/scenario/scenarios/airt/scam.py +++ b/pyrit/scenario/scenarios/airt/scam.py @@ -90,7 +90,7 @@ class Scam(Scenario): """ VERSION: int = 1 - version: int = 1 # Alias for backward compatibility + version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -267,7 +267,7 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: AtomicAttack: Configured for the specified strategy. Raises: - ValueError: If an unknown ScamStrategy is provided. + ValueError: If scenario is not properly initialized or an unknown ScamStrategy is provided. """ # objective_target is guaranteed to be non-None by parent class validation if self._objective_target is None: diff --git a/pyrit/scenario/scenarios/foundry/red_team_agent.py b/pyrit/scenario/scenarios/foundry/red_team_agent.py index a4ec6d32ad..af9b1682f5 100644 --- a/pyrit/scenario/scenarios/foundry/red_team_agent.py +++ b/pyrit/scenario/scenarios/foundry/red_team_agent.py @@ -215,7 +215,7 @@ class RedTeamAgent(Scenario): """ VERSION: int = 1 - version: int = 1 # Alias for backward compatibility + version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> Type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/garak/encoding.py b/pyrit/scenario/scenarios/garak/encoding.py index 3ce929136f..5cbc9a6437 100644 --- a/pyrit/scenario/scenarios/garak/encoding.py +++ b/pyrit/scenario/scenarios/garak/encoding.py @@ -134,7 +134,7 @@ class Encoding(Scenario): """ VERSION: int = 1 - version: int = 1 # Alias for backward compatibility + version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -333,6 +333,9 @@ def _get_prompt_attacks(self, *, converters: list[PromptConverter], encoding_nam Returns: list[AtomicAttack]: List of atomic attacks for this encoding scheme. + + Raises: + ValueError: If scenario is not properly initialized. """ converter_configs = [ AttackConverterConfig( From 63f885ba7e7266511df5f2b6d062ade137ac523c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 11 Feb 2026 18:41:08 +0000 Subject: [PATCH 6/7] Remove version alias and standardize on VERSION only Co-authored-by: ValbuenaVC <50061128+ValbuenaVC@users.noreply.github.com> --- pyrit/scenario/scenarios/airt/content_harms.py | 1 - pyrit/scenario/scenarios/airt/cyber.py | 1 - pyrit/scenario/scenarios/airt/jailbreak.py | 1 - pyrit/scenario/scenarios/airt/leakage_scenario.py | 1 - pyrit/scenario/scenarios/airt/scam.py | 1 - pyrit/scenario/scenarios/foundry/red_team_agent.py | 1 - pyrit/scenario/scenarios/garak/encoding.py | 1 - tests/unit/scenarios/test_content_harms.py | 4 ++-- tests/unit/scenarios/test_cyber.py | 6 +++--- tests/unit/scenarios/test_encoding.py | 2 +- tests/unit/scenarios/test_foundry.py | 2 +- tests/unit/scenarios/test_jailbreak.py | 2 +- tests/unit/scenarios/test_leakage_scenario.py | 6 +++--- tests/unit/scenarios/test_scam.py | 6 +++--- 14 files changed, 14 insertions(+), 21 deletions(-) diff --git a/pyrit/scenario/scenarios/airt/content_harms.py b/pyrit/scenario/scenarios/airt/content_harms.py index ab26409490..6b806cd109 100644 --- a/pyrit/scenario/scenarios/airt/content_harms.py +++ b/pyrit/scenario/scenarios/airt/content_harms.py @@ -105,7 +105,6 @@ class ContentHarms(Scenario): """ VERSION: int = 1 - version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> Type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/airt/cyber.py b/pyrit/scenario/scenarios/airt/cyber.py index e69b95373c..ea8ddf41df 100644 --- a/pyrit/scenario/scenarios/airt/cyber.py +++ b/pyrit/scenario/scenarios/airt/cyber.py @@ -61,7 +61,6 @@ class Cyber(Scenario): """ VERSION: int = 1 - version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index e9092831e9..0478b4ea94 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -50,7 +50,6 @@ class Jailbreak(Scenario): """ VERSION: int = 1 - version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/airt/leakage_scenario.py b/pyrit/scenario/scenarios/airt/leakage_scenario.py index aaed955dd3..0d70f9d9bb 100644 --- a/pyrit/scenario/scenarios/airt/leakage_scenario.py +++ b/pyrit/scenario/scenarios/airt/leakage_scenario.py @@ -90,7 +90,6 @@ class LeakageScenario(Scenario): """ VERSION: int = 1 - version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/airt/scam.py b/pyrit/scenario/scenarios/airt/scam.py index 39d5d3b1dd..2afc2e4b29 100644 --- a/pyrit/scenario/scenarios/airt/scam.py +++ b/pyrit/scenario/scenarios/airt/scam.py @@ -90,7 +90,6 @@ class Scam(Scenario): """ VERSION: int = 1 - version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/foundry/red_team_agent.py b/pyrit/scenario/scenarios/foundry/red_team_agent.py index af9b1682f5..b85242bcd1 100644 --- a/pyrit/scenario/scenarios/foundry/red_team_agent.py +++ b/pyrit/scenario/scenarios/foundry/red_team_agent.py @@ -215,7 +215,6 @@ class RedTeamAgent(Scenario): """ VERSION: int = 1 - version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> Type[ScenarioStrategy]: diff --git a/pyrit/scenario/scenarios/garak/encoding.py b/pyrit/scenario/scenarios/garak/encoding.py index 5cbc9a6437..b4b36e3e8a 100644 --- a/pyrit/scenario/scenarios/garak/encoding.py +++ b/pyrit/scenario/scenarios/garak/encoding.py @@ -134,7 +134,6 @@ class Encoding(Scenario): """ VERSION: int = 1 - version: int = VERSION # Alias for backward compatibility @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/tests/unit/scenarios/test_content_harms.py b/tests/unit/scenarios/test_content_harms.py index bfd0fbf8cd..d0725beb18 100644 --- a/tests/unit/scenarios/test_content_harms.py +++ b/tests/unit/scenarios/test_content_harms.py @@ -242,7 +242,7 @@ async def test_initialization_with_minimal_parameters( # Constructor should set adversarial chat and basic metadata assert scenario._adversarial_chat == mock_adversarial_target assert scenario.name == "Content Harms" - assert scenario.version == 1 + assert scenario.VERSION == 1 # Initialization populates objective target and scenario composites await scenario.initialize_async(objective_target=mock_objective_target) @@ -391,7 +391,7 @@ def test_get_default_scorer(self, mock_objective_target): def test_scenario_version(self): """Test that scenario has correct version.""" - assert ContentHarms.version == 1 + assert ContentHarms.VERSION == 1 @patch.dict( "os.environ", diff --git a/tests/unit/scenarios/test_cyber.py b/tests/unit/scenarios/test_cyber.py index 37eed916d4..5be36d7da2 100644 --- a/tests/unit/scenarios/test_cyber.py +++ b/tests/unit/scenarios/test_cyber.py @@ -140,7 +140,7 @@ def test_init_with_custom_objectives(self, mock_objective_scorer, sample_objecti # objectives are stored as _deprecated_objectives; _seed_groups is resolved lazily assert scenario._deprecated_objectives == sample_objectives assert scenario.name == "Cyber" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_objectives(self, mock_objective_scorer, malware_prompts, mock_memory_seed_groups): """Test initialization with default objectives.""" @@ -151,7 +151,7 @@ def test_init_with_default_objectives(self, mock_objective_scorer, malware_promp # seed_groups are resolved lazily; _deprecated_objectives should be None assert scenario._deprecated_objectives is None assert scenario.name == "Cyber" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_scorer(self, mock_memory_seed_groups): """Test initialization with default scorer.""" @@ -351,7 +351,7 @@ def test_scenario_version_is_set(self, mock_objective_scorer, mock_memory_seed_g objective_scorer=mock_objective_scorer, ) - assert scenario.version == 1 + assert scenario.VERSION == 1 @pytest.mark.asyncio async def test_no_target_duplication(self, mock_objective_target, mock_memory_seed_groups, mock_dataset_config): diff --git a/tests/unit/scenarios/test_encoding.py b/tests/unit/scenarios/test_encoding.py index 699304aa14..5a2aa5be3e 100644 --- a/tests/unit/scenarios/test_encoding.py +++ b/tests/unit/scenarios/test_encoding.py @@ -108,7 +108,7 @@ def test_init_with_custom_seed_prompts(self, mock_objective_target, mock_objecti assert scenario._deprecated_seed_prompts == sample_seeds assert scenario.name == "Encoding" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_seed_prompts(self, mock_objective_target, mock_objective_scorer, mock_memory_seeds): """Test initialization with default seed prompts (Garak dataset).""" diff --git a/tests/unit/scenarios/test_foundry.py b/tests/unit/scenarios/test_foundry.py index 59bcca446b..b093b72a46 100644 --- a/tests/unit/scenarios/test_foundry.py +++ b/tests/unit/scenarios/test_foundry.py @@ -732,7 +732,7 @@ def test_scenario_version_is_set(self, mock_objective_target, mock_objective_sco attack_scoring_config=AttackScoringConfig(objective_scorer=mock_objective_scorer), ) - assert scenario.version == 1 + assert scenario.VERSION == 1 @patch.dict( "os.environ", diff --git a/tests/unit/scenarios/test_jailbreak.py b/tests/unit/scenarios/test_jailbreak.py index 047334131c..88d2aabd15 100644 --- a/tests/unit/scenarios/test_jailbreak.py +++ b/tests/unit/scenarios/test_jailbreak.py @@ -259,7 +259,7 @@ def test_scenario_version_is_set( objective_scorer=mock_objective_scorer, ) - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_scenario_default_dataset(self) -> None: """Test that scenario default dataset is correct.""" diff --git a/tests/unit/scenarios/test_leakage_scenario.py b/tests/unit/scenarios/test_leakage_scenario.py index 29ec0b0bf6..f341e7929e 100644 --- a/tests/unit/scenarios/test_leakage_scenario.py +++ b/tests/unit/scenarios/test_leakage_scenario.py @@ -144,7 +144,7 @@ def test_init_with_custom_objectives(self, mock_objective_scorer, sample_objecti assert len(scenario._objectives) == len(sample_objectives) assert scenario.name == "Leakage Scenario" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_objectives(self, mock_objective_scorer, leakage_prompts, mock_memory_seeds): """Test initialization with default objectives.""" @@ -153,7 +153,7 @@ def test_init_with_default_objectives(self, mock_objective_scorer, leakage_promp assert scenario._objectives == leakage_prompts assert scenario.name == "Leakage Scenario" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_scorer(self, mock_memory_seeds): """Test initialization with default scorer.""" @@ -434,7 +434,7 @@ def test_scenario_version_is_set(self, mock_objective_scorer, sample_objectives) objective_scorer=mock_objective_scorer, ) - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_get_strategy_class_returns_leakage_strategy(self): """Test that get_strategy_class returns LeakageStrategy.""" diff --git a/tests/unit/scenarios/test_scam.py b/tests/unit/scenarios/test_scam.py index a4573f551b..67ab923828 100644 --- a/tests/unit/scenarios/test_scam.py +++ b/tests/unit/scenarios/test_scam.py @@ -148,7 +148,7 @@ def test_init_with_custom_objectives( # objectives are stored as _deprecated_objectives; _seed_groups is resolved lazily assert scenario._deprecated_objectives == sample_objectives assert scenario.name == "Scam" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_objectives( self, @@ -162,7 +162,7 @@ def test_init_with_default_objectives( # seed_groups are resolved lazily; _deprecated_objectives should be None assert scenario._deprecated_objectives is None assert scenario.name == "Scam" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_scorer(self, mock_memory_seed_groups) -> None: """Test initialization with default scorer.""" @@ -373,7 +373,7 @@ def test_scenario_version_is_set( objective_scorer=mock_objective_scorer, ) - assert scenario.version == 1 + assert scenario.VERSION == 1 @pytest.mark.asyncio async def test_no_target_duplication_async( From f63e8c6d959f91fb99d9ab28351dda6a04d8859c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 11 Feb 2026 18:57:43 +0000 Subject: [PATCH 7/7] Fix PEP 8 violations: rename _default_validator to _DEFAULT_VALIDATOR Co-authored-by: ValbuenaVC <50061128+ValbuenaVC@users.noreply.github.com> --- pyrit/score/conversation_scorer.py | 4 ++-- pyrit/score/float_scale/azure_content_filter_scorer.py | 4 ++-- pyrit/score/float_scale/insecure_code_scorer.py | 4 ++-- pyrit/score/float_scale/plagiarism_scorer.py | 4 ++-- .../score/float_scale/self_ask_general_float_scale_scorer.py | 4 ++-- pyrit/score/float_scale/self_ask_likert_scorer.py | 4 ++-- pyrit/score/float_scale/self_ask_scale_scorer.py | 4 ++-- pyrit/score/float_scale/video_float_scale_scorer.py | 4 ++-- pyrit/score/human/human_in_the_loop_gradio.py | 4 ++-- pyrit/score/true_false/decoding_scorer.py | 4 ++-- pyrit/score/true_false/gandalf_scorer.py | 4 ++-- pyrit/score/true_false/markdown_injection.py | 4 ++-- pyrit/score/true_false/prompt_shield_scorer.py | 4 ++-- pyrit/score/true_false/question_answer_scorer.py | 4 ++-- pyrit/score/true_false/self_ask_category_scorer.py | 4 ++-- pyrit/score/true_false/self_ask_general_true_false_scorer.py | 4 ++-- pyrit/score/true_false/self_ask_question_answer_scorer.py | 2 +- pyrit/score/true_false/self_ask_refusal_scorer.py | 4 ++-- pyrit/score/true_false/self_ask_true_false_scorer.py | 4 ++-- pyrit/score/true_false/substring_scorer.py | 4 ++-- pyrit/score/true_false/video_true_false_scorer.py | 4 ++-- 21 files changed, 41 insertions(+), 41 deletions(-) diff --git a/pyrit/score/conversation_scorer.py b/pyrit/score/conversation_scorer.py index 1b93088cb7..a390a0df2b 100644 --- a/pyrit/score/conversation_scorer.py +++ b/pyrit/score/conversation_scorer.py @@ -28,7 +28,7 @@ class ConversationScorer(Scorer, ABC): Note: This class cannot be instantiated directly. Use create_conversation_scorer() factory instead. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], enforce_all_pieces_valid=True, ) @@ -189,7 +189,7 @@ class DynamicConversationScorer(ConversationScorer, scorer_base_class): # type: def __init__(self) -> None: # Initialize with the validator and wrapped scorer - Scorer.__init__(self, validator=validator or ConversationScorer._default_validator) + Scorer.__init__(self, validator=validator or ConversationScorer._DEFAULT_VALIDATOR) self._wrapped_scorer = scorer def _get_wrapped_scorer(self) -> Scorer: diff --git a/pyrit/score/float_scale/azure_content_filter_scorer.py b/pyrit/score/float_scale/azure_content_filter_scorer.py index 9b71f20a31..e40303eb0f 100644 --- a/pyrit/score/float_scale/azure_content_filter_scorer.py +++ b/pyrit/score/float_scale/azure_content_filter_scorer.py @@ -47,7 +47,7 @@ class AzureContentFilterScorer(FloatScaleScorer): MAX_TEXT_LENGTH = 10000 # Azure Content Safety API limit - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text", "image_path"], ) @@ -140,7 +140,7 @@ def __init__( else: raise ValueError("Please provide the Azure Content Safety endpoint and api_key") - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) @property def _category_values(self) -> list[str]: diff --git a/pyrit/score/float_scale/insecure_code_scorer.py b/pyrit/score/float_scale/insecure_code_scorer.py index 6086493d38..5ab98db2fa 100644 --- a/pyrit/score/float_scale/insecure_code_scorer.py +++ b/pyrit/score/float_scale/insecure_code_scorer.py @@ -20,7 +20,7 @@ class InsecureCodeScorer(FloatScaleScorer): Configuration is loaded from a YAML file for dynamic prompts and instructions. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -38,7 +38,7 @@ def __init__( Defaults to the default insecure code scoring prompt if not provided. validator (Optional[ScorerPromptValidator]): Custom validator for the scorer. Defaults to None. """ - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py index f1a36cd13b..1423654bf4 100644 --- a/pyrit/score/float_scale/plagiarism_scorer.py +++ b/pyrit/score/float_scale/plagiarism_scorer.py @@ -32,7 +32,7 @@ class PlagiarismScorer(FloatScaleScorer): 3. Word-level n-gram Jaccard similarity """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -50,7 +50,7 @@ def __init__( n (int): The n-gram size for n-gram similarity. Defaults to 5. validator (Optional[ScorerPromptValidator]): Custom validator for the scorer. Defaults to None. """ - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) self.reference_text = reference_text self.metric = metric diff --git a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py index e1d4829289..50f566effa 100644 --- a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py @@ -18,7 +18,7 @@ class SelfAskGeneralFloatScaleScorer(FloatScaleScorer): system prompt and prompt format. The final score is normalized to [0, 1]. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], is_objective_required=True, ) @@ -69,7 +69,7 @@ def __init__( ValueError: If system_prompt_format_string is not provided or empty. ValueError: If min_value is greater than max_value. """ - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target if not system_prompt_format_string: raise ValueError("system_prompt_format_string must be provided and non-empty.") diff --git a/pyrit/score/float_scale/self_ask_likert_scorer.py b/pyrit/score/float_scale/self_ask_likert_scorer.py index c01fe65073..763484ab03 100644 --- a/pyrit/score/float_scale/self_ask_likert_scorer.py +++ b/pyrit/score/float_scale/self_ask_likert_scorer.py @@ -153,7 +153,7 @@ class SelfAskLikertScorer(FloatScaleScorer): A class that represents a "self-ask" score for text scoring for a likert scale. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -170,7 +170,7 @@ def __init__( likert_scale (LikertScalePaths): The Likert scale configuration to use for scoring. validator (Optional[ScorerPromptValidator]): Custom validator for the scorer. Defaults to None. """ - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target self._likert_scale = likert_scale diff --git a/pyrit/score/float_scale/self_ask_scale_scorer.py b/pyrit/score/float_scale/self_ask_scale_scorer.py index 5e502681d0..05f954c3f0 100644 --- a/pyrit/score/float_scale/self_ask_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_scale_scorer.py @@ -35,7 +35,7 @@ class SystemPaths(enum.Enum): RED_TEAMER_SYSTEM_PROMPT = Path(SCORER_SCALES_PATH, "red_teamer_system_prompt.yaml").resolve() CRITERIA_SYSTEM_PROMPT = Path(SCORER_SCALES_PATH, "criteria_system_prompt.yaml").resolve() - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], is_objective_required=True, ) @@ -59,7 +59,7 @@ def __init__( Defaults to GENERAL_SYSTEM_PROMPT if not provided. validator (Optional[ScorerPromptValidator]): Custom validator for the scorer. Defaults to None. """ - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target diff --git a/pyrit/score/float_scale/video_float_scale_scorer.py b/pyrit/score/float_scale/video_float_scale_scorer.py index 54c81ec1fe..5b01c5ba1e 100644 --- a/pyrit/score/float_scale/video_float_scale_scorer.py +++ b/pyrit/score/float_scale/video_float_scale_scorer.py @@ -30,7 +30,7 @@ class VideoFloatScaleScorer(FloatScaleScorer, _BaseVideoScorer): use FloatScaleScoreAggregator.MAX, FloatScaleScorerAllCategories.MAX, etc. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["video_path"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["video_path"]) def __init__( self, @@ -55,7 +55,7 @@ def __init__( Use FloatScaleScoreAggregator.MAX/AVERAGE/MIN for simple aggregation preserving all categories (returns single score with all categories preserved). """ - FloatScaleScorer.__init__(self, validator=validator or self._default_validator) + FloatScaleScorer.__init__(self, validator=validator or self._DEFAULT_VALIDATOR) _BaseVideoScorer.__init__( self, image_capable_scorer=image_capable_scorer, num_sampled_frames=num_sampled_frames diff --git a/pyrit/score/human/human_in_the_loop_gradio.py b/pyrit/score/human/human_in_the_loop_gradio.py index f9b3226ba9..05031bd9bb 100644 --- a/pyrit/score/human/human_in_the_loop_gradio.py +++ b/pyrit/score/human/human_in_the_loop_gradio.py @@ -21,7 +21,7 @@ class HumanInTheLoopScorerGradio(TrueFalseScorer): In the future this will not be a TrueFalseScorer. However, it is all that is supported currently. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -43,7 +43,7 @@ def __init__( # Import here to avoid importing rpyc in the main module that might not be installed from pyrit.ui.rpc import AppRPCServer - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) self._rpc_server = AppRPCServer(open_browser=open_browser) self._rpc_server.start() diff --git a/pyrit/score/true_false/decoding_scorer.py b/pyrit/score/true_false/decoding_scorer.py index 8034f5a000..17d320fe9c 100644 --- a/pyrit/score/true_false/decoding_scorer.py +++ b/pyrit/score/true_false/decoding_scorer.py @@ -24,7 +24,7 @@ class DecodingScorer(TrueFalseScorer): text matching strategy. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], supported_roles=["assistant"] ) @@ -50,7 +50,7 @@ def __init__( self._text_matcher = text_matcher if text_matcher else ExactTextMatching(case_sensitive=False) self._score_categories = categories if categories else [] - super().__init__(score_aggregator=aggregator, validator=validator or self._default_validator) + super().__init__(score_aggregator=aggregator, validator=validator or self._DEFAULT_VALIDATOR) def _build_identifier(self) -> ScorerIdentifier: """ diff --git a/pyrit/score/true_false/gandalf_scorer.py b/pyrit/score/true_false/gandalf_scorer.py index 932bf49dfe..a4d5cf345b 100644 --- a/pyrit/score/true_false/gandalf_scorer.py +++ b/pyrit/score/true_false/gandalf_scorer.py @@ -29,7 +29,7 @@ class GandalfScorer(TrueFalseScorer): if the password is correct. Returns True if the password was successfully extracted. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -49,7 +49,7 @@ def __init__( score_aggregator (TrueFalseAggregatorFunc): Aggregator for combining scores. Defaults to TrueFalseScoreAggregator.OR. """ - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) self._prompt_target = chat_target self._defender = level.value diff --git a/pyrit/score/true_false/markdown_injection.py b/pyrit/score/true_false/markdown_injection.py index f45fc90f14..f4e87871fd 100644 --- a/pyrit/score/true_false/markdown_injection.py +++ b/pyrit/score/true_false/markdown_injection.py @@ -23,7 +23,7 @@ class MarkdownInjectionScorer(TrueFalseScorer): might be exploited. Returns True if markdown injection is detected. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -41,7 +41,7 @@ def __init__( """ self._category = "security" - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) def _build_identifier(self) -> ScorerIdentifier: """ diff --git a/pyrit/score/true_false/prompt_shield_scorer.py b/pyrit/score/true_false/prompt_shield_scorer.py index edfc697b1d..d8069e44e0 100644 --- a/pyrit/score/true_false/prompt_shield_scorer.py +++ b/pyrit/score/true_false/prompt_shield_scorer.py @@ -27,7 +27,7 @@ class PromptShieldScorer(TrueFalseScorer): scorer_type: ScoreType _prompt_shield_target: PromptShieldTarget - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -47,7 +47,7 @@ def __init__( """ self._prompt_target = prompt_shield_target - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) def _build_identifier(self) -> ScorerIdentifier: """ diff --git a/pyrit/score/true_false/question_answer_scorer.py b/pyrit/score/true_false/question_answer_scorer.py index 63ba8c6c17..1717c8c44f 100644 --- a/pyrit/score/true_false/question_answer_scorer.py +++ b/pyrit/score/true_false/question_answer_scorer.py @@ -22,7 +22,7 @@ class QuestionAnswerScorer(TrueFalseScorer): CORRECT_ANSWER_MATCHING_PATTERNS = ["{correct_answer_index}:", "{correct_answer}"] - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], required_metadata=["correct_answer_index", "correct_answer"] ) @@ -49,7 +49,7 @@ def __init__( self._correct_answer_matching_patterns = correct_answer_matching_patterns self._score_category = category if category is not None else [] - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) def _build_identifier(self) -> ScorerIdentifier: """ diff --git a/pyrit/score/true_false/self_ask_category_scorer.py b/pyrit/score/true_false/self_ask_category_scorer.py index 5697d17c51..9c5b1dd930 100644 --- a/pyrit/score/true_false/self_ask_category_scorer.py +++ b/pyrit/score/true_false/self_ask_category_scorer.py @@ -36,7 +36,7 @@ class SelfAskCategoryScorer(TrueFalseScorer): There is also a false category that is used if the MessagePiece does not fit any of the categories. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator() + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator() def __init__( self, @@ -56,7 +56,7 @@ def __init__( Defaults to TrueFalseScoreAggregator.OR. validator (Optional[ScorerPromptValidator]): Custom validator. Defaults to None. """ - super().__init__(score_aggregator=score_aggregator, validator=validator or self._default_validator) + super().__init__(score_aggregator=score_aggregator, validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target diff --git a/pyrit/score/true_false/self_ask_general_true_false_scorer.py b/pyrit/score/true_false/self_ask_general_true_false_scorer.py index edc1cc5c08..2de626e2a1 100644 --- a/pyrit/score/true_false/self_ask_general_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_general_true_false_scorer.py @@ -22,7 +22,7 @@ class SelfAskGeneralTrueFalseScorer(TrueFalseScorer): system prompt and prompt format. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], is_objective_required=False, ) @@ -71,7 +71,7 @@ def __init__( Raises: ValueError: If system_prompt_format_string is not provided or empty. """ - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) self._prompt_target = chat_target if not system_prompt_format_string: raise ValueError("system_prompt_format_string must be provided and non-empty.") diff --git a/pyrit/score/true_false/self_ask_question_answer_scorer.py b/pyrit/score/true_false/self_ask_question_answer_scorer.py index 0cb0e704db..93a1dcfcc6 100644 --- a/pyrit/score/true_false/self_ask_question_answer_scorer.py +++ b/pyrit/score/true_false/self_ask_question_answer_scorer.py @@ -26,7 +26,7 @@ class SelfAskQuestionAnswerScorer(SelfAskTrueFalseScorer): to objective target or you need more flexibility in determining if the questions were answered correctly. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], is_objective_required=True, ) diff --git a/pyrit/score/true_false/self_ask_refusal_scorer.py b/pyrit/score/true_false/self_ask_refusal_scorer.py index 4341bdc33f..70b7274bed 100644 --- a/pyrit/score/true_false/self_ask_refusal_scorer.py +++ b/pyrit/score/true_false/self_ask_refusal_scorer.py @@ -32,7 +32,7 @@ class SelfAskRefusalScorer(TrueFalseScorer): evaluating whether AI systems are appropriately refusing harmful requests. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator() + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator() def __init__( self, @@ -60,7 +60,7 @@ def __init__( result_file="refusal_scorer/refusal_metrics.jsonl", ) - super().__init__(score_aggregator=score_aggregator, validator=validator or self._default_validator) + super().__init__(score_aggregator=score_aggregator, validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target self._system_prompt_with_objective = ( diff --git a/pyrit/score/true_false/self_ask_true_false_scorer.py b/pyrit/score/true_false/self_ask_true_false_scorer.py index 45f323d0ca..80cb967892 100644 --- a/pyrit/score/true_false/self_ask_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_true_false_scorer.py @@ -87,7 +87,7 @@ class SelfAskTrueFalseScorer(TrueFalseScorer): If no descriptions are provided, it defaults to the TASK_ACHIEVED scorer. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text", "image_path"], ) @@ -117,7 +117,7 @@ def __init__( ValueError: If both true_false_question_path and true_false_question are provided. ValueError: If required keys are missing in true_false_question. """ - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) self._prompt_target = chat_target diff --git a/pyrit/score/true_false/substring_scorer.py b/pyrit/score/true_false/substring_scorer.py index e4548378b1..59607c07de 100644 --- a/pyrit/score/true_false/substring_scorer.py +++ b/pyrit/score/true_false/substring_scorer.py @@ -22,7 +22,7 @@ class SubStringScorer(TrueFalseScorer): Supports both exact substring matching and approximate matching. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -49,7 +49,7 @@ def __init__( self._text_matcher = text_matcher if text_matcher else ExactTextMatching(case_sensitive=False) self._score_categories = categories if categories else [] - super().__init__(score_aggregator=aggregator, validator=validator or self._default_validator) + super().__init__(score_aggregator=aggregator, validator=validator or self._DEFAULT_VALIDATOR) def _build_identifier(self) -> ScorerIdentifier: """ diff --git a/pyrit/score/true_false/video_true_false_scorer.py b/pyrit/score/true_false/video_true_false_scorer.py index 2d50d780e8..1d2ecec707 100644 --- a/pyrit/score/true_false/video_true_false_scorer.py +++ b/pyrit/score/true_false/video_true_false_scorer.py @@ -23,7 +23,7 @@ class VideoTrueFalseScorer(TrueFalseScorer, _BaseVideoScorer): meaning if any frame meets the objective, the entire video is scored as True). """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["video_path"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["video_path"]) def __init__( self, @@ -47,7 +47,7 @@ def __init__( ) TrueFalseScorer.__init__( - self, validator=validator or self._default_validator, score_aggregator=score_aggregator + self, validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator ) def _build_identifier(self) -> ScorerIdentifier: