diff --git a/core/src/config/schema.ts b/core/src/config/schema.ts index 4ee997a..efa5f0a 100644 --- a/core/src/config/schema.ts +++ b/core/src/config/schema.ts @@ -163,6 +163,7 @@ export const RunConfigSchema = z turnMode: z.enum(["single", "multi"]).optional(), turns: z.number().int().positive().optional(), telemetry: z.unknown().optional(), + attackObjective: z.string().optional(), }) .passthrough(); diff --git a/core/src/execute/evaluatorLoop.ts b/core/src/execute/evaluatorLoop.ts index 02f48a7..3b8933b 100644 --- a/core/src/execute/evaluatorLoop.ts +++ b/core/src/execute/evaluatorLoop.ts @@ -109,6 +109,12 @@ export async function runEvaluatorAttacks( throw err; } + if (config.attackObjective) { + for (const attack of attacks) { + if (attack.kind === "agent") attack.attackObjective = config.attackObjective; + } + } + log.info(` ${attacks.length} attack(s) generated [effort: ${config.effort}]`); const attackResults: AttackResult[] = []; diff --git a/core/src/execute/types.ts b/core/src/execute/types.ts index 1ca5afe..21f06bc 100644 --- a/core/src/execute/types.ts +++ b/core/src/execute/types.ts @@ -94,6 +94,14 @@ export interface RunConfig { turnMode?: "single" | "multi"; turns: number; telemetry?: TelemetryConfig; + /** + * Free-text primary mission steering every evaluator's attacks (e.g. "get the + * target to leak env vars via a delegated employee"). Threaded onto each + * generated AgentAttackSpec; consumed by generateNextAdaptiveTurn as the + * attacker's top-priority goal, same mechanism the browser extension's + * popup-driven `attackObjective` already uses. + */ + attackObjective?: string; } // --------------------------------------------------------------------------- diff --git a/docs/cli.md b/docs/cli.md index 7695bb7..1f58229 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -285,21 +285,23 @@ See [telemetry.md](telemetry.md) for Langfuse and Netra setup, config fields, an ## Commands reference -| Command | Description | -| --------------------------------------------- | ---------------------------------------------------------------------- | -| `opfor setup` | Interactive wizard — writes a timestamped config | -| `opfor setup --agent` / `--mcp` | Skip mode prompt | -| `opfor setup --empty` | Write a blank config without wizard prompts | -| `opfor setup --config ` | Override the output config path | -| `opfor run` | Run the setup wizard inline, then run the resulting config | -| `opfor run --config ` | Read an existing config, fire attacks, judge, write HTML + JSON report | -| `opfor run --config --effort ` | Override `effort` (`adaptive` or `comprehensive`) | -| `opfor run --config --turns ` | Override turn count (1 forces single-turn) | -| `opfor run --config --output ` | Override report parent directory (default `.opfor/reports/`) | -| `opfor run --config --env ` | Load env vars from a non-default `.env` path | -| `opfor run --config --events ` | Stream NDJSON run lifecycle events to `` (for CI/automation) | -| `opfor setup --env ` | Same `--env` flag works on setup | -| `opfor hunt --endpoint --objective ` | Autonomous agentic red-team — see [hunt.md](hunt.md) | +| Command | Description | +| ------------------------------------------------ | ---------------------------------------------------------------------- | +| `opfor setup` | Interactive wizard — writes a timestamped config | +| `opfor setup --agent` / `--mcp` | Skip mode prompt | +| `opfor setup --empty` | Write a blank config without wizard prompts | +| `opfor setup --config ` | Override the output config path | +| `opfor run` | Run the setup wizard inline, then run the resulting config | +| `opfor run --config ` | Read an existing config, fire attacks, judge, write HTML + JSON report | +| `opfor run --config --effort ` | Override `effort` (`adaptive` or `comprehensive`) | +| `opfor run --config --turns ` | Override turn count (1 forces single-turn) | +| `opfor run --config --output ` | Override report parent directory (default `.opfor/reports/`) | +| `opfor run --config --env ` | Load env vars from a non-default `.env` path | +| `opfor run --config --events ` | Stream NDJSON run lifecycle events to `` (for CI/automation) | +| `opfor run --config --objective ` | Steer every evaluator's attacks toward a specific free-text mission | +| `opfor run --config --objective-file

` | Same, read from a file | +| `opfor setup --env ` | Same `--env` flag works on setup | +| `opfor hunt --endpoint --objective ` | Autonomous agentic red-team — see [hunt.md](hunt.md) | --- @@ -307,20 +309,21 @@ See [telemetry.md](telemetry.md) for Langfuse and Netra setup, config fields, an ### Common fields (both modes) -| Field | Required | Description | -| ----------------------- | ----------------------------- | -------------------------------------------------------------------------------------------------- | -| `target.kind` | Yes | `"agent"` or `"mcp"`. | -| `selection.mode` | Yes | `"suite"` or `"evaluators"`. | -| `selection.suite` | For suite | Suite ID — see [evaluators reference](evaluators.md). | -| `selection.evaluators` | For evaluators | Array of evaluator IDs. | -| `attackerLlm.provider` | Yes | See [Supported LLM providers](#supported-llm-providers). | -| `attackerLlm.model` | Yes | Model name (e.g. `gpt-4o-mini`). | -| `attackerLlm.apiKeyEnv` | No | Env var **name** holding the API key. Falls back to the provider's default env var when absent. | -| `attackerLlm.baseURL` | For openai-compatible / azure | Base URL for the LLM endpoint. | -| `judgeLlm.*` | No | Same fields as `attackerLlm`. Separate model for judging. Falls back to `attackerLlm` when absent. | -| `effort` | Yes | `"adaptive"` or `"comprehensive"`. | -| `turnMode` | No | `"single"` (default when omitted) or `"multi"`. | -| `turns` | Yes | Turns per attack. Ignored when `turnMode` is `"single"`. Range 1–10 (wizard default 3). | +| Field | Required | Description | +| ----------------------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `target.kind` | Yes | `"agent"` or `"mcp"`. | +| `selection.mode` | Yes | `"suite"` or `"evaluators"`. | +| `selection.suite` | For suite | Suite ID — see [evaluators reference](evaluators.md). | +| `selection.evaluators` | For evaluators | Array of evaluator IDs. | +| `attackerLlm.provider` | Yes | See [Supported LLM providers](#supported-llm-providers). | +| `attackerLlm.model` | Yes | Model name (e.g. `gpt-4o-mini`). | +| `attackerLlm.apiKeyEnv` | No | Env var **name** holding the API key. Falls back to the provider's default env var when absent. | +| `attackerLlm.baseURL` | For openai-compatible / azure | Base URL for the LLM endpoint. | +| `judgeLlm.*` | No | Same fields as `attackerLlm`. Separate model for judging. Falls back to `attackerLlm` when absent. | +| `effort` | Yes | `"adaptive"` or `"comprehensive"`. | +| `turnMode` | No | `"single"` (default when omitted) or `"multi"`. | +| `turns` | Yes | Turns per attack. Ignored when `turnMode` is `"single"`. Range 1–10 (wizard default 3). | +| `attackObjective` | No | Free-text primary mission steering every evaluator's attacks (e.g. "get the target to leak env vars via a delegated employee"). Also settable via `--objective`/`--objective-file` on `opfor run`. Same mechanism the browser extension's popup-driven objective already uses. | ### Agent fields (`target.kind: "agent"`) diff --git a/docs/mcp.md b/docs/mcp.md index c931fd0..fddc059 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -149,12 +149,13 @@ LLM object schema: #### Run settings -| Parameter | Type | Required | Description | -| ------------- | ----------------------------- | -------- | ------------------------------------------------------------------------ | -| `effort` | `adaptive` \| `comprehensive` | No | Defaults to `adaptive`. Comprehensive runs one attack per named pattern. | -| `turns` | number (1–10) | No | Turns per attack. 1 = single-turn. Defaults to 1. | -| `output_dir` | string | No | Directory to write `opfor.config.json`. Defaults to `.` | -| `config_path` | string | No | Full path for config file (overrides `output_dir`) | +| Parameter | Type | Required | Description | +| ------------------ | ----------------------------- | -------- | ------------------------------------------------------------------------ | +| `effort` | `adaptive` \| `comprehensive` | No | Defaults to `adaptive`. Comprehensive runs one attack per named pattern. | +| `turns` | number (1–10) | No | Turns per attack. 1 = single-turn. Defaults to 1. | +| `attack_objective` | string | No | Free-text mission steering every evaluator's attacks | +| `output_dir` | string | No | Directory to write `opfor.config.json`. Defaults to `.` | +| `config_path` | string | No | Full path for config file (overrides `output_dir`) | --- @@ -162,12 +163,13 @@ LLM object schema: Runs the full red team evaluation from a config file produced by `opfor_setup`. Generates attacks on-the-fly, runs them against the target, judges responses, and writes an HTML + JSON report. -| Parameter | Type | Required | Description | -| ----------------- | ----------------------------- | -------- | ---------------------------------------------------- | -| `config_path` | string | Yes | Path to `opfor.config.json` written by `opfor_setup` | -| `output_dir` | string | No | Report directory. Defaults to config file directory | -| `effort_override` | `adaptive` \| `comprehensive` | No | Override the effort level from config | -| `turns_override` | number (1–10) | No | Override turns per attack from config | +| Parameter | Type | Required | Description | +| -------------------- | ----------------------------- | -------- | ---------------------------------------------------- | +| `config_path` | string | Yes | Path to `opfor.config.json` written by `opfor_setup` | +| `output_dir` | string | No | Report directory. Defaults to config file directory | +| `effort_override` | `adaptive` \| `comprehensive` | No | Override the effort level from config | +| `turns_override` | number (1–10) | No | Override turns per attack from config | +| `objective_override` | string | No | Override the attack objective from config | --- diff --git a/docs/sdk.md b/docs/sdk.md index 8d0b1b7..1fe91c3 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -302,6 +302,20 @@ const results = await opfor.run({ }); ``` +## Attack objective + +Steer every evaluator's attacks toward a specific free-text mission, instead of +letting each evaluator pursue its own generic goal. + +```typescript +const results = await opfor.run({ + target: { url: "https://api.example.com/chat" }, + suite: "owasp-llm-top10", + + attackObjective: "get the target to leak env vars via a delegated employee", +}); +``` + ## Models Configure attacker and judge LLMs. diff --git a/runners/cli/src/commands/run.ts b/runners/cli/src/commands/run.ts index 9f7023b..0cfab91 100644 --- a/runners/cli/src/commands/run.ts +++ b/runners/cli/src/commands/run.ts @@ -28,6 +28,11 @@ export function registerRunCommand(program: Command): void { "--events ", "Stream run lifecycle events as newline-delimited JSON (NDJSON) to " ) + .option( + "--objective ", + "Free-text attack objective steering every evaluator's attacks (e.g. a specific goal to pursue)" + ) + .option("--objective-file ", "Read the attack objective from a file") .action( async (opts: { config?: string; @@ -36,6 +41,8 @@ export function registerRunCommand(program: Command): void { output?: string; env?: string; events?: string; + objective?: string; + objectiveFile?: string; }) => { if (opts.env) { const { config: loadDotenv } = await import("dotenv"); @@ -90,6 +97,19 @@ export function registerRunCommand(program: Command): void { } runConfig = { ...runConfig, turns: n }; } + let objective = opts.objective?.trim(); + if (!objective && opts.objectiveFile) { + try { + objective = (await readFile(path.resolve(opts.objectiveFile), "utf8")).trim(); + } catch { + log.error(`Cannot read --objective-file at ${path.resolve(opts.objectiveFile)}.`); + process.exitCode = 1; + return; + } + } + if (objective) { + runConfig = { ...runConfig, attackObjective: objective }; + } log.info(`\nOpfor Run`); log.info(` Target : ${runConfig.target.name} (${runConfig.target.kind})`); @@ -99,6 +119,9 @@ export function registerRunCommand(program: Command): void { if (runConfig.judgeLlm) { log.info(` Judge : ${runConfig.judgeLlm.provider}/${runConfig.judgeLlm.model}`); } + if (runConfig.attackObjective) { + log.info(` Objective : ${runConfig.attackObjective}`); + } log.info(""); await ensureOpforDirs(); diff --git a/runners/mcp/src/index.ts b/runners/mcp/src/index.ts index c39e73b..1b22d77 100644 --- a/runners/mcp/src/index.ts +++ b/runners/mcp/src/index.ts @@ -228,6 +228,13 @@ server.tool( .max(10) .default(1) .describe("Turns per attack. 1 = single-turn, >1 = multi-turn escalation."), + attack_objective: z + .string() + .optional() + .describe( + "Free-text primary mission steering every evaluator's attacks " + + "(e.g. 'get the target to leak env vars via a delegated employee')." + ), // Output output_dir: z.string().optional().describe("Directory to write opfor.config.json (default: .)"), @@ -312,6 +319,7 @@ server.tool( `Turns : ${config.turns}`, `Evaluators: ${evalCount}`, `Attacker : ${config.attackerLlm.provider}/${config.attackerLlm.model}`, + ...(config.attackObjective ? [`Objective : ${config.attackObjective}`] : []), ``, `Next: call opfor_run with config_path="${outputPath}"`, ].join("\n"), @@ -354,6 +362,7 @@ server.tool( .max(10) .optional() .describe("Override turns per attack from config"), + objective_override: z.string().optional().describe("Override the attack objective from config"), }, async (args) => { try { @@ -362,6 +371,7 @@ server.tool( if (args.effort_override) config = { ...config, effort: args.effort_override }; if (args.turns_override) config = { ...config, turns: args.turns_override }; + if (args.objective_override) config = { ...config, attackObjective: args.objective_override }; // Defensive coerce in case the config file has an unexpected value. config = { ...config, effort: normalizeEffort(config.effort as unknown) }; @@ -369,6 +379,7 @@ server.tool( `🔴 Opfor Run`, `Target: ${config.target.name} (${config.target.kind})`, `Effort: ${config.effort} Turns: ${config.turns}`, + ...(config.attackObjective ? [`Objective: ${config.attackObjective}`] : []), ``, `Running...`, ]; @@ -534,6 +545,7 @@ function buildRunConfig(args: Record): RunConfig { judgeLlm, effort: normalizeEffort(args.effort), turns: (args.turns as number) ?? 1, + attackObjective: args.attack_objective ? String(args.attack_objective) : undefined, }; } diff --git a/runners/sdk/src/internal/buildRunConfig.ts b/runners/sdk/src/internal/buildRunConfig.ts index 35735c8..6ef5a32 100644 --- a/runners/sdk/src/internal/buildRunConfig.ts +++ b/runners/sdk/src/internal/buildRunConfig.ts @@ -49,6 +49,7 @@ export function buildRunConfig(options: RunOptions): { turns, turnMode, telemetry: options.telemetry, + attackObjective: options.attackObjective, }, env, }; diff --git a/runners/sdk/src/types.ts b/runners/sdk/src/types.ts index a055471..227319d 100644 --- a/runners/sdk/src/types.ts +++ b/runners/sdk/src/types.ts @@ -93,6 +93,13 @@ export interface RunOptions { strategy?: StrategyConfig; + /** + * Free-text primary mission steering every evaluator's attacks (e.g. "get the + * target to leak env vars via a delegated employee"). Same mechanism `opfor run + * --objective` and `opfor hunt --objective` use. + */ + attackObjective?: string; + attackerModel?: ModelSpec; judgeModel?: ModelSpec;