diff --git a/.github/workflows/frontend_tests.yml b/.github/workflows/frontend_tests.yml index 2db2f4811d..e016093e80 100644 --- a/.github/workflows/frontend_tests.yml +++ b/.github/workflows/frontend_tests.yml @@ -101,10 +101,12 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps chromium - - name: Run E2E tests (mock mode — no backend/secrets needed) - run: npx playwright test --project=mock + - name: Run E2E tests (mock and seeded — no credentials needed) + run: npx playwright test --fail-on-flaky-tests --project=mock --project=seeded env: CI: true + E2E_SEEDED_MODE: true + FAIL_ON_SKIPPED_TESTS: true - name: Upload Playwright report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -150,3 +152,6 @@ jobs: - name: Run TypeScript type check run: npx tsc --noEmit + + - name: Build production frontend + run: npm run build diff --git a/frontend/README.md b/frontend/README.md index d834549d10..2e50f0ea76 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -91,20 +91,20 @@ E2E flow tests run in two modes controlled by Playwright projects and an environ - **Seeded** (`--project seeded`, default for CI): Messages are stored directly in the database with `send: false` using dummy credentials. No real API keys needed. Tests cover the full UI flow (display, branching, conversation switching, promoting) without calling any external service. -- **Live** (`--project live`, requires `E2E_LIVE_MODE=true`): Messages are sent to real OpenAI endpoints with `send: true`. Each target variant requires its own set of environment variables (e.g., `OPENAI_CHAT_ENDPOINT`, `OPENAI_CHAT_KEY`, `OPENAI_CHAT_MODEL`). Variants whose env vars are missing are automatically skipped. Tests verify that real target responses render correctly. +- **Live** (`--project live`, requires `E2E_LIVE_MODE=true`): Messages are sent to real OpenAI endpoints with `send: true`. Each target variant requires endpoint and model environment variables plus either an API key or an Azure endpoint accessible through the current Entra identity. Variants without a usable configuration are automatically skipped. Tests verify that real target responses render correctly. ```bash -# CI (seeded only — no credentials needed) +# Seeded integration (no credentials needed) npx playwright test --project seeded -# Live integration (requires real API keys) +# Live integration (uses API keys when present, otherwise Entra authentication) E2E_LIVE_MODE=true npx playwright test --project live # Run both E2E_LIVE_MODE=true npx playwright test ``` -The seeded project runs in the **GitHub Actions** workflow. The live project is intended for an **Azure DevOps pipeline** that has the required secret API keys. +The mock and seeded projects run in the **GitHub Actions** pull-request workflow. The live project is intended for a protected pipeline with an Entra identity or API keys. E2E tests use `dev.py` to automatically start both frontend and backend servers. If servers are already running, they will be reused. diff --git a/frontend/e2e/api.spec.ts b/frontend/e2e/api.spec.ts index f0daa144b2..c21b95cbb6 100644 --- a/frontend/e2e/api.spec.ts +++ b/frontend/e2e/api.spec.ts @@ -1,6 +1,6 @@ import { test, expect } from "@playwright/test"; -// API tests go through the Vite dev server proxy (/api -> backend:8000) +// API tests go through the Vite dev server proxy (/api -> configured backend) // rather than hitting the backend directly, so they work as soon as // Playwright's webServer (port 3000) is ready. @@ -68,8 +68,10 @@ test.describe("Targets API", () => { }); test("should create and retrieve a target @seeded", async ({ request }) => { + test.setTimeout(90_000); const createPayload = { type: "OpenAIChatTarget", + auth_mode: "api_key", params: { endpoint: "https://e2e-test.openai.azure.com", model_name: "gpt-4o-e2e-test", @@ -77,17 +79,15 @@ test.describe("Targets API", () => { }, }; - const createResp = await request.post("/api/targets", { data: createPayload }); - // The endpoint may require credentials or env setup that isn't available - // in CI. Skip gracefully rather than masking real regressions. - if (!createResp.ok()) { - test.skip(true, `POST /api/targets returned ${createResp.status()} — skipping`); - return; - } + const createResp = await request.post("/api/targets", { + data: createPayload, + timeout: 60_000, + }); + expect(createResp.ok()).toBe(true); const created = await createResp.json(); expect(created).toHaveProperty("target_registry_name"); - expect(created.target_type).toBe("OpenAIChatTarget"); + expect(created.identifier.class_name).toBe("OpenAIChatTarget"); // Retrieve via list and check it's there const listResp = await request.get("/api/targets?limit=200"); @@ -120,12 +120,6 @@ test.describe("Attacks API", () => { test("should list attacks @seeded", async ({ request }) => { const response = await request.get("/api/attacks"); - // Backend may return 500 due to stale DB schema or 404 if not implemented. - // Only assert when the endpoint is actually healthy. - if (!response.ok()) { - test.skip(true, `GET /api/attacks returned ${response.status()} — skipping`); - return; - } expect(response.ok()).toBe(true); }); }); diff --git a/frontend/e2e/chat.spec.ts b/frontend/e2e/chat.spec.ts index 05f75a44b8..d951b0c0e7 100644 --- a/frontend/e2e/chat.spec.ts +++ b/frontend/e2e/chat.spec.ts @@ -500,8 +500,7 @@ test.describe("Multi-modal: Video response", () => { }, ]); - // Marking skipped for now because this should not use the getByRole query which is too general - test.skip("should display video player for video response", async ({ page }) => { + test("should display video player for video response", async ({ page }) => { await setupVideoMock(page); await page.goto("/"); await activateMockTarget(page); diff --git a/frontend/e2e/flows.spec.ts b/frontend/e2e/flows.spec.ts index 4bce02db0c..69f3cacc23 100644 --- a/frontend/e2e/flows.spec.ts +++ b/frontend/e2e/flows.spec.ts @@ -1,4 +1,14 @@ -import { test, expect, type Page, type APIRequestContext } from "@playwright/test"; +import { readFileSync } from "node:fs"; + +import { + test, + expect, + type APIRequestContext, + type Locator, + type Page, +} from "@playwright/test"; + +import type { AddMessageResponse } from "@/types"; // --------------------------------------------------------------------------- // Mode detection @@ -9,6 +19,25 @@ import { test, expect, type Page, type APIRequestContext } from "@playwright/tes * Without it, only seeded tests run (safe for CI, no credentials needed). */ const LIVE_MODE = process.env.E2E_LIVE_MODE === "true"; +const AZURE_OPENAI_HOSTNAME_SUFFIXES = [ + ".openai.azure.com", + ".ai.azure.com", + ".services.ai.azure.com", + ".cognitiveservices.azure.com", +]; + +type AuthMode = "api_key" | "identity"; + +function isAzureOpenAiEndpoint(endpoint: string): boolean { + try { + const hostname = new URL(endpoint).hostname.toLowerCase(); + return AZURE_OPENAI_HOSTNAME_SUFFIXES.some((suffix) => + hostname.endsWith(suffix), + ); + } catch { + return false; + } +} // --------------------------------------------------------------------------- // Helpers - shared between seeded and live modes @@ -36,9 +65,10 @@ async function createTarget( request: APIRequestContext, targetType: string, params: Record = {}, + authMode: AuthMode = "api_key", ): Promise { const resp = await request.post("/api/targets", { - data: { type: targetType, params }, + data: { type: targetType, params, auth_mode: authMode }, }); expect(resp.ok()).toBeTruthy(); const body = await resp.json(); @@ -113,6 +143,20 @@ async function sendMessage( { data }, ); expect(resp.ok()).toBeTruthy(); + const body: AddMessageResponse = await resp.json(); + const messages = body.messages.messages; + const assistantMessage = messages[messages.length - 1]; + if (!assistantMessage || assistantMessage.role !== "assistant") { + throw new Error("Target response did not include an assistant message"); + } + const responseErrors = assistantMessage.message_pieces + .map((piece) => piece.response_error) + .filter((errorType) => errorType !== "none"); + if (responseErrors.length > 0) { + throw new Error( + `Target returned response errors: ${responseErrors.join(", ")}`, + ); + } } /** Convenience: store a text-only message. */ @@ -152,14 +196,19 @@ async function createConversation( return body.conversation_id; } -/** Activate a target via the Configuration view so the chat UI is unlocked. */ -async function activateTarget(page: Page, targetType: string): Promise { +/** Activate an exact target instance via the Configuration view. */ +async function activateTarget( + page: Page, + targetRegistryName: string, +): Promise { await page.getByTitle("Configuration").click(); await expect(page.getByText("Target Configuration")).toBeVisible({ timeout: 10_000 }); - // The table displays target_type (not registry name), so match by type. - // Use .first() because multiple targets of the same type may exist. - const row = page.locator("tr", { has: page.getByText(targetType, { exact: true }) }).first(); - await row.getByRole("button", { name: /set active/i }).click(); + const row = page.getByTestId(`target-row-${targetRegistryName}`); + await expect(row).toBeVisible({ timeout: 10_000 }); + const setActiveButton = row.getByRole("button", { name: /set active/i }); + if (await setActiveButton.isVisible()) { + await setActiveButton.click(); + } await page.getByTitle("Chat").click(); await expect(page.getByTestId("new-attack-btn")).toBeVisible({ timeout: 5_000 }); } @@ -177,6 +226,13 @@ async function openAttackInHistory( const row = page.getByTestId(`attack-row-${attackResultId}`); await expect(row).toBeVisible({ timeout: 10_000 }); await row.click(); + await expect(page).toHaveURL(new RegExp(`/attacks/${attackResultId}$`)); + await expect(page.getByTestId("toggle-panel-btn")).toBeEnabled({ + timeout: 10_000, + }); + await expect(page.getByTestId("loading-state")).not.toBeVisible({ + timeout: 10_000, + }); } /** Open the conversation side-panel (idempotent — does nothing if already open). */ @@ -187,13 +243,20 @@ async function openConversationPanel(page: Page): Promise { await expect(panel).toBeVisible({ timeout: 5_000 }); } +/** Locate text only within rendered chat messages, excluding history previews. */ +function getMessageText(page: Page, text: string): Locator { + return page + .locator('[data-testid^="message-bubble-"]') + .getByText(text, { exact: true }); +} + // --------------------------------------------------------------------------- // Target variant configurations // --------------------------------------------------------------------------- -// Minimal 1x1 red PNG as base64 -const TINY_PNG = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4DwkAAwAB/QHRAYAAAAABJRU5ErkJggg=="; +const INPUT_IMAGE_BASE64 = readFileSync( + new URL("../public/roakey.png", import.meta.url), +).toString("base64"); const DUMMY_OPENAI_PARAMS = { endpoint: "https://e2e-dummy.openai.azure.com", @@ -209,11 +272,12 @@ interface TargetVariant { targetType: string; /** Constructor kwargs for seeded mode (dummy credentials). */ targetParams: Record; - /** - * Environment variables that must ALL be set for live mode. - * If any is missing, the live test is skipped for this variant. - */ - liveEnvVars: string[]; + /** Environment variables used to configure the target in live mode. */ + liveEnvironment: { + endpoint: string; + apiKey: string; + model: string; + }; /** Whether the target supports multi-turn conversations. */ multiTurn: boolean; /** User turn pieces (seeded mode uses these directly). */ @@ -244,11 +308,11 @@ const TARGET_VARIANTS: TargetVariant[] = [ targetType: "OpenAIChatTarget", targetParams: DUMMY_OPENAI_PARAMS, multiTurn: true, - liveEnvVars: [ - "OPENAI_CHAT_ENDPOINT", - "OPENAI_CHAT_KEY", - "OPENAI_CHAT_MODEL", - ], + liveEnvironment: { + endpoint: "OPENAI_CHAT_ENDPOINT", + apiKey: "OPENAI_CHAT_KEY", + model: "OPENAI_CHAT_MODEL", + }, userPieces: [{ data_type: "text", original_value: "Hello chat" }], assistantPieces: [ { data_type: "text", original_value: "Chat text response" }, @@ -261,16 +325,16 @@ const TARGET_VARIANTS: TargetVariant[] = [ targetType: "OpenAIChatTarget", targetParams: DUMMY_OPENAI_PARAMS, multiTurn: true, - liveEnvVars: [ - "OPENAI_CHAT_ENDPOINT", - "OPENAI_CHAT_KEY", - "OPENAI_CHAT_MODEL", - ], + liveEnvironment: { + endpoint: "OPENAI_CHAT_ENDPOINT", + apiKey: "OPENAI_CHAT_KEY", + model: "OPENAI_CHAT_MODEL", + }, userPieces: [ { data_type: "text", original_value: "Describe this image" }, { data_type: "image_path", - original_value: TINY_PNG, + original_value: INPUT_IMAGE_BASE64, mime_type: "image/png", }, ], @@ -285,18 +349,18 @@ const TARGET_VARIANTS: TargetVariant[] = [ targetType: "OpenAIImageTarget", targetParams: DUMMY_OPENAI_PARAMS, multiTurn: false, - liveEnvVars: [ - "OPENAI_IMAGE_ENDPOINT", - "OPENAI_IMAGE_API_KEY", - "OPENAI_IMAGE_MODEL", - ], + liveEnvironment: { + endpoint: "OPENAI_IMAGE_ENDPOINT", + apiKey: "OPENAI_IMAGE_API_KEY", + model: "OPENAI_IMAGE_MODEL", + }, userPieces: [ { data_type: "text", original_value: "Generate a red dot" }, ], assistantPieces: [ { data_type: "image_path", - original_value: TINY_PNG, + original_value: INPUT_IMAGE_BASE64, mime_type: "image/png", }, ], @@ -308,23 +372,23 @@ const TARGET_VARIANTS: TargetVariant[] = [ targetType: "OpenAIImageTarget", targetParams: DUMMY_OPENAI_PARAMS, multiTurn: false, - liveEnvVars: [ - "OPENAI_IMAGE_ENDPOINT", - "OPENAI_IMAGE_API_KEY", - "OPENAI_IMAGE_MODEL", - ], + liveEnvironment: { + endpoint: "OPENAI_IMAGE_ENDPOINT", + apiKey: "OPENAI_IMAGE_API_KEY", + model: "OPENAI_IMAGE_MODEL", + }, userPieces: [ { data_type: "text", original_value: "Edit this image" }, { data_type: "image_path", - original_value: TINY_PNG, + original_value: INPUT_IMAGE_BASE64, mime_type: "image/png", }, ], assistantPieces: [ { data_type: "image_path", - original_value: TINY_PNG, + original_value: INPUT_IMAGE_BASE64, mime_type: "image/png", }, ], @@ -336,11 +400,11 @@ const TARGET_VARIANTS: TargetVariant[] = [ targetType: "OpenAIVideoTarget", targetParams: DUMMY_OPENAI_PARAMS, multiTurn: false, - liveEnvVars: [ - "OPENAI_VIDEO_ENDPOINT", - "OPENAI_VIDEO_KEY", - "OPENAI_VIDEO_MODEL", - ], + liveEnvironment: { + endpoint: "OPENAI_VIDEO_ENDPOINT", + apiKey: "OPENAI_VIDEO_KEY", + model: "OPENAI_VIDEO_MODEL", + }, userPieces: [ { data_type: "text", original_value: "Generate a video" }, ], @@ -359,11 +423,11 @@ const TARGET_VARIANTS: TargetVariant[] = [ targetType: "OpenAITTSTarget", targetParams: DUMMY_OPENAI_PARAMS, multiTurn: false, - liveEnvVars: [ - "OPENAI_TTS_ENDPOINT", - "OPENAI_TTS_KEY", - "OPENAI_TTS_MODEL", - ], + liveEnvironment: { + endpoint: "OPENAI_TTS_ENDPOINT", + apiKey: "OPENAI_TTS_KEY", + model: "OPENAI_TTS_MODEL", + }, userPieces: [{ data_type: "text", original_value: "Say hello" }], assistantPieces: [ { @@ -380,11 +444,11 @@ const TARGET_VARIANTS: TargetVariant[] = [ targetType: "OpenAIResponseTarget", targetParams: DUMMY_OPENAI_PARAMS, multiTurn: true, - liveEnvVars: [ - "OPENAI_RESPONSES_ENDPOINT", - "OPENAI_RESPONSES_KEY", - "OPENAI_RESPONSES_MODEL", - ], + liveEnvironment: { + endpoint: "OPENAI_RESPONSES_ENDPOINT", + apiKey: "OPENAI_RESPONSES_KEY", + model: "OPENAI_RESPONSES_MODEL", + }, userPieces: [ { data_type: "text", original_value: "Hello responses API" }, ], @@ -399,11 +463,11 @@ const TARGET_VARIANTS: TargetVariant[] = [ targetType: "OpenAIResponseTarget", targetParams: DUMMY_OPENAI_PARAMS, multiTurn: true, - liveEnvVars: [ - "OPENAI_RESPONSES_ENDPOINT", - "OPENAI_RESPONSES_KEY", - "OPENAI_RESPONSES_MODEL", - ], + liveEnvironment: { + endpoint: "OPENAI_RESPONSES_ENDPOINT", + apiKey: "OPENAI_RESPONSES_KEY", + model: "OPENAI_RESPONSES_MODEL", + }, userPieces: [ { data_type: "text", @@ -411,7 +475,7 @@ const TARGET_VARIANTS: TargetVariant[] = [ }, { data_type: "image_path", - original_value: TINY_PNG, + original_value: INPUT_IMAGE_BASE64, mime_type: "image/png", }, ], @@ -433,7 +497,7 @@ async function assertSeededAssistant( exp: TargetVariant["expectAssistantSeeded"], ): Promise { if (exp.text) { - await expect(page.getByText(exp.text)).toBeVisible({ + await expect(getMessageText(page, exp.text)).toBeVisible({ timeout: 10_000, }); } @@ -463,27 +527,22 @@ async function assertLiveAssistant( page: Page, exp: TargetVariant["expectAssistantLive"], ): Promise { + const assistantBubble = page.getByTestId("message-bubble-1"); if (exp.hasText) { - // At least one assistant bubble with non-empty text content - await expect( - page - .locator('[class*="assistantMessage"], [class*="assistantBubble"]') - .first(), - ).toBeVisible({ timeout: 90_000 }); + await expect(assistantBubble).toContainText(/\S/, { timeout: 90_000 }); } if (exp.hasImage) { - const imgs = page.locator( - '[class*="assistantMessage"] img, [class*="assistantBubble"] img', - ); - await expect(imgs.first()).toBeVisible({ timeout: 90_000 }); + await expect(assistantBubble.locator("img").first()).toBeVisible({ + timeout: 90_000, + }); } if (exp.hasVideo) { - await expect(page.locator("video").first()).toBeVisible({ + await expect(assistantBubble.locator("video").first()).toBeVisible({ timeout: 90_000, }); } if (exp.hasAudio) { - await expect(page.locator("audio").first()).toBeVisible({ + await expect(assistantBubble.locator("audio").first()).toBeVisible({ timeout: 90_000, }); } @@ -519,9 +578,18 @@ async function seedFullTurn( return attack; } -/** Check if all required env vars for a live variant are present. */ -function hasLiveCredentials(variant: TargetVariant): boolean { - return variant.liveEnvVars.every((v) => !!process.env[v]); +/** Check whether live mode can use an API key or Azure identity. */ +function hasLiveConfiguration(variant: TargetVariant): boolean { + const { endpoint, apiKey, model } = variant.liveEnvironment; + const endpointValue = process.env[endpoint]; + if (!endpointValue || !process.env[model]) { + return false; + } + return !!process.env[apiKey] || isAzureOpenAiEndpoint(endpointValue); +} + +function getLiveAuthMode(variant: TargetVariant): AuthMode { + return process.env[variant.liveEnvironment.apiKey] ? "api_key" : "identity"; } // --------------------------------------------------------------------------- @@ -546,7 +614,7 @@ for (const variant of TARGET_VARIANTS) { test.beforeEach(async ({ page }) => { await page.goto("/"); - await activateTarget(page, variant.targetType); + await activateTarget(page, targetRegistryName); }); test("should display seeded messages @seeded", async ({ @@ -567,15 +635,18 @@ for (const variant of TARGET_VARIANTS) { ); if (userText) { await expect( - page.getByText(userText.original_value), + getMessageText(page, userText.original_value), ).toBeVisible({ timeout: 10_000 }); } // Assert user image if (variant.userPieces.some((p) => p.data_type === "image_path")) { - await expect(page.locator("img").first()).toBeVisible({ - timeout: 10_000, - }); + await expect( + page + .locator('[data-testid^="message-bubble-"]') + .locator("img") + .first(), + ).toBeVisible({ timeout: 10_000 }); } // Assert assistant response @@ -598,7 +669,7 @@ for (const variant of TARGET_VARIANTS) { ); if (userText) { await expect( - page.getByText(userText.original_value), + getMessageText(page, userText.original_value), ).toBeVisible({ timeout: 10_000 }); } @@ -612,7 +683,7 @@ for (const variant of TARGET_VARIANTS) { if (userText) { await expect( - page.getByText(userText.original_value), + getMessageText(page, userText.original_value), ).not.toBeVisible({ timeout: 5_000 }); } }); @@ -642,7 +713,7 @@ for (const variant of TARGET_VARIANTS) { ); if (userText) { await expect( - page.getByText(userText.original_value), + getMessageText(page, userText.original_value), ).toBeVisible({ timeout: 10_000 }); } // Deterministically ensure main conversation is active: open the @@ -652,23 +723,23 @@ for (const variant of TARGET_VARIANTS) { await page.getByTestId(`conversation-item-${conversationId}`).click(); if (userText) { await expect( - page.getByText(userText.original_value), + getMessageText(page, userText.original_value), ).toBeVisible({ timeout: 5_000 }); } await page.getByTestId("toggle-panel-btn").click(); await expect(page.getByTestId("conversation-panel")).not.toBeVisible({ timeout: 3_000 }); await expect( - page.getByText("Branch-only text message"), + getMessageText(page, "Branch-only text message"), ).not.toBeVisible(); await openConversationPanel(page); await page.getByTestId(`conversation-item-${newConversationId}`).click(); await expect( - page.getByText("Branch-only text message").first(), + getMessageText(page, "Branch-only text message"), ).toBeVisible({ timeout: 5_000 }); if (userText) { await expect( - page.getByText(userText.original_value), + getMessageText(page, userText.original_value), ).not.toBeVisible(); } @@ -677,7 +748,7 @@ for (const variant of TARGET_VARIANTS) { .click(); if (userText) { await expect( - page.getByText(userText.original_value), + getMessageText(page, userText.original_value), ).toBeVisible({ timeout: 5_000 }); } }); @@ -701,7 +772,7 @@ for (const variant of TARGET_VARIANTS) { ); if (userText) { await expect( - page.getByText(userText.original_value), + getMessageText(page, userText.original_value), ).toBeVisible({ timeout: 10_000 }); } else { await page.waitForTimeout(3_000); @@ -759,7 +830,7 @@ for (const variant of TARGET_VARIANTS) { const expText = variant.expectAssistantSeeded.text; if (expText) { - await expect(page.getByText(expText)).toBeVisible({ + await expect(getMessageText(page, expText)).toBeVisible({ timeout: 10_000, }); } else { @@ -859,7 +930,7 @@ for (const variant of TARGET_VARIANTS) { // Wait for the chat view to fully load const expText = variant.expectAssistantSeeded.text; if (expText) { - await expect(page.getByText(expText)).toBeVisible({ + await expect(getMessageText(page, expText)).toBeVisible({ timeout: 10_000, }); } else { @@ -881,7 +952,7 @@ for (const variant of TARGET_VARIANTS) { .getByTestId(`conversation-item-${branchConversationId}`) .click(); - await expect(page.getByText("Second turn")).not.toBeVisible({ + await expect(getMessageText(page, "Second turn")).not.toBeVisible({ timeout: 5_000, }); @@ -915,19 +986,24 @@ for (const variant of TARGET_VARIANTS) { let targetRegistryName: string; test.beforeAll(async ({ request }) => { - if (!hasLiveCredentials(variant)) return; + if (!hasLiveConfiguration(variant)) return; await waitForBackend(request); - // In live mode, create target without explicit creds - the backend - // picks them up from environment variables automatically. - targetRegistryName = await createTarget(request, variant.targetType); + targetRegistryName = await createTarget( + request, + variant.targetType, + {}, + getLiveAuthMode(variant), + ); }); test.beforeEach(async ({ page }) => { test.skip( - !hasLiveCredentials(variant), - "Missing required env vars for " + variant.label, + !hasLiveConfiguration(variant), + "Missing endpoint/model and API key or Azure identity configuration for " + + variant.label, ); await page.goto("/"); + await activateTarget(page, targetRegistryName); }); test("should send a real message and display the response @live", async ({ @@ -992,9 +1068,16 @@ for (const variant of TARGET_VARIANTS) { await openAttackInHistory(page, attackResultId); await assertLiveAssistant(page, variant.expectAssistantLive); - const branchBtn = page.getByTestId("branch-conv-btn-1"); - await expect(branchBtn).toBeVisible({ timeout: 10_000 }); - await branchBtn.click(); + if (variant.multiTurn) { + const branchBtn = page.getByTestId("branch-conv-btn-1"); + await expect(branchBtn).toBeVisible({ timeout: 10_000 }); + await branchBtn.click(); + } else { + await createConversation(request, attackResultId, { + sourceConversationId: conversationId, + cutoffIndex: 1, + }); + } await expect .poll( @@ -1049,25 +1132,19 @@ for (const variant of TARGET_VARIANTS) { await openAttackInHistory(page, attackResultId); await assertLiveAssistant(page, variant.expectAssistantLive); - const branchBtn = page.getByTestId("branch-conv-btn-1"); - await expect(branchBtn).toBeVisible({ timeout: 10_000 }); - await branchBtn.click(); + const branchConversationId = await createConversation( + request, + attackResultId, + { sourceConversationId: conversationId, cutoffIndex: 1 }, + ); await openConversationPanel(page); const items = page.locator('[data-testid^="conversation-item-"]'); await expect(items).toHaveCount(2, { timeout: 15_000 }); // 5. Switch to branch - const convResp = await request.get( - `/api/attacks/${encodeURIComponent(attackResultId)}/conversations`, - ); - const convData = await convResp.json(); - const branchConv = convData.conversations.find( - (c: { conversation_id: string }) => c.conversation_id !== convData.main_conversation_id, - ); - expect(branchConv).toBeDefined(); await page - .getByTestId(`conversation-item-${branchConv.conversation_id}`) + .getByTestId(`conversation-item-${branchConversationId}`) .click(); // Second-turn messages should not be in branch @@ -1077,7 +1154,7 @@ for (const variant of TARGET_VARIANTS) { // 6. Promote await page - .getByTestId(`star-btn-${branchConv.conversation_id}`) + .getByTestId(`star-btn-${branchConversationId}`) .click(); await expect @@ -1091,7 +1168,7 @@ for (const variant of TARGET_VARIANTS) { }, { timeout: 10_000 }, ) - .toBe(branchConv.conversation_id); + .toBe(branchConversationId); }); }); } diff --git a/frontend/e2e/noSkippedTestsReporter.ts b/frontend/e2e/noSkippedTestsReporter.ts new file mode 100644 index 0000000000..c7bfd6ffcc --- /dev/null +++ b/frontend/e2e/noSkippedTestsReporter.ts @@ -0,0 +1,31 @@ +import type { + Reporter, + TestCase, + TestResult, +} from "@playwright/test/reporter"; + +class NoSkippedTestsReporter implements Reporter { + private readonly skippedTests: string[] = []; + + onTestEnd(test: TestCase, result: TestResult): void { + if ( + process.env.FAIL_ON_SKIPPED_TESTS === "true" && + result.status === "skipped" + ) { + this.skippedTests.push(test.titlePath().filter(Boolean).join(" > ")); + } + } + + onEnd(): { status: "failed" } | void { + if (this.skippedTests.length === 0) { + return; + } + + console.error( + `Unexpected skipped tests:\n${this.skippedTests.map((title) => `- ${title}`).join("\n")}`, + ); + return { status: "failed" }; + } +} + +export default NoSkippedTestsReporter; diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index f00c3e0440..8edf8735e0 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -1,12 +1,21 @@ import { defineConfig, devices } from "@playwright/test"; +const CI_SEEDED_MODE = + !!process.env.CI && process.env.E2E_SEEDED_MODE === "true"; +const E2E_BACKEND_PORT = process.env.PYRIT_E2E_BACKEND_PORT ?? "18000"; +const E2E_BACKEND_URL = `http://127.0.0.1:${E2E_BACKEND_PORT}`; + export default defineConfig({ testDir: "./e2e", fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, workers: process.env.CI ? 1 : undefined, - reporter: [["html", { open: "never" }], ["list"]], + reporter: [ + ["html", { open: "never" }], + ["list"], + ["./e2e/noSkippedTestsReporter.ts"], + ], timeout: 30000, use: { @@ -52,16 +61,34 @@ export default defineConfig({ ], /* Automatically start servers before running tests */ - webServer: { - // CI runs only the mock project (no backend needed) — start Vite directly. - // Locally, dev.py starts both backend + frontend for seeded/live tests. - command: process.env.CI - ? "npx vite --port 3000" - : "python dev.py", - // Use 127.0.0.1 to avoid Node.js 17+ resolving localhost to IPv6 ::1 - url: "http://127.0.0.1:3000", - reuseExistingServer: !process.env.CI, - // CI needs extra time for uv sync + backend startup - timeout: 120_000, - }, + webServer: CI_SEEDED_MODE + ? [ + { + command: + `cd .. && uv run python -m pyrit.backend.pyrit_backend ` + + `--host 127.0.0.1 --port ${E2E_BACKEND_PORT} --log-level warning ` + + "--config-file tests/end_to_end/test_config.yaml", + env: { PYRIT_DEV_MODE: "true" }, + url: `${E2E_BACKEND_URL}/api/health`, + reuseExistingServer: false, + timeout: 120_000, + }, + { + command: "npx vite --host 127.0.0.1 --port 3000 --strictPort", + env: { PYRIT_BACKEND_URL: E2E_BACKEND_URL }, + // Use 127.0.0.1 to avoid Node.js 17+ resolving localhost to IPv6 ::1 + url: "http://127.0.0.1:3000", + reuseExistingServer: false, + timeout: 120_000, + }, + ] + : { + // Mock CI needs only Vite. Local seeded/live runs use dev.py. + command: process.env.CI + ? "npx vite --port 3000" + : "python dev.py", + url: "http://127.0.0.1:3000", + reuseExistingServer: !process.env.CI, + timeout: 120_000, + }, }); diff --git a/frontend/src/components/Config/TargetTable.tsx b/frontend/src/components/Config/TargetTable.tsx index d78f807721..a4b3a176d1 100644 --- a/frontend/src/components/Config/TargetTable.tsx +++ b/frontend/src/components/Config/TargetTable.tsx @@ -402,6 +402,7 @@ export default function TargetTable({ targets, activeTarget, onSetActiveTarget } {isActive(target) ? ( diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 1c367dfdf8..235174dc93 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -6,11 +6,12 @@ import path from 'path' // Without this, Vite logs dozens of "http proxy error" stack traces. const logger = createLogger() const originalError = logger.error +const backendUrl = process.env.PYRIT_BACKEND_URL ?? 'http://127.0.0.1:8000' let proxyWarned = false logger.error = (msg, options) => { if (typeof msg === 'string' && msg.includes('http proxy error')) { if (!proxyWarned) { - console.log('[vite] Waiting for backend on port 8000...') + console.log(`[vite] Waiting for backend at ${backendUrl}...`) proxyWarned = true } return @@ -40,7 +41,7 @@ export default defineConfig({ proxy: { '/api': { // Use 127.0.0.1 to avoid Node.js 17+ resolving localhost to IPv6 ::1 - target: 'http://127.0.0.1:8000', + target: backendUrl, changeOrigin: true, // Return 502 on proxy errors so in-flight requests fail fast // instead of hanging until the backend comes up. diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py index 0b6b3b2acb..9101fea87b 100644 --- a/pyrit/prompt_target/openai/openai_response_target.py +++ b/pyrit/prompt_target/openai/openai_response_target.py @@ -6,6 +6,7 @@ from collections.abc import Awaitable, Callable, MutableSequence from enum import Enum from typing import ( + TYPE_CHECKING, Any, Literal, cast, @@ -37,6 +38,9 @@ from pyrit.prompt_target.openai.openai_error_handling import _is_content_filter_error from pyrit.prompt_target.openai.openai_target import OpenAITarget +if TYPE_CHECKING: + from openai.types.responses import ResponseInputImageParam + logger = logging.getLogger(__name__) @@ -242,7 +246,12 @@ async def _construct_input_item_from_piece_async(self, piece: MessagePiece) -> d } if piece.converted_value_data_type == "image_path": data_url = await convert_local_image_to_data_url_async(piece.converted_value) - return {"type": "input_image", "image_url": {"url": data_url}} + image_item: ResponseInputImageParam = { + "detail": "auto", + "type": "input_image", + "image_url": data_url, + } + return dict(image_item) raise ValueError(f"Unsupported piece type for inline content: {piece.converted_value_data_type}") async def _build_input_for_multi_modal_async(self, conversation: MutableSequence[Message]) -> list[dict[str, Any]]: diff --git a/tests/integration/targets/test_entra_auth_targets.py b/tests/integration/targets/test_entra_auth_targets.py index 9ae437e8a4..3e4a7e4024 100644 --- a/tests/integration/targets/test_entra_auth_targets.py +++ b/tests/integration/targets/test_entra_auth_targets.py @@ -13,6 +13,7 @@ ) from pyrit.common.path import HOME_PATH from pyrit.executor.attack import PromptSendingAttack +from pyrit.memory.sqlite_memory import SQLiteMemory from pyrit.models import Message, MessagePiece from pyrit.prompt_target import ( OpenAIChatTarget, @@ -231,6 +232,41 @@ async def test_openai_responses_target_entra_auth(sqlite_instance, endpoint, mod assert result.last_response is not None +@pytest.mark.run_only_if_all_tests +async def test_openai_responses_target_image_entra_auth(sqlite_instance: SQLiteMemory) -> None: + endpoint = os.environ["OPENAI_RESPONSES_ENDPOINT"] + target = OpenAIResponseTarget( + endpoint=endpoint, + model_name=os.environ["OPENAI_RESPONSES_MODEL"], + api_key=get_azure_openai_auth(endpoint), + ) + conversation_id = str(uuid.uuid4()) + message = Message( + message_pieces=[ + MessagePiece( + role="user", + original_value="Briefly describe this image.", + original_value_data_type="text", + conversation_id=conversation_id, + ), + MessagePiece( + role="user", + original_value=str(SAMPLE_IMAGE_FILE), + original_value_data_type="image_path", + conversation_id=conversation_id, + ), + ] + ) + + result = await target.send_prompt_async(message=message) + + assert result + assert any( + piece.response_error == "none" and piece.converted_value_data_type == "text" + for piece in result[0].message_pieces + ) + + @pytest.mark.parametrize( ("endpoint", "model_name"), [ diff --git a/tests/unit/prompt_target/target/test_openai_response_target.py b/tests/unit/prompt_target/target/test_openai_response_target.py index d0fbd1e131..42f0d0266c 100644 --- a/tests/unit/prompt_target/target/test_openai_response_target.py +++ b/tests/unit/prompt_target/target/test_openai_response_target.py @@ -671,7 +671,8 @@ async def test_build_input_for_multi_modal_async_image_and_text(target: OpenAIRe assert result[0]["role"] == "user" assert result[0]["content"][0]["type"] == "input_text" assert result[0]["content"][1]["type"] == "input_image" - assert result[0]["content"][1]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert result[0]["content"][1]["detail"] == "auto" + assert result[0]["content"][1]["image_url"].startswith("data:image/jpeg;base64,") async def test_construct_request_body_filters_none(