From 42157aa31639275b728107d7a81bd539fa12367a Mon Sep 17 00:00:00 2001 From: Phyllis Wu Date: Thu, 2 Apr 2026 17:24:51 -0400 Subject: [PATCH] E2E: cleanup utility --- package.json | 9 +- packages/e2e/scripts/cleanup.ts | 286 ++++++++++++++++++ packages/e2e/tests/app-deploy.spec.ts | 7 +- packages/e2e/tests/app-dev-server.spec.ts | 7 +- packages/e2e/tests/app-scaffold.spec.ts | 21 +- packages/e2e/tests/dev-hot-reload.spec.ts | 21 +- packages/e2e/tests/multi-config-dev.spec.ts | 14 +- .../e2e/tests/toml-config-invalid.spec.ts | 5 +- packages/e2e/tests/toml-config.spec.ts | 14 +- 9 files changed, 357 insertions(+), 27 deletions(-) create mode 100644 packages/e2e/scripts/cleanup.ts diff --git a/package.json b/package.json index cd81b6826d4..f12b050de86 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "shopify:run": "node packages/cli/bin/dev.js", "shopify": "nx build cli && node packages/cli/bin/dev.js", "test:e2e": "nx run-many --target=build --projects=cli,create-app --skip-nx-cache && pnpm --filter e2e exec playwright test", + "test:e2e-cleanup": "npx tsx packages/e2e/scripts/cleanup.ts", "test:regenerate-snapshots": "packages/e2e/scripts/regenerate-snapshots.sh", "test": "pnpm vitest run", "type-check:affected": "nx affected --target=type-check", @@ -145,9 +146,13 @@ "unresolved": "error" }, "ignoreBinaries": [ - "playwright" + "playwright", + "tsx" + ], + "ignoreDependencies": [ + "dotenv", + "@playwright/test" ], - "ignoreDependencies": [], "ignoreWorkspaces": [ "packages/eslint-plugin-cli", "packages/e2e" diff --git a/packages/e2e/scripts/cleanup.ts b/packages/e2e/scripts/cleanup.ts new file mode 100644 index 00000000000..1fd41b3cb32 --- /dev/null +++ b/packages/e2e/scripts/cleanup.ts @@ -0,0 +1,286 @@ +/* eslint-disable no-console, no-restricted-imports, no-await-in-loop */ + +/** + * E2E Cleanup Utility + * + * Finds and deletes leftover E2E test apps from the Dev Dashboard. + * Apps are matched by the "E2E-" prefix in their name. + * + * Usage: + * npx tsx packages/e2e/scripts/cleanup.ts # Full cleanup: uninstall + delete + * npx tsx packages/e2e/scripts/cleanup.ts --list # List matching apps without action + * npx tsx packages/e2e/scripts/cleanup.ts --uninstall # Uninstall from all stores only (no delete) + * npx tsx packages/e2e/scripts/cleanup.ts --delete # Delete only (skip uninstall — delete only apps with 0 installs) + * npx tsx packages/e2e/scripts/cleanup.ts --headed # Show browser window + * npx tsx packages/e2e/scripts/cleanup.ts --pattern X # Match apps containing "X" (default: "E2E-") + * + * Environment variables (loaded from packages/e2e/.env): + * E2E_ACCOUNT_EMAIL — Shopify account email for login + * E2E_ACCOUNT_PASSWORD — Shopify account password + * E2E_ORG_ID — Organization ID to scan for apps + * + * This module also exports `cleanupAllApps()` for use as a Playwright globalTeardown + * or from other scripts/tests. + */ + +import {config} from 'dotenv' +import * as path from 'path' +import {fileURLToPath} from 'url' +import {chromium} from '@playwright/test' +import {navigateToDashboard} from '../setup/browser.js' +import {findAppsOnDashboard, uninstallApp, deleteApp} from '../setup/app.js' +import type {Page} from '@playwright/test' + +// Load .env from packages/e2e/ (not cwd) only if not already configured +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +if (!process.env.E2E_ACCOUNT_EMAIL) { + config({path: path.resolve(__dirname, '../.env')}) +} + +// --------------------------------------------------------------------------- +// Core cleanup logic — reusable from tests, teardown, or CLI +// --------------------------------------------------------------------------- + +export type CleanupMode = 'full' | 'list' | 'uninstall' | 'delete' + +const MODE_LABELS: Record = { + full: 'Uninstall + Delete', + list: 'List only', + uninstall: 'Uninstall only', + delete: 'Delete only', +} + +export interface CleanupOptions { + /** Cleanup mode (default: "full" — uninstall + delete) */ + mode?: CleanupMode + /** App name pattern to match (default: "E2E-") */ + pattern?: string + /** Show browser window */ + headed?: boolean + /** Organization ID (default: from E2E_ORG_ID env) */ + orgId?: string + /** Max retries per app on failure (default: 2) */ + retries?: number +} + +/** + * Find and delete all E2E test apps matching a pattern. + * Handles browser login, dashboard navigation, uninstall, and deletion. + */ +export async function cleanupAllApps(opts: CleanupOptions = {}): Promise { + const mode = opts.mode ?? 'full' + const pattern = opts.pattern ?? 'E2E-' + const orgId = opts.orgId ?? (process.env.E2E_ORG_ID ?? '').trim() + const maxRetries = opts.retries ?? 2 + const email = process.env.E2E_ACCOUNT_EMAIL + const password = process.env.E2E_ACCOUNT_PASSWORD + + // Banner + console.log('') + console.log(`[cleanup] Mode: ${MODE_LABELS[mode]}`) + console.log(`[cleanup] Org: ${orgId || '(not set)'}`) + console.log(`[cleanup] Pattern: "${pattern}"`) + console.log('') + + if (!email || !password) { + throw new Error('E2E_ACCOUNT_EMAIL and E2E_ACCOUNT_PASSWORD are required') + } + + if (!orgId) { + throw new Error('E2E_ORG_ID is required') + } + + const browser = await chromium.launch({headless: !opts.headed}) + const context = await browser.newContext({ + extraHTTPHeaders: { + 'X-Shopify-Loadtest-Bf8d22e7-120e-4b5b-906c-39ca9d5499a9': 'true', + }, + }) + context.setDefaultTimeout(30_000) + context.setDefaultNavigationTimeout(30_000) + const page = await context.newPage() + + try { + // Step 1: Log into Shopify directly in the browser + console.log('[cleanup] Logging in...') + await browserLogin(page, email, password) + + // Step 2: Navigate to dashboard + console.log('[cleanup] Navigating to dashboard...') + await navigateToDashboard({browserPage: page, email, orgId}) + + // Step 3: Find matching apps + const apps = await findAppsOnDashboard({browserPage: page, namePattern: pattern}) + console.log(`[cleanup] Found ${apps.length} app(s)`) + console.log('') + + if (apps.length === 0) return + + for (let i = 0; i < apps.length; i++) { + console.log(` ${i + 1}. ${apps[i]!.name}`) + } + console.log('') + + if (mode === 'list') return + + // Step 4: Process each app with retries + let succeeded = 0 + let skipped = 0 + let failed = 0 + + for (let i = 0; i < apps.length; i++) { + const app = apps[i]! + const tag = `[cleanup] [${i + 1}/${apps.length}]` + let ok = false + let wasSkipped = false + + console.log(`${tag} ${app.name}`) + + for (let attempt = 1; attempt <= maxRetries + 1; attempt++) { + try { + if (attempt > 1) { + console.log(` Retry ${attempt - 1}/${maxRetries}...`) + await navigateToDashboard({browserPage: page, email, orgId}) + } + + if (mode === 'full' || mode === 'uninstall') { + const noInstalls = await checkNoInstalls(page, app.url) + if (noInstalls) { + console.log(' Not installed') + } else { + console.log(' Uninstalling...') + const allUninstalled = await uninstallApp({browserPage: page, appUrl: app.url, appName: app.name, orgId}) + if (!allUninstalled) { + throw new Error('Uninstall incomplete — some stores may remain') + } + console.log(' Uninstalled') + } + } + + if (mode === 'full' || mode === 'delete') { + if (mode === 'delete') { + const noInstalls = await checkNoInstalls(page, app.url) + if (!noInstalls) { + console.log(' Delete skipped (still installed)') + wasSkipped = true + skipped++ + break + } + } + console.log(' Deleting...') + await deleteApp({browserPage: page, appUrl: app.url}) + console.log(' Deleted') + } + + ok = true + break + } catch (err) { + const msg = err instanceof Error ? err.message : String(err) + if (attempt <= maxRetries) { + console.warn(` Attempt ${attempt} failed: ${msg}`) + await page.waitForTimeout(3000) + } else { + console.warn(` Failed: ${msg}`) + } + } + } + + if (ok) succeeded++ + else if (!wasSkipped) failed++ + console.log('') + } + + // Summary + const parts = [`${succeeded} succeeded`] + if (skipped > 0) parts.push(`${skipped} skipped`) + if (failed > 0) parts.push(`${failed} failed`) + console.log('') + console.log(`[cleanup] Complete: ${parts.join(', ')}`) + } finally { + await browser.close() + } +} + +// --------------------------------------------------------------------------- +// Browser-only login — go to accounts.shopify.com directly +// --------------------------------------------------------------------------- + +async function browserLogin(page: Page, email: string, password: string): Promise { + await page.goto('https://accounts.shopify.com/lookup', {waitUntil: 'domcontentloaded'}) + + // Fill email + await page.waitForSelector('input[name="account[email]"], input[type="email"]', {timeout: 30_000}) + await page.locator('input[name="account[email]"], input[type="email"]').first().fill(email) + await page.locator('button[type="submit"]').first().click() + + // Fill password + await page.waitForSelector('input[name="account[password]"], input[type="password"]', {timeout: 30_000}) + await page.locator('input[name="account[password]"], input[type="password"]').first().fill(password) + await page.locator('button[type="submit"]').first().click() + + // Wait for login to complete + await page.waitForTimeout(3000) + + console.log('[cleanup] Logged in successfully.') +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Check if an app has no store installs (by visiting its installs page). */ +async function checkNoInstalls(page: Page, appUrl: string): Promise { + await page.goto(`${appUrl}/installs`, {waitUntil: 'domcontentloaded'}) + await page.waitForTimeout(3000) + + const emptyStatePatterns = ['no install', 'not installed', '0 install'] + const rows = await page.locator('table tbody tr').all() + + if (rows.length === 0) { + // No table rows — check body text for empty state indicators + const bodyText = (await page.textContent('body'))?.toLowerCase() ?? '' + return emptyStatePatterns.some((pattern) => bodyText.includes(pattern)) + } + + for (const row of rows) { + const text = (await row.locator('td').first().textContent())?.trim().toLowerCase() ?? '' + if (text && !emptyStatePatterns.some((pattern) => text.includes(pattern))) return false + } + return true +} + +// --------------------------------------------------------------------------- +// CLI entry point +// --------------------------------------------------------------------------- + +async function main() { + const args = process.argv.slice(2) + const headed = args.includes('--headed') + const patternIdx = args.indexOf('--pattern') + let pattern: string | undefined + if (patternIdx !== -1) { + const nextArg = args[patternIdx + 1] + if (!nextArg || nextArg.startsWith('--')) { + console.error('[cleanup] --pattern requires a value') + process.exitCode = 1 + return + } + pattern = nextArg + } + + let mode: CleanupMode = 'full' + if (args.includes('--list')) mode = 'list' + else if (args.includes('--uninstall')) mode = 'uninstall' + else if (args.includes('--delete')) mode = 'delete' + + await cleanupAllApps({mode, pattern, headed}) +} + +// Run if executed directly (not imported) +const isDirectRun = process.argv[1] === fileURLToPath(import.meta.url) +if (isDirectRun) { + main().catch((err) => { + console.error('[cleanup] Fatal error:', err) + process.exitCode = 1 + }) +} diff --git a/packages/e2e/tests/app-deploy.spec.ts b/packages/e2e/tests/app-deploy.spec.ts index 4e85bd524be..68d74a9e0e5 100644 --- a/packages/e2e/tests/app-deploy.spec.ts +++ b/packages/e2e/tests/app-deploy.spec.ts @@ -39,8 +39,11 @@ test.describe('App deploy', () => { expect(listResult.exitCode, `versions list failed:\n${listOutput}`).toBe(0) expect(listOutput).toContain(versionTag) } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) }) diff --git a/packages/e2e/tests/app-dev-server.spec.ts b/packages/e2e/tests/app-dev-server.spec.ts index 358553b458d..a9dba4feeee 100644 --- a/packages/e2e/tests/app-dev-server.spec.ts +++ b/packages/e2e/tests/app-dev-server.spec.ts @@ -43,8 +43,11 @@ test.describe('App dev server', () => { const exitCode = await dev.waitForExit(30_000) expect(exitCode).toBe(0) } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) }) diff --git a/packages/e2e/tests/app-scaffold.spec.ts b/packages/e2e/tests/app-scaffold.spec.ts index 5155fd55510..411fe2c5e04 100644 --- a/packages/e2e/tests/app-scaffold.spec.ts +++ b/packages/e2e/tests/app-scaffold.spec.ts @@ -38,8 +38,11 @@ test.describe('App scaffold', () => { const buildResult = await buildApp({cli, appDir}) expect(buildResult.exitCode, `build failed:\nstderr: ${buildResult.stderr}`).toBe(0) } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) @@ -63,8 +66,11 @@ test.describe('App scaffold', () => { expect(fs.existsSync(initResult.appDir)).toBe(true) expect(fs.existsSync(path.join(initResult.appDir, 'shopify.app.toml'))).toBe(true) } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) @@ -105,8 +111,11 @@ test.describe('App scaffold', () => { const buildResult = await buildApp({cli, appDir}) expect(buildResult.exitCode, `build failed:\nstderr: ${buildResult.stderr}`).toBe(0) } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) }) diff --git a/packages/e2e/tests/dev-hot-reload.spec.ts b/packages/e2e/tests/dev-hot-reload.spec.ts index 33c7b6e4118..f3e04ef9534 100644 --- a/packages/e2e/tests/dev-hot-reload.spec.ts +++ b/packages/e2e/tests/dev-hot-reload.spec.ts @@ -84,8 +84,11 @@ test.describe('Dev hot reload', () => { proc.kill() } } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) @@ -127,8 +130,11 @@ test.describe('Dev hot reload', () => { proc.kill() } } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) @@ -176,8 +182,11 @@ test.describe('Dev hot reload', () => { proc.kill() } } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) }) diff --git a/packages/e2e/tests/multi-config-dev.spec.ts b/packages/e2e/tests/multi-config-dev.spec.ts index 4bb00985f12..5b4a68c990c 100644 --- a/packages/e2e/tests/multi-config-dev.spec.ts +++ b/packages/e2e/tests/multi-config-dev.spec.ts @@ -80,8 +80,11 @@ include_config_on_deploy = true proc.kill() } } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) @@ -142,8 +145,11 @@ api_version = "2025-01" proc.kill() } } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) }) diff --git a/packages/e2e/tests/toml-config-invalid.spec.ts b/packages/e2e/tests/toml-config-invalid.spec.ts index df455be7cc2..5f930d08db5 100644 --- a/packages/e2e/tests/toml-config-invalid.spec.ts +++ b/packages/e2e/tests/toml-config-invalid.spec.ts @@ -35,7 +35,10 @@ test.describe('TOML config invalid', () => { expect(result.exitCode, `expected deploy to fail for ${label}, but it succeeded:\n${output}`).not.toBe(0) expect(output.toLowerCase(), `expected error output for ${label}:\n${output}`).toMatch(/error|invalid|failed/) } finally { - fs.rmSync(appDir, {recursive: true, force: true}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(appDir, {recursive: true, force: true}) + } } }) } diff --git a/packages/e2e/tests/toml-config.spec.ts b/packages/e2e/tests/toml-config.spec.ts index 7d478aa9d0b..318b7a9872e 100644 --- a/packages/e2e/tests/toml-config.spec.ts +++ b/packages/e2e/tests/toml-config.spec.ts @@ -32,8 +32,11 @@ test.describe('TOML config regression', () => { const output = result.stdout + result.stderr expect(result.exitCode, `deploy failed:\n${output}`).toBe(0) } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) @@ -66,8 +69,11 @@ test.describe('TOML config regression', () => { proc.kill() } } finally { - fs.rmSync(parentDir, {recursive: true, force: true}) - await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + // E2E_SKIP_CLEANUP=1 skips cleanup for debugging. Run `pnpm test:e2e-cleanup` afterward. + if (!process.env.E2E_SKIP_CLEANUP) { + fs.rmSync(parentDir, {recursive: true, force: true}) + await teardownApp({browserPage, appName, email: process.env.E2E_ACCOUNT_EMAIL, orgId: env.orgId}) + } } }) })