From a6d6a0cd9037189bb3265ddb3c41f45fa92d88bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 16 Jun 2026 15:25:51 +0100 Subject: [PATCH] fix: report maestro ios runner setup failures --- .github/workflows/ios.yml | 3 +- .github/workflows/perf-nightly.yml | 3 +- .github/workflows/replays-nightly.yml | 3 +- .../__tests__/session-replay-vars.test.ts | 42 ++++++++++++++++ src/daemon/handlers/__tests__/session.test.ts | 50 +++++++++++++++++++ src/daemon/handlers/session-open.ts | 5 +- .../__tests__/runner-command-retry.test.ts | 19 +++++++ .../ios/__tests__/runner-session.test.ts | 47 ++++++++++++----- src/platforms/ios/runner-client.ts | 14 ++++-- src/platforms/ios/runner-session.ts | 2 +- 10 files changed, 167 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 50970600a..e1f610587 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -30,6 +30,7 @@ jobs: IOS_RUNTIME_VERSION: '26.2' AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived + AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000' steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -57,7 +58,7 @@ jobs: - name: Prepare iOS runner run: | pnpm clean:daemon - node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 300000 --json + node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json pnpm clean:daemon - name: Run iOS simulator smoke replay diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index 2140f62cf..e4eeb583d 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -36,6 +36,7 @@ jobs: IOS_RUNTIME_VERSION: "26.2" AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived + AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: "420000" steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -63,7 +64,7 @@ jobs: - name: Prepare iOS runner run: | pnpm clean:daemon - node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000 + node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" pnpm clean:daemon - name: Run iOS command perf benchmark diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index 8dbeff270..1aad1ba45 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -53,6 +53,7 @@ jobs: IOS_RUNTIME_VERSION: '26.2' AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived + AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000' steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -80,7 +81,7 @@ jobs: - name: Prepare iOS runner run: | pnpm clean:daemon - node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 300000 --json + node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json pnpm clean:daemon - name: Run iOS simulator replay suite diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 65c21179c..30afaab3f 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -768,6 +768,48 @@ test('runReplayScriptFile reports Maestro runScript failures at the runScript st assert.equal(calls.length, 0); }); +test('runReplayScriptFile reports iOS Maestro openLink setup failures before assertions', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-ios-openlink-prewarm-fail', + script: [ + 'appId: demo.app', + '---', + '- openLink: demo://screen', + '- assertVisible: Ready', + '', + ].join('\n'), + flags: { replayBackend: 'maestro', platform: 'ios' }, + invoke: async (req) => { + if (req.command === 'open') { + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'Developer mode is disabled for Apple development tools', + details: { + hint: 'Run `sudo DevToolsSecurity -enable`.', + }, + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /open "demo\.app" "demo:\/\/screen"/); + assert.match(response.error.message, /Developer mode is disabled/); + assert.match(String(response.error.details?.hint ?? ''), /DevToolsSecurity -enable/); + } + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['open', ['demo.app', 'demo://screen']]], + ); + assert.equal(calls[0]?.flags?.maestro?.prewarmRunnerBeforeOpen, true); +}); + test('runReplayScriptFile explains empty Maestro runScript JSON bodies', async () => { const { response, calls } = await runReplayFixture({ label: 'maestro-runscript-empty-json', diff --git a/src/daemon/handlers/__tests__/session.test.ts b/src/daemon/handlers/__tests__/session.test.ts index 719565f80..77d2164a6 100644 --- a/src/daemon/handlers/__tests__/session.test.ts +++ b/src/daemon/handlers/__tests__/session.test.ts @@ -2287,6 +2287,56 @@ test('open iOS Maestro app link waits for runner prewarm before launching app', }); }); +test('open iOS Maestro app link reports blocking runner prewarm failures before launching app', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-maestro-open-link-prewarm-failed'; + sessionStore.set(sessionName, { + ...makeSession(sessionName, { + platform: 'ios', + id: 'ios-device-1', + name: 'iPhone Device', + kind: 'device', + booted: true, + }), + appBundleId: 'com.example.previous', + appName: 'Previous App', + }); + mockPrewarmIosRunnerSession.mockRejectedValueOnce( + new AppError('COMMAND_FAILED', 'Developer mode is disabled for Apple development tools', { + hint: 'Run `sudo DevToolsSecurity -enable`.', + }), + ); + + await expect( + handleSessionCommands({ + req: { + token: 't', + session: sessionName, + command: 'open', + positionals: ['com.example.app', 'rne://screen-layout'], + flags: { + maestro: { prewarmRunnerBeforeOpen: true }, + }, + }, + sessionName, + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: noopInvoke, + }), + ).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + message: 'Developer mode is disabled for Apple development tools', + details: { + hint: expect.stringContaining('DevToolsSecurity -enable'), + }, + }); + expect(mockDispatch).not.toHaveBeenCalled(); + expect(mockPrewarmIosRunnerSession).toHaveBeenCalledWith( + expect.objectContaining({ platform: 'ios', id: 'ios-device-1' }), + expect.objectContaining({ propagateError: true }), + ); +}); + test('open iOS URL without app bundle id skips runner prewarm', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-device-session'; diff --git a/src/daemon/handlers/session-open.ts b/src/daemon/handlers/session-open.ts index 3566fe042..2fd6e29e0 100644 --- a/src/daemon/handlers/session-open.ts +++ b/src/daemon/handlers/session-open.ts @@ -198,7 +198,10 @@ async function completeOpenCommand(params: { timing.runnerPrewarmKind = 'session'; timing.runnerPrewarmScheduled = true; if (shouldPrewarmRunnerBeforeOpen) { - runnerPrewarm = prewarmIosRunnerSession(device, runnerPrewarmOptions); + runnerPrewarm = prewarmIosRunnerSession(device, { + ...runnerPrewarmOptions, + propagateError: true, + }); const runnerPrewarmStartedAtMs = Date.now(); await runnerPrewarm; timing.runnerPrewarmWaited = true; diff --git a/src/platforms/ios/__tests__/runner-command-retry.test.ts b/src/platforms/ios/__tests__/runner-command-retry.test.ts index ae7bcf67a..58882942f 100644 --- a/src/platforms/ios/__tests__/runner-command-retry.test.ts +++ b/src/platforms/ios/__tests__/runner-command-retry.test.ts @@ -196,6 +196,25 @@ test('prewarmIosRunnerSession proves cached runner health with uptime', async () assert.equal(mockExecuteRunnerCommandWithSession.mock.calls[0]?.[4], 45_000); }); +test('prewarmIosRunnerSession can propagate setup failures for blocking callers', async () => { + const failure = new AppError('COMMAND_FAILED', 'Developer mode is disabled'); + mockEnsureRunnerSession.mockRejectedValueOnce(failure); + const prewarm = prewarmIosRunnerSession(IOS_SIMULATOR, { propagateError: true }); + + assert.ok(prewarm); + await assert.rejects(prewarm, (error: unknown) => error === failure); + + assert.deepEqual(mockEmitDiagnostic.mock.calls[0]?.[0], { + level: 'warn', + phase: 'ios_runner_session_prewarm_failed', + data: { + deviceId: IOS_SIMULATOR.id, + error: 'Developer mode is disabled', + }, + }); + assert.equal(mockEnsureRunnerSession.mock.calls[0]?.[1]?.propagateError, undefined); +}); + test('prepareIosRunner does not force a rebuild when the relaunched fresh session still cannot connect', async () => { const missArtifact = makeRunnerArtifact({ xctestrunPath: '/tmp/miss.xctestrun', diff --git a/src/platforms/ios/__tests__/runner-session.test.ts b/src/platforms/ios/__tests__/runner-session.test.ts index 6ea6e7a98..fa4c34991 100644 --- a/src/platforms/ios/__tests__/runner-session.test.ts +++ b/src/platforms/ios/__tests__/runner-session.test.ts @@ -4,7 +4,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { beforeEach, test, vi } from 'vitest'; -import { IOS_SIMULATOR } from '../../../__tests__/test-utils/index.ts'; +import { IOS_DEVICE, IOS_SIMULATOR } from '../../../__tests__/test-utils/index.ts'; import { AppError } from '../../../utils/errors.ts'; import { flushDiagnosticsToSessionFile, withDiagnosticsScope } from '../../../utils/diagnostics.ts'; import type { RunnerSession } from '../runner-session-types.ts'; @@ -612,18 +612,9 @@ test('runner session starts xcodebuild through provider seams and reuses an aliv await stopRunnerSession(session); }); -test('runner session fails early when Apple developer mode is disabled', async () => { - const device = { ...IOS_SIMULATOR, id: 'runner-session-devtools-disabled-sim' }; - mockRunAppleToolCommand.mockImplementation(async (cmd, args) => { - if (cmd === 'DevToolsSecurity' && args[0] === '-status') { - return { - exitCode: 0, - stdout: 'Developer mode is currently disabled.\n', - stderr: '', - }; - } - return { exitCode: 0, stdout: '', stderr: '' }; - }); +test('runner session fails early for physical iOS devices when Apple developer mode is disabled', async () => { + const device = { ...IOS_DEVICE, id: 'runner-session-devtools-disabled-device' }; + mockDevToolsSecurityDisabled(); await assert.rejects( () => ensureRunnerSession(device, {}), @@ -640,6 +631,18 @@ test('runner session fails early when Apple developer mode is disabled', async ( assert.equal(mockRunCmdBackground.mock.calls.length, 0); }); +test('runner session does not require Apple developer mode for iOS simulators', async () => { + const device = { ...IOS_SIMULATOR, id: 'runner-session-devtools-disabled-sim' }; + mockDevToolsSecurityDisabled(); + + const session = await ensureRunnerSession(device, {}); + + assert.equal(session.deviceId, device.id); + assert.equal(mockEnsureXctestrunArtifact.mock.calls.length, 1); + assert.equal(mockRunCmdBackground.mock.calls.length, 1); + assert.equal(mockRunAppleToolCommand.mock.calls.some(isDevToolsSecurityStatusCall), false); +}); + test('runner session startup kills legacy ownerless xcodebuild before launching a new runner', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-session-startup-stale-sim' }; @@ -884,6 +887,24 @@ function isXcodebuildPkillCall(call: unknown[]): boolean { return call[0] === 'pkill' && Array.isArray(args) && args.includes('-f'); } +function isDevToolsSecurityStatusCall(call: unknown[]): boolean { + const args = call[1]; + return call[0] === 'DevToolsSecurity' && Array.isArray(args) && args[0] === '-status'; +} + +function mockDevToolsSecurityDisabled(): void { + mockRunAppleToolCommand.mockImplementation(async (cmd, args) => { + if (cmd === 'DevToolsSecurity' && args[0] === '-status') { + return { + exitCode: 0, + stdout: 'Developer mode is currently disabled.\n', + stderr: '', + }; + } + return { exitCode: 0, stdout: '', stderr: '' }; + }); +} + function isSimctlTerminateCall(call: unknown[]): boolean { const args = call[0]; return Array.isArray(args) && args.includes('simctl') && args.includes('terminate'); diff --git a/src/platforms/ios/runner-client.ts b/src/platforms/ios/runner-client.ts index 35ef20923..beee51ffe 100644 --- a/src/platforms/ios/runner-client.ts +++ b/src/platforms/ios/runner-client.ts @@ -59,14 +59,19 @@ export async function runIosRunnerCommand( return provider.runCommand(device, runnerCommand, options); } +type PrewarmIosRunnerSessionOptions = RunnerSessionOptions & { + propagateError?: boolean; +}; + export function prewarmIosRunnerSession( device: DeviceInfo, - options: RunnerSessionOptions = {}, + options: PrewarmIosRunnerSessionOptions = {}, ): Promise | undefined { if (device.platform !== 'ios') { return undefined; } - const provider = resolveAppleRunnerRuntime(device, options); + const { propagateError = false, ...runnerOptions } = options; + const provider = resolveAppleRunnerRuntime(device, runnerOptions); if (!provider.prewarm) { emitDiagnostic({ level: 'debug', @@ -76,7 +81,7 @@ export function prewarmIosRunnerSession( return undefined; } const prewarm = provider - .prewarm(device, options) + .prewarm(device, runnerOptions) .then(() => {}) .catch((error: unknown) => { emitDiagnostic({ @@ -87,6 +92,9 @@ export function prewarmIosRunnerSession( error: error instanceof Error ? error.message : String(error), }, }); + if (propagateError) { + throw error; + } }); void prewarm; return prewarm; diff --git a/src/platforms/ios/runner-session.ts b/src/platforms/ios/runner-session.ts index 46a7b51ce..a3e9a13a1 100644 --- a/src/platforms/ios/runner-session.ts +++ b/src/platforms/ios/runner-session.ts @@ -464,7 +464,7 @@ async function ensureBooted(device: DeviceInfo): Promise { } async function verifyDeveloperModeForIosRunner(device: DeviceInfo): Promise { - if (device.platform !== 'ios') return; + if (device.platform !== 'ios' || device.kind !== 'device') return; const result = await runAppleToolCommand('DevToolsSecurity', ['-status'], { allowFailure: true, timeoutMs: 2_000,