diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index 045f4e4d5..2472b638c 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -106,11 +106,8 @@ test('test command prints suite summary and exits non-zero on failures', async ( assert.equal(result.calls.length, 1); assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test'); assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./); - assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/); - assert.match( - result.stdout, - /FAIL "Checkout failure" in 02-fail\.ad after 2 attempts \(total 0\.005s\)/, - ); + assert.doesNotMatch(result.stdout, /✓ 01-pass\.ad \(0\.01s\)/); + assert.doesNotMatch(result.stdout, /⨯ "Checkout failure" in 02-fail\.ad/); assert.match(result.stdout, /Replay failed at step 1 \(open Demo\): boom/); assert.match(result.stdout, /artifacts: \/tmp\/test-artifacts\/02-fail/); assert.doesNotMatch(result.stdout, /SKIP \/tmp\/03-skip\.ad/); @@ -125,11 +122,12 @@ test('test command --verbose prints all test statuses', async () => { assert.equal(result.code, 1); assert.equal(result.calls[0]?.meta?.debug, false); assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./); - assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/); - assert.match(result.stdout, /SKIP 03-skip\.ad/); + assert.doesNotMatch(result.stdout, /✓ 01-pass\.ad \(0\.01s\)/); + assert.doesNotMatch(result.stdout, /SKIP 03-skip\.ad/); + assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 0\.025s/); }); -test('test command --verbose prints step telemetry for passing tests without debug mode', async () => { +test('test command --verbose omits step telemetry for passing tests without debug mode', async () => { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-')); const artifactsDir = path.join(tmpDir, 'auth-flow'); const attemptDir = path.join(artifactsDir, 'attempt-1'); @@ -203,16 +201,16 @@ test('test command --verbose prints step telemetry for passing tests without deb assert.equal(result.code, null); assert.equal(result.calls[0]?.meta?.debug, false); - assert.match(result.stdout, /PASS "Authentication flow" \(0\.5s\)/); - assert.match(result.stdout, /steps:/); - assert.match(result.stdout, /tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/); - assert.match(result.stdout, /assertVisible "text=\\"Home\\"" \(line 4, 0\.075s\)/); + assert.doesNotMatch(result.stdout, /✓ "Authentication flow" in auth-flow\.yml \(0\.5s\)/); + assert.doesNotMatch(result.stdout, /steps:/); + assert.doesNotMatch(result.stdout, /tapOn "text=\\"Log in\\""/); + assert.doesNotMatch(result.stdout, /assertVisible "text=\\"Home\\""/); } finally { await fs.rm(tmpDir, { recursive: true, force: true }); } }); -test('test command --verbose keeps nested retry and open step telemetry distinct', async () => { +test('test command --verbose omits nested passing step telemetry', async () => { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-')); const artifactsDir = path.join(tmpDir, 'material-top-tabs'); const attemptDir = path.join(artifactsDir, 'attempt-1'); @@ -300,15 +298,15 @@ test('test command --verbose keeps nested retry and open step telemetry distinct })); assert.equal(result.code, null); - assert.match( + assert.doesNotMatch( result.stdout, /open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 0\.727s\)/, ); - assert.match( + assert.doesNotMatch( result.stdout, /assertVisible "label=\\"Chat\\" \|\| text=\\"Chat\\" \|\| id=\\"Chat\\"" "60000" \(line 4, 2\.58s\)/, ); - assert.match(result.stdout, /retry "3" \(line 4, 3\.31s\)/); + assert.doesNotMatch(result.stdout, /retry "3" \(line 4, 3\.31s\)/); assert.doesNotMatch( result.stdout, /open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 3\.31s\)/, @@ -354,15 +352,15 @@ test('test command reports flaky passed-on-retry cases in the default summary', assert.equal(result.code, null); assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./); assert.doesNotMatch(result.stdout, /FLAKY/); - assert.match( + assert.doesNotMatch( result.stdout, - /PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/, + /^✓ "Authentication flow" in auth-flow\.yml \(passed attempt 17\.5s, total 112\.2s\)$/m, ); assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/); assert.match(result.stdout, /Flaky tests:/); assert.match( result.stdout, - /PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/, + /✓ "Authentication flow" in auth-flow\.yml after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/, ); assert.match( result.stdout, @@ -370,7 +368,7 @@ test('test command reports flaky passed-on-retry cases in the default summary', ); }); -test('test command prints failed attempt step telemetry when timing trace exists', async () => { +test('test command --debug prints failed attempt step window when timing trace exists', async () => { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-steps-')); const artifactsDir = path.join(tmpDir, 'checkout-flow'); const attemptDir = path.join(artifactsDir, 'attempt-2'); @@ -378,6 +376,21 @@ test('test command prints failed attempt step telemetry when timing trace exists await fs.writeFile( path.join(attemptDir, 'replay-timing.ndjson'), [ + { + type: 'replay_action_start', + step: 0, + line: 2, + command: 'close', + positionals: ['Demo'], + }, + { + type: 'replay_action_stop', + step: 0, + line: 2, + command: 'close', + ok: true, + durationMs: 50, + }, { type: 'replay_action_start', step: 1, @@ -406,6 +419,21 @@ test('test command prints failed attempt step telemetry when timing trace exists step: 2, line: 4, command: '__maestroTapOn', + ok: true, + durationMs: 200, + }, + { + type: 'replay_action_start', + step: 3, + line: 5, + command: '__maestroAssertVisible', + positionals: ['text="Receipt"', '3000'], + }, + { + type: 'replay_action_stop', + step: 3, + line: 5, + command: '__maestroAssertVisible', ok: false, durationMs: 1500, errorCode: 'ASSERTION_FAILED', @@ -426,10 +454,10 @@ test('test command prints failed attempt step telemetry when timing trace exists artifactsDir, error: { code: 'ASSERTION_FAILED', - message: 'Replay failed at step 2 (click "Pay"): selector not found', + message: 'Replay failed at step 3 (assertVisible "Receipt"): selector not found', }, }; - const result = await runCliCapture(['test', './suite'], async () => ({ + const result = await runCliCapture(['test', './suite', '--debug'], async () => ({ ok: true, data: { total: 1, @@ -445,11 +473,18 @@ test('test command prints failed attempt step telemetry when timing trace exists })); assert.equal(result.code, 1); + assert.equal(result.calls[0]?.meta?.debug, true); + assert.match( + result.stdout, + /Replay failed at step 3 \(assertVisible "Receipt"\): selector not found/, + ); assert.match(result.stdout, /steps \(attempt 2\):/); + assert.doesNotMatch(result.stdout, /close "Demo" \(line 2, 0\.050s\)/); assert.match(result.stdout, /open "Demo" \(line 3, 0\.125s, timing \{"launchMs":100\}\)/); + assert.match(result.stdout, /tapOn "text=\\"Pay\\"" \(line 4, 0\.2s\)/); assert.match( result.stdout, - /\[FAIL\] tapOn "text=\\"Pay\\"" \(line 4, 1\.50s, ASSERTION_FAILED\)/, + /\[FAIL\] assertVisible "text=\\"Receipt\\"" "3000" \(line 5, 1\.50s, ASSERTION_FAILED\)/, ); } finally { await fs.rm(tmpDir, { recursive: true, force: true }); diff --git a/src/__tests__/cli-test-progress.test.ts b/src/__tests__/cli-test-progress.test.ts index d1ff4089f..c28690867 100644 --- a/src/__tests__/cli-test-progress.test.ts +++ b/src/__tests__/cli-test-progress.test.ts @@ -3,7 +3,19 @@ import assert from 'node:assert/strict'; import { formatReplayTestProgressEvent } from '../cli-test-progress.ts'; import type { RequestProgressEvent } from '../daemon/request-progress.ts'; -test('formatReplayTestProgressEvent renders replay suite start context', () => { +function withStreamTty(stream: NodeJS.WriteStream, isTTY: boolean, run: () => T): T { + const descriptor = Object.getOwnPropertyDescriptor(stream, 'isTTY'); + const mutableStream = stream as unknown as Record; + try { + Object.defineProperty(stream, 'isTTY', { configurable: true, value: isTTY }); + return run(); + } finally { + if (descriptor) Object.defineProperty(stream, 'isTTY', descriptor); + else delete mutableStream.isTTY; + } +} + +test('formatReplayTestProgressEvent suppresses replay suite start context', () => { const line = formatReplayTestProgressEvent({ type: 'replay-test-suite', status: 'start', @@ -15,17 +27,10 @@ test('formatReplayTestProgressEvent renders replay suite start context', () => { shardCount: 2, }); - assert.equal( - line, - [ - 'Running replay suite: 4 files', - ' sharding: split across 2 devices', - ' artifacts: /tmp/replay-suite', - ].join('\n'), - ); + assert.equal(line, undefined); }); -test('formatReplayTestProgressEvent renders replay test start context with shard metadata', () => { +test('formatReplayTestProgressEvent suppresses replay test start context', () => { const line = formatReplayTestProgressEvent({ type: 'replay-test', file: '/tmp/auth-flow.yml', @@ -40,14 +45,7 @@ test('formatReplayTestProgressEvent renders replay test start context with shard deviceId: 'E140A942-965C-4A92-AC63-F3B23756BE02', }); - assert.equal( - line, - [ - '[2/5] START "Authentication flow" in auth-flow.yml [shard 2/2 E140A942-965C-4A92-AC63-F3B23756BE02]', - ' session: maestro-test:test:suite:2:attempt-1', - ' artifacts: /tmp/replay-suite/auth-flow', - ].join('\n'), - ); + assert.equal(line, undefined); }); test('formatReplayTestProgressEvent ignores unknown progress event types', () => { @@ -72,7 +70,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', maxAttempts: 2, durationMs: 12_345, }, - expected: /^\[1\/3] PASS 01-login\.ad after 2 attempts \(total 12\.3s\)$/, + expected: /^✓ 01-login\.ad \(12\.3s\)$/, }, { event: { @@ -87,7 +85,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', retrying: true, message: 'first attempt failed', }, - expected: /^\[2\/3] RETRY 02-checkout\.ad attempt 1\/2 \(1\.23s\)\n first attempt failed$/, + expected: /^$/, }, { event: { @@ -104,7 +102,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', artifactsDir: '/tmp/replay-suite/payment', }, expected: - /^\[3\/3] FAIL 03-payment\.ad after 2 attempts \(total 9\.88s\)\n assertVisible failed\n session: maestro-test:test:suite:3:attempt-2\n artifacts: \/tmp\/replay-suite\/payment$/, + /^⨯ 03-payment\.ad \(9\.88s\)\n failed at: assertVisible failed\n session: maestro-test:test:suite:3:attempt-2\n artifacts: \/tmp\/replay-suite\/payment$/, }, { event: { @@ -115,7 +113,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', total: 5, message: 'missing platform metadata for --platform ios', }, - expected: /^\[4\/5] SKIP 04-skip\.ad\n missing platform metadata for --platform ios$/, + expected: /^- 04-skip\.ad\n missing platform metadata for --platform ios$/, }, ]; @@ -123,3 +121,91 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', assert.match(formatReplayTestProgressEvent(event) ?? '', expected); } }); + +test('formatReplayTestProgressEvent colors stderr progress rows when stdout is piped', () => { + const originalForceColor = process.env.FORCE_COLOR; + const originalNoColor = process.env.NO_COLOR; + delete process.env.FORCE_COLOR; + delete process.env.NO_COLOR; + try { + const line = withStreamTty(process.stdout, false, () => + withStreamTty(process.stderr, true, () => + formatReplayTestProgressEvent({ + type: 'replay-test', + file: '/tmp/01-pass.ad', + status: 'pass', + index: 1, + total: 1, + attempt: 1, + durationMs: 10, + }), + ), + ); + + assert.equal(line, '\u001B[32m✓\u001B[39m 01-pass.ad (0.01s)'); + } finally { + if (typeof originalForceColor === 'string') process.env.FORCE_COLOR = originalForceColor; + else delete process.env.FORCE_COLOR; + if (typeof originalNoColor === 'string') process.env.NO_COLOR = originalNoColor; + else delete process.env.NO_COLOR; + } +}); + +test('formatReplayTestProgressEvent colors completed result markers when color is enabled', () => { + const originalForceColor = process.env.FORCE_COLOR; + const originalNoColor = process.env.NO_COLOR; + process.env.FORCE_COLOR = '1'; + delete process.env.NO_COLOR; + try { + formatReplayTestProgressEvent({ + type: 'replay-test-suite', + status: 'start', + total: 3, + runnable: 3, + skipped: 0, + artifactsDir: '/tmp/replay-suite', + }); + assert.equal( + formatReplayTestProgressEvent({ + type: 'replay-test', + file: '/tmp/01-pass.ad', + status: 'pass', + index: 1, + total: 3, + attempt: 1, + durationMs: 10, + }), + '\u001B[32m✓\u001B[39m 01-pass.ad (0.01s)', + ); + assert.equal( + formatReplayTestProgressEvent({ + type: 'replay-test', + file: '/tmp/02-flaky.yml', + title: 'Retry flow', + status: 'pass', + index: 2, + total: 3, + attempt: 2, + durationMs: 30, + }), + '\u001B[33m✓\u001B[39m "Retry flow" in 02-flaky.yml (0.03s)', + ); + const failedLine = formatReplayTestProgressEvent({ + type: 'replay-test', + file: '/tmp/03-fail.ad', + title: 'Checkout failure', + status: 'fail', + index: 3, + total: 3, + attempt: 1, + durationMs: 5, + message: 'boom', + }); + assert.ok(failedLine?.startsWith('\u001B[31m⨯\u001B[39m "Checkout failure" in 03-fail.ad')); + } finally { + if (typeof originalForceColor === 'string') process.env.FORCE_COLOR = originalForceColor; + else delete process.env.FORCE_COLOR; + if (typeof originalNoColor === 'string') process.env.NO_COLOR = originalNoColor; + else delete process.env.NO_COLOR; + } +}); diff --git a/src/__tests__/daemon-client-progress.test.ts b/src/__tests__/daemon-client-progress.test.ts index eaf4b3c69..0a552409d 100644 --- a/src/__tests__/daemon-client-progress.test.ts +++ b/src/__tests__/daemon-client-progress.test.ts @@ -50,6 +50,32 @@ function readSocketProgressResponse( }); } +function withStderrTerminal(params: { isTTY: boolean; columns: number }, run: () => T): T { + const stderr = process.stderr as typeof process.stderr & { + isTTY?: boolean; + columns?: number; + }; + const mutableStderr = stderr as unknown as Record; + const originalIsTTY = Object.getOwnPropertyDescriptor(stderr, 'isTTY'); + const originalColumns = Object.getOwnPropertyDescriptor(stderr, 'columns'); + try { + Object.defineProperty(stderr, 'isTTY', { + configurable: true, + value: params.isTTY, + }); + Object.defineProperty(stderr, 'columns', { + configurable: true, + value: params.columns, + }); + return run(); + } finally { + if (originalIsTTY) Object.defineProperty(stderr, 'isTTY', originalIsTTY); + else delete mutableStderr.isTTY; + if (originalColumns) Object.defineProperty(stderr, 'columns', originalColumns); + else delete mutableStderr.columns; + } +} + test('readDaemonSocketProgressResponse parses split progress lines before response envelopes', async () => { const socket = createMockSocket(); const req: DaemonRequest = { @@ -76,14 +102,12 @@ test('readDaemonSocketProgressResponse parses split progress lines before respon type: 'replay-test', file: '/tmp/01-login.ad', title: 'Login flow', - status: 'fail', + status: 'pass', index: 1, total: 2, attempt: 1, maxAttempts: 2, durationMs: 1234, - retrying: true, - message: 'first attempt failed', }, }); const responseLine = JSON.stringify({ @@ -99,8 +123,146 @@ test('readDaemonSocketProgressResponse parses split progress lines before respon assert.deepEqual(await responsePromise, { ok: true, data: { via: 'socket-progress' } }); assert.equal(socket.encoding, 'utf8'); assert.equal(socket.ended, true); - assert.match(stderr, /\[1\/2] RETRY "Login flow" in 01-login\.ad attempt 1\/2 \(1\.23s\)/); - assert.match(stderr, / first attempt failed/); + assert.match(stderr, /✓ "Login flow" in 01-login\.ad \(1\.23s\)/); + } finally { + process.stderr.write = originalStderrWrite; + } +}); + +test('readDaemonSocketProgressResponse rewrites live progress and clears it for final result', async () => { + const socket = createMockSocket(); + const req: DaemonRequest = { + session: 'default', + command: 'test', + positionals: ['/tmp/replays'], + flags: {}, + token: 'secret', + meta: { requestId: 'req-live-progress', requestProgress: 'replay-test' }, + }; + let stderr = ''; + const originalStderrWrite = process.stderr.write.bind(process.stderr); + const originalCi = process.env.CI; + + try { + delete process.env.CI; + (process.stderr as any).write = ((chunk: unknown) => { + stderr += String(chunk); + return true; + }) as typeof process.stderr.write; + + const responsePromise = withStderrTerminal({ isTTY: true, columns: 53 }, () => + readSocketProgressResponse(socket, req), + ); + const progress = (stepIndex: number) => + JSON.stringify({ + type: 'progress', + event: { + type: 'replay-test', + file: '/tmp/tab-view-coverflow.yml', + title: 'Tab View - Coverflow', + status: 'progress', + index: 1, + total: 1, + attempt: 1, + maxAttempts: 1, + stepIndex, + stepTotal: 10, + }, + }); + const pass = JSON.stringify({ + type: 'progress', + event: { + type: 'replay-test', + file: '/tmp/tab-view-coverflow.yml', + title: 'Tab View - Coverflow', + status: 'pass', + index: 1, + total: 1, + attempt: 1, + maxAttempts: 1, + durationMs: 17_800, + }, + }); + const responseLine = JSON.stringify({ + type: 'response', + response: { ok: true, data: { via: 'socket-progress' } }, + }); + + socket.emit('data', `${progress(3)}\n${progress(4)}\n${pass}\n${responseLine}\n`); + + assert.deepEqual(await responsePromise, { ok: true, data: { via: 'socket-progress' } }); + assert.ok(stderr.includes('\r\u001B[2K⊙ "Tab View - Co..." in tab-view-coverflow.yml [3/10]')); + assert.ok(stderr.includes('\r\u001B[2K⊙ "Tab View - Co..." in tab-view-coverflow.yml [4/10]')); + assert.ok( + stderr.includes('\r\u001B[2K✓ "Tab View - Coverflow" in tab-view-coverflow.yml (17.8s)\n'), + ); + } finally { + if (typeof originalCi === 'string') process.env.CI = originalCi; + else delete process.env.CI; + process.stderr.write = originalStderrWrite; + } +}); + +test('readDaemonSocketProgressResponse suppresses live progress outside interactive terminals', async () => { + const socket = createMockSocket(); + const req: DaemonRequest = { + session: 'default', + command: 'test', + positionals: ['/tmp/replays'], + flags: {}, + token: 'secret', + meta: { requestId: 'req-non-tty-progress', requestProgress: 'replay-test' }, + }; + let stderr = ''; + const originalStderrWrite = process.stderr.write.bind(process.stderr); + + try { + (process.stderr as any).write = ((chunk: unknown) => { + stderr += String(chunk); + return true; + }) as typeof process.stderr.write; + + const responsePromise = withStderrTerminal({ isTTY: false, columns: 53 }, () => + readSocketProgressResponse(socket, req), + ); + const progress = JSON.stringify({ + type: 'progress', + event: { + type: 'replay-test', + file: '/tmp/tab-view-coverflow.yml', + title: 'Tab View - Coverflow', + status: 'progress', + index: 1, + total: 1, + attempt: 1, + maxAttempts: 1, + stepIndex: 3, + stepTotal: 10, + }, + }); + const pass = JSON.stringify({ + type: 'progress', + event: { + type: 'replay-test', + file: '/tmp/tab-view-coverflow.yml', + title: 'Tab View - Coverflow', + status: 'pass', + index: 1, + total: 1, + attempt: 1, + maxAttempts: 1, + durationMs: 17_800, + }, + }); + const responseLine = JSON.stringify({ + type: 'response', + response: { ok: true, data: { via: 'socket-progress' } }, + }); + + socket.emit('data', `${progress}\n${pass}\n${responseLine}\n`); + + assert.deepEqual(await responsePromise, { ok: true, data: { via: 'socket-progress' } }); + assert.equal(stderr, '✓ "Tab View - Coverflow" in tab-view-coverflow.yml (17.8s)\n'); } finally { process.stderr.write = originalStderrWrite; } diff --git a/src/cli-test-progress.ts b/src/cli-test-progress.ts index b199404a5..4a135613f 100644 --- a/src/cli-test-progress.ts +++ b/src/cli-test-progress.ts @@ -1,69 +1,133 @@ import path from 'node:path'; import type { RequestProgressEvent } from './daemon/request-progress.ts'; +import { replayTestStepLines } from './cli-test-trace.ts'; +import type { ReplaySuiteTestResult } from './daemon/types.ts'; import { formatDurationSeconds } from './utils/duration-format.ts'; +import { colorize, supportsColor } from './utils/output.ts'; type ReplayTestCaseProgressEvent = Extract; -type ReplayTestCaseStatus = ReplayTestCaseProgressEvent['status']; +type ReplayTestProgressFormatOptions = { + verbose?: boolean; + liveProgress?: boolean; + columns?: number; +}; -const REPLAY_TEST_STATUS_LABELS: Record = { - start: 'START', - pass: 'PASS', - fail: 'FAIL', - skip: 'SKIP', +export type ReplayTestProgressRender = { + text: string; + newline: boolean; }; -export function formatReplayTestProgressEvent(event: RequestProgressEvent): string | undefined { - if (event.type === 'replay-test-suite') { - return formatReplayTestSuiteProgressEvent(event); - } - const eventType = (event as { type?: string }).type; - if (eventType !== 'replay-test') { - return undefined; - } - return formatReplayTestCaseProgressEvent(event); +export type ReplayTestProgressRenderer = { + render(event: RequestProgressEvent): ReplayTestProgressRender | undefined; +}; + +export function createReplayTestProgressRenderer( + options: ReplayTestProgressFormatOptions = {}, +): ReplayTestProgressRenderer { + const completedKeys = new Set(); + let hasLiveProgressLine = false; + return { + render(event) { + if (event.type === 'replay-test-suite') { + completedKeys.clear(); + hasLiveProgressLine = false; + return undefined; + } + if (event.type !== 'replay-test') { + return undefined; + } + if (event.status === 'progress') { + if (!options.liveProgress) return undefined; + hasLiveProgressLine = true; + return { + text: clearLinePrefix(formatReplayTestLiveProgressLine(event, options)), + newline: false, + }; + } + if (isReplayTestCompletionProgressEvent(event)) { + const key = replayTestCompletionProgressKey(event); + if (completedKeys.has(key)) return undefined; + completedKeys.add(key); + } + const line = formatReplayTestProgressEvent(event, options); + if (!line) return undefined; + const text = hasLiveProgressLine ? clearLinePrefix(line) : line; + hasLiveProgressLine = false; + return { text, newline: true }; + }, + }; } -function formatReplayTestSuiteProgressEvent( - event: Extract, -): string { - const lines = [`Running replay suite: ${event.total} ${event.total === 1 ? 'file' : 'files'}`]; - if (event.shardMode && event.shardCount && event.shardCount > 1) { - lines.push(` sharding: ${event.shardMode} across ${event.shardCount} devices`); +export function formatReplayTestProgressEvent( + event: RequestProgressEvent, + options: ReplayTestProgressFormatOptions = {}, +): string | undefined { + if (event.type !== 'replay-test') { + return undefined; } - lines.push(` artifacts: ${event.artifactsDir}`); - return lines.join('\n'); + return formatReplayTestCaseProgressEvent(event, options); } -function formatReplayTestCaseProgressEvent(event: ReplayTestCaseProgressEvent): string { +function formatReplayTestCaseProgressEvent( + event: ReplayTestCaseProgressEvent, + options: ReplayTestProgressFormatOptions, +): string | undefined { + if (event.status === 'start' || event.status === 'progress') return undefined; + if (event.status === 'fail' && event.retrying) return undefined; const lines = [formatReplayTestCaseSummaryLine(event)]; - addReplayTestCaseDetailLines(lines, event); + addReplayTestCaseDetailLines(lines, event, options); + if (options.verbose) { + lines.push(...replayTestProgressStepLines(event)); + } return lines.join('\n'); } -function addReplayTestCaseDetailLines(lines: string[], event: ReplayTestCaseProgressEvent): void { - if (event.status === 'start') { - if (event.session) lines.push(` session: ${event.session}`); - if (event.artifactsDir) lines.push(` artifacts: ${event.artifactsDir}`); - return; - } +function formatReplayTestLiveProgressLine( + event: ReplayTestCaseProgressEvent, + options: ReplayTestProgressFormatOptions, +): string { + const title = event.title?.trim(); + const file = path.basename(event.file); + const shardSuffix = formatReplayTestProgressShardSuffix(event); + const stepIndex = event.stepIndex ?? 0; + const stepTotal = event.stepTotal ?? 0; + const suffix = `${shardSuffix} [${stepIndex}/${stepTotal}]`; + const prefix = '⊙ '; + if (!title) return trimToColumns(`${prefix}${file}${suffix}`, options.columns); + const titlePrefix = `${prefix}"`; + const titleSuffix = `" in ${file}${suffix}`; + const availableTitleColumns = Math.max( + 0, + resolveColumns(options.columns) - titlePrefix.length - titleSuffix.length, + ); + const formattedTitle = trimToColumns(title, availableTitleColumns); + return trimToColumns(`${titlePrefix}${formattedTitle}${titleSuffix}`, options.columns); +} + +function addReplayTestCaseDetailLines( + lines: string[], + event: ReplayTestCaseProgressEvent, + options: ReplayTestProgressFormatOptions, +): void { + if (options.verbose && event.status === 'fail') return; const message = event.message?.replace(/\s+/g, ' ').trim(); - if (message) lines.push(` ${message}`); + if (message) { + lines.push(` ${event.status === 'fail' ? `failed at: ${message}` : message}`); + } if (event.status === 'fail' && !event.retrying) { - if (event.session) lines.push(` session: ${event.session}`); - if (event.artifactsDir) lines.push(` artifacts: ${event.artifactsDir}`); + if (event.session) lines.push(` session: ${event.session}`); + if (event.artifactsDir) lines.push(` artifacts: ${event.artifactsDir}`); } } function formatReplayTestCaseSummaryLine(event: ReplayTestCaseProgressEvent): string { - const indexPrefix = `[${event.index}/${event.total}]`; const statusLabel = formatReplayTestProgressStatusLabel(event); const name = formatReplayTestProgressName(event); const shardSuffix = formatReplayTestProgressShardSuffix(event); - const attemptSuffix = formatReplayProgressAttemptSuffix(event); const durationSuffix = event.durationMs !== undefined ? ` (${formatReplayProgressDuration(event)})` : ''; - return `${indexPrefix} ${statusLabel} ${name}${shardSuffix}${attemptSuffix}${durationSuffix}`; + return `${statusLabel} ${name}${shardSuffix}${durationSuffix}`; } function formatReplayTestProgressName(event: ReplayTestCaseProgressEvent): string { @@ -73,7 +137,18 @@ function formatReplayTestProgressName(event: ReplayTestCaseProgressEvent): strin } function formatReplayTestProgressStatusLabel(event: ReplayTestCaseProgressEvent): string { - return event.retrying ? 'RETRY' : REPLAY_TEST_STATUS_LABELS[event.status]; + const useColor = supportsColor(process.stderr); + if (event.status === 'pass') { + const format = event.attempt && event.attempt > 1 ? 'yellow' : 'green'; + return useColor ? colorizeProgressMarker('✓', format) : '✓'; + } + if (event.status === 'fail') return useColor ? colorizeProgressMarker('⨯', 'red') : '⨯'; + if (event.status === 'progress') return '⊙'; + return useColor ? colorizeProgressMarker('-', 'dim') : '-'; +} + +function colorizeProgressMarker(text: string, format: Parameters[1]): string { + return colorize(text, format, { validateStream: false }); } function formatReplayTestProgressShardSuffix(event: ReplayTestCaseProgressEvent): string { @@ -83,17 +158,79 @@ function formatReplayTestProgressShardSuffix(event: ReplayTestCaseProgressEvent) return ` [shard ${event.shardIndex + 1}/${shardCount}${device}]`; } -function formatReplayProgressAttemptSuffix(event: ReplayTestCaseProgressEvent): string { - if (event.attempt === undefined) return ''; - if (event.status === 'start') return ''; - if (event.status === 'fail' && event.retrying && event.maxAttempts !== undefined) { - return ` attempt ${event.attempt}/${event.maxAttempts}`; - } - if (event.attempt > 1) return ` after ${event.attempt} attempts`; - return ''; +function formatReplayProgressDuration(event: ReplayTestCaseProgressEvent): string { + return formatDurationSeconds(event.durationMs ?? 0); } -function formatReplayProgressDuration(event: ReplayTestCaseProgressEvent): string { - const duration = formatDurationSeconds(event.durationMs ?? 0); - return event.attempt && event.attempt > 1 && !event.retrying ? `total ${duration}` : duration; +function isReplayTestCompletionProgressEvent(event: ReplayTestCaseProgressEvent): boolean { + return ( + event.status === 'pass' || + event.status === 'skip' || + (event.status === 'fail' && !event.retrying) + ); +} + +function replayTestCompletionProgressKey(event: ReplayTestCaseProgressEvent): string { + const shard = typeof event.shardIndex === 'number' ? event.shardIndex : ''; + return [event.status, event.index, event.total, event.file, event.title ?? '', shard].join('\0'); +} + +function clearLinePrefix(text: string): string { + return `\r\x1B[2K${text}`; +} + +function resolveColumns(columns: number | undefined): number { + return typeof columns === 'number' && Number.isFinite(columns) && columns > 0 + ? Math.floor(columns) + : 80; +} + +function trimToColumns(value: string, columns: number | undefined): string { + const limit = resolveColumns(columns); + if (value.length <= limit) return value; + if (limit <= 0) return ''; + if (limit <= 3) return '.'.repeat(limit); + return `${value.slice(0, limit - 3)}...`; +} + +function replayTestProgressStepLines(event: ReplayTestCaseProgressEvent): string[] { + if (event.status !== 'pass' && event.status !== 'fail') return []; + if (!event.artifactsDir || !event.attempt) return []; + const result = + event.status === 'pass' + ? buildPassedReplayTestProgressResult(event) + : buildFailedReplayTestProgressResult(event); + return replayTestStepLines(result).map((line) => ` ${line}`); +} + +function buildPassedReplayTestProgressResult( + event: ReplayTestCaseProgressEvent, +): Extract { + return { + ...replayTestProgressResultBase(event), + status: 'passed', + replayed: 0, + healed: 0, + }; +} + +function buildFailedReplayTestProgressResult( + event: ReplayTestCaseProgressEvent, +): Extract { + return { + ...replayTestProgressResultBase(event), + status: 'failed', + error: { code: 'COMMAND_FAILED', message: event.message ?? 'Unknown test failure' }, + }; +} + +function replayTestProgressResultBase(event: ReplayTestCaseProgressEvent) { + return { + file: event.file, + title: event.title, + durationMs: event.durationMs ?? 0, + attempts: event.attempt ?? 1, + artifactsDir: event.artifactsDir, + session: event.session ?? '', + }; } diff --git a/src/cli-test-trace.ts b/src/cli-test-trace.ts index 47cf46953..c44c2c97d 100644 --- a/src/cli-test-trace.ts +++ b/src/cli-test-trace.ts @@ -24,6 +24,31 @@ type ReplayActionStopTrace = { export function replayTestStepLines(result: ReplaySuiteTestResult): string[] { if (result.status === 'skipped') return []; + const stops = readReplayStepTraces(result); + if (stops.length === 0) return []; + + return [ + result.attempts > 1 ? `steps (attempt ${result.attempts}):` : 'steps:', + ...stops.map(({ stop, start }) => renderReplayStepTrace(stop, start)), + ]; +} + +export function replayTestFailureStepLines( + result: Extract, +): string[] { + const stops = readReplayStepTraces(result); + const failedIndex = stops.findIndex(({ stop }) => stop.ok === false); + if (failedIndex < 0) return []; + const window = stops.slice(Math.max(0, failedIndex - 2), failedIndex + 1); + return [ + result.attempts > 1 ? `steps (attempt ${result.attempts}):` : 'steps:', + ...window.map(({ stop, start }) => renderReplayStepTrace(stop, start)), + ]; +} + +function readReplayStepTraces( + result: Extract, +): Array<{ stop: ReplayActionStopTrace; start: ReplayActionStartTrace | undefined }> { const tracePath = replayTestTimingTracePath(result); if (!tracePath) return []; const events = readReplayTimingTrace(tracePath); @@ -41,12 +66,7 @@ export function replayTestStepLines(result: ReplaySuiteTestResult): string[] { stops.push({ stop: event, start: consumeReplayActionStart(starts, event) }); } } - if (stops.length === 0) return []; - - return [ - result.attempts > 1 ? `steps (attempt ${result.attempts}):` : 'steps:', - ...stops.map(({ stop, start }) => renderReplayStepTrace(stop, start)), - ]; + return stops; } function consumeReplayActionStart( diff --git a/src/cli-test.ts b/src/cli-test.ts index f3f5f67da..c5eaf39d1 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -1,10 +1,10 @@ import fs from 'node:fs'; import path from 'node:path'; import type { ReplaySuiteResult, ReplaySuiteTestResult } from './daemon/types.ts'; -import { replayTestStepLines } from './cli-test-trace.ts'; +import { replayTestFailureStepLines } from './cli-test-trace.ts'; import { formatDurationSeconds } from './utils/duration-format.ts'; import { AppError } from './utils/errors.ts'; -import { printJson } from './utils/output.ts'; +import { colorize, printJson, supportsColor } from './utils/output.ts'; type PassedReplayTestResult = Extract; type FailedReplayTestResult = Extract; @@ -13,10 +13,10 @@ type ReplayTestError = FailedReplayTestResult['error']; export function renderReplayTestResponse(options: { suite: ReplaySuiteResult; json?: boolean; - verbose?: boolean; + debug?: boolean; reportJunit?: string; }): number { - const { suite, json, verbose, reportJunit } = options; + const { suite, json, debug, reportJunit } = options; if (reportJunit) { writeReplayJunitReport(reportJunit, suite); } @@ -24,89 +24,34 @@ export function renderReplayTestResponse(options: { printJson({ success: true, data: suite }); return getReplayTestExitCode(suite); } - return renderReplayTestSummary(suite, { verbose }); + return renderReplayTestSummary(suite, { debug }); } function renderReplayTestSummary( data: ReplaySuiteResult, - options: { verbose?: boolean } = {}, + options: { debug?: boolean } = {}, ): number { const flaky = data.tests.filter(isFlakyReplayTestResult); - if (options.verbose) { - for (const entry of data.tests) { - renderVerboseTestResult(entry); - } - } else { - for (const entry of data.tests) { - renderDefaultTestResult(entry); - } - } - - const durationMs = typeof data.durationMs === 'number' ? data.durationMs : undefined; - const flakySuffix = flaky.length > 0 ? `, ${flaky.length} flaky` : ''; - const durationSuffix = durationMs !== undefined ? ` in ${formatDurationSeconds(durationMs)}` : ''; - process.stdout.write( - `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationSuffix}\n`, - ); + process.stdout.write(`${formatReplayTestSummaryLine(data, flaky.length)}\n`); + renderFailureDetails(data.tests.filter(isFailedReplayTestResult), { debug: options.debug }); renderFlakyTestSummary(flaky); return getReplayTestExitCode(data); } -function renderDefaultTestResult(result: ReplaySuiteTestResult): void { - if (result.status === 'failed') { - renderFailedTestResult(result); - return; - } - if (result.status !== 'passed') return; - - process.stdout.write( - `PASS ${replayTestDisplayName(result)}${formatReplayTestDurationSuffix(result)}\n`, - ); - for (const line of replayTestWarningLines(result)) { - process.stdout.write(` ${line}\n`); - } -} - -function renderVerboseTestResult(result: ReplaySuiteTestResult): void { - if (result.status === 'failed') { - renderFailedTestResult(result); - return; - } - - const durationSuffix = formatReplayTestDurationSuffix(result); - process.stdout.write( - `${replayResultPrefix(result)} ${replayTestDisplayName(result)}${durationSuffix}\n`, - ); - if (result.status === 'skipped') { - process.stdout.write(` ${result.message ?? 'skipped'}\n`); - } - for (const line of replayTestWarningLines(result)) { - process.stdout.write(` ${line}\n`); - } - for (const line of replayTestStepLines(result)) { - process.stdout.write(` ${line}\n`); - } +function formatReplayTestSummaryLine(data: ReplaySuiteResult, flakyCount: number): string { + const durationMs = typeof data.durationMs === 'number' ? data.durationMs : undefined; + const flakySuffix = flakyCount > 0 ? `, ${flakyCount} flaky` : ''; + const durationSuffix = durationMs !== undefined ? ` in ${formatDurationSeconds(durationMs)}` : ''; + return `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationSuffix}`; } -function renderFailedTestResult(result: FailedReplayTestResult): void { - const attemptSuffix = result.attempts > 1 ? ` after ${result.attempts} attempts` : ''; - const durationSuffix = formatReplayTestDurationSuffix(result); - process.stdout.write( - `FAIL ${replayFailedTestDisplayName(result)}${attemptSuffix}${durationSuffix}\n`, - ); - process.stdout.write(` ${result.error?.message ?? 'Unknown test failure'}\n`); - for (const line of replayFailureConsoleLines(result)) { - process.stdout.write(` ${line}\n`); - } - for (const line of replayTestStepLines(result)) { - process.stdout.write(` ${line}\n`); - } +function replayFlakyStatusIcon(): string { + const useColor = supportsColor(); + return useColor ? colorize('✓', 'yellow') : '✓'; } -function replayResultPrefix(result: ReplaySuiteTestResult): string { - if (result.status === 'passed') return 'PASS'; - if (result.status === 'skipped') return 'SKIP'; - return 'INFO'; +function isFailedReplayTestResult(result: ReplaySuiteTestResult): result is FailedReplayTestResult { + return result.status === 'failed'; } function replayFailureConsoleLines(result: FailedReplayTestResult): string[] { @@ -124,10 +69,11 @@ function isFlakyReplayTestResult(result: ReplaySuiteTestResult): result is Passe function renderFlakyTestSummary(results: PassedReplayTestResult[]): void { if (results.length === 0) return; + process.stdout.write('\n'); process.stdout.write('Flaky tests:\n'); for (const result of results) { process.stdout.write( - ` PASS ${replayTestDisplayName(result)} after ${result.attempts} attempts${formatFlakyReplayDurationSuffix(result)}\n`, + ` ${replayFlakyStatusIcon()} ${replayTestDisplayNameWithFile(result)} after ${result.attempts} attempts${formatFlakyReplayDurationSuffix(result)}\n`, ); for (const failure of result.attemptFailures ?? []) { const attemptDuration = @@ -141,13 +87,35 @@ function renderFlakyTestSummary(results: PassedReplayTestResult[]): void { } } -function replayTestDisplayName(result: ReplaySuiteTestResult): string { - const title = replayTestTitle(result); - const base = title && title.length > 0 ? JSON.stringify(title) : path.basename(result.file); - return `${base}${formatReplayTestShardSuffix(result)}`; +function renderFailureDetails( + results: FailedReplayTestResult[], + options: { debug?: boolean } = {}, +): void { + if (results.length === 0) return; + process.stdout.write('\n'); + process.stdout.write('Failures:\n'); + for (const result of results) { + process.stdout.write(` ${replayTestDisplayNameWithFile(result)}\n`); + renderReplayFailureBody(result, { debug: options.debug, indent: ' ' }); + } } -function replayFailedTestDisplayName(result: FailedReplayTestResult): string { +function renderReplayFailureBody( + result: FailedReplayTestResult, + options: { debug?: boolean; indent: string }, +): void { + const { debug, indent } = options; + process.stdout.write(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`); + for (const line of replayFailureConsoleLines(result)) { + process.stdout.write(`${indent}${line}\n`); + } + if (!debug) return; + for (const line of replayTestFailureStepLines(result)) { + process.stdout.write(`${indent}${line}\n`); + } +} + +function replayTestDisplayNameWithFile(result: ReplaySuiteTestResult): string { const title = replayTestTitle(result); const filename = path.basename(result.file); const base = title && title.length > 0 ? `${JSON.stringify(title)} in ${filename}` : filename; @@ -163,21 +131,6 @@ function replayTestTitle(result: ReplaySuiteTestResult): string | undefined { return title && title.length > 0 ? title : undefined; } -function formatReplayTestDurationSuffix(result: ReplaySuiteTestResult): string { - if (result.status === 'passed' && result.attempts > 1) { - return formatFlakyReplayDurationSuffix(result); - } - if (result.status === 'failed' && result.attempts > 1 && result.durationMs > 0) { - return ` (total ${formatDurationSeconds(result.durationMs)})`; - } - - const durationMs = - result.status === 'passed' && typeof result.finalAttemptDurationMs === 'number' - ? result.finalAttemptDurationMs - : result.durationMs; - return durationMs > 0 ? ` (${formatDurationSeconds(durationMs)})` : ''; -} - function formatFlakyReplayDurationSuffix(result: PassedReplayTestResult): string { const timings = [ typeof result.finalAttemptDurationMs === 'number' diff --git a/src/cli.ts b/src/cli.ts index 0252c673c..904bb9e61 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -317,6 +317,7 @@ export async function runCli(argv: string[], deps: CliDeps = DEFAULT_CLI_DEPS): positionals, flags: { ...effectiveFlags, batchSteps }, client, + debug: debugOutputEnabled, }) ) { return; @@ -327,7 +328,13 @@ export async function runCli(argv: string[], deps: CliDeps = DEFAULT_CLI_DEPS): 'runtime command was removed. Use connect --remote-config for remote runs, or metro prepare --remote-config for inspection.', ); } else if ( - await tryRunClientBackedCommand({ command, positionals, flags: effectiveFlags, client }) + await tryRunClientBackedCommand({ + command, + positionals, + flags: effectiveFlags, + client, + debug: debugOutputEnabled, + }) ) { return; } diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index db8fed2e1..c1c573896 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -15,6 +15,7 @@ export async function runGenericClientBackedCommand({ positionals, flags, client, + debug, }: ClientCommandParams & { command: ClientBackedCliCommandName }): Promise { const { result, cliOutput } = await runCliCommandWithOutput({ client, @@ -25,7 +26,7 @@ export async function runGenericClientBackedCommand({ if (cliOutput) { writeCliOutput(flags, cliOutput); } else { - const exitCode = writeGenericCliOutput(command, flags, result); + const exitCode = writeGenericCliOutput(command, flags, result, { debug }); if (exitCode !== 0) { process.exit(exitCode); } @@ -37,11 +38,12 @@ function writeGenericCliOutput( command: ClientBackedCliCommandName, flags: CliFlags, data: CommandRequestResult, + options: { debug?: boolean } = {}, ): number { if (command === 'test') { return renderReplayTestResponse({ suite: data as ReplaySuiteResult, - verbose: flags.verbose, + debug: options.debug, json: flags.json, reportJunit: flags.reportJunit, }); diff --git a/src/cli/commands/router-types.ts b/src/cli/commands/router-types.ts index 14f8ef96d..5c8dda99d 100644 --- a/src/cli/commands/router-types.ts +++ b/src/cli/commands/router-types.ts @@ -6,6 +6,7 @@ export type ClientCommandParams = { positionals: string[]; flags: CliFlags; client: AgentDeviceClient; + debug?: boolean; }; /** diff --git a/src/cli/commands/router.ts b/src/cli/commands/router.ts index 3d3c0b326..1e2104e8a 100644 --- a/src/cli/commands/router.ts +++ b/src/cli/commands/router.ts @@ -31,6 +31,7 @@ export async function tryRunClientBackedCommand(params: { positionals: string[]; flags: CliFlags; client: AgentDeviceClient; + debug?: boolean; }): Promise { const flags = { ...params.flags }; const dedicatedHandler = diff --git a/src/daemon-client-progress.ts b/src/daemon-client-progress.ts index f2c6b4abf..26a647027 100644 --- a/src/daemon-client-progress.ts +++ b/src/daemon-client-progress.ts @@ -4,16 +4,35 @@ import { AppError } from './utils/errors.ts'; import type { DaemonRequest, DaemonResponse } from './daemon/types.ts'; import type { RequestProgressEvent } from './daemon/request-progress.ts'; import { consumeTextLines } from './utils/line-stream.ts'; -import { formatReplayTestProgressEvent } from './cli-test-progress.ts'; +import { + createReplayTestProgressRenderer, + type ReplayTestProgressRenderer, +} from './cli-test-progress.ts'; import { isDaemonProgressEnvelope, isDaemonResponseEnvelope, shouldStreamRequestProgress, } from './daemon/request-progress-protocol.ts'; -function writeRequestProgressEvent(event: RequestProgressEvent): void { - const line = formatReplayTestProgressEvent(event); - if (line) process.stderr.write(`${line}\n`); +function createRequestProgressRenderer(req: DaemonRequest): ReplayTestProgressRenderer { + return createReplayTestProgressRenderer({ + verbose: Boolean(req.flags?.verbose || req.meta?.debug), + liveProgress: shouldRenderLiveProgress(), + columns: process.stderr.columns, + }); +} + +function writeRequestProgressEvent( + event: RequestProgressEvent, + renderer: ReplayTestProgressRenderer, +): void { + const output = renderer.render(event); + if (!output) return; + process.stderr.write(output.newline ? `${output.text}\n` : output.text); +} + +function shouldRenderLiveProgress(): boolean { + return process.stderr.isTTY === true && !process.env.CI; } export function shouldReadDaemonProgressStream( @@ -40,6 +59,7 @@ export function readDaemonSocketProgressResponse( ): void { const { req, isSettled, resolve, reject, clearTimeout } = options; let buffer = ''; + const progressRenderer = createRequestProgressRenderer(req); const rejectInvalidLine = (line: string, error: unknown) => { clearTimeout(); @@ -65,7 +85,7 @@ export function readDaemonSocketProgressResponse( try { const message = JSON.parse(line) as unknown; if (isDaemonProgressEnvelope(message)) { - writeRequestProgressEvent(message.event); + writeRequestProgressEvent(message.event, progressRenderer); continue; } const response = isDaemonResponseEnvelope(message) ? message.response : message; @@ -93,6 +113,7 @@ export function readDaemonHttpProgressResponse( const { req, handleResponseBody, reject, clearTimeout } = options; let buffer = ''; let settled = false; + const progressRenderer = createRequestProgressRenderer(req); const rejectInvalidLine = (line: string, error: unknown) => { settled = true; clearTimeout(); @@ -113,7 +134,7 @@ export function readDaemonHttpProgressResponse( try { const message = JSON.parse(line) as unknown; if (isDaemonProgressEnvelope(message)) { - writeRequestProgressEvent(message.event); + writeRequestProgressEvent(message.event, progressRenderer); return false; } if (isDaemonResponseEnvelope(message)) { diff --git a/src/daemon/handlers/__tests__/session-test-suite.test.ts b/src/daemon/handlers/__tests__/session-test-suite.test.ts index f732be1b7..0c4db5d61 100644 --- a/src/daemon/handlers/__tests__/session-test-suite.test.ts +++ b/src/daemon/handlers/__tests__/session-test-suite.test.ts @@ -171,8 +171,25 @@ test('test emits progress when attempts retry and pass', async () => { skipped: 0, }); const testEvents = events.filter((event) => event.type === 'replay-test'); - expect(testEvents.map((event) => event.status)).toEqual(['start', 'fail', 'pass']); + expect(testEvents.map((event) => event.status)).toEqual([ + 'start', + 'progress', + 'fail', + 'progress', + 'pass', + ]); expect(testEvents[1]).toMatchObject({ + type: 'replay-test', + title: undefined, + status: 'progress', + index: 1, + total: 1, + attempt: 1, + maxAttempts: 2, + stepIndex: 1, + stepTotal: 1, + }); + expect(testEvents[2]).toMatchObject({ type: 'replay-test', title: undefined, status: 'fail', @@ -184,7 +201,18 @@ test('test emits progress when attempts retry and pass', async () => { retrying: true, message: 'Replay failed at step 1 (open "Demo"): first attempt failed', }); - expect(testEvents[2]).toMatchObject({ + expect(testEvents[3]).toMatchObject({ + type: 'replay-test', + title: undefined, + status: 'progress', + index: 1, + total: 1, + attempt: 2, + maxAttempts: 2, + stepIndex: 1, + stepTotal: 1, + }); + expect(testEvents[4]).toMatchObject({ type: 'replay-test', title: undefined, status: 'pass', @@ -231,7 +259,7 @@ test('test emits skip progress without synthetic duration', async () => { skipped: 1, }); const testEvents = events.filter((event) => event.type === 'replay-test'); - expect(testEvents.map((event) => event.status)).toEqual(['skip', 'start', 'pass']); + expect(testEvents.map((event) => event.status)).toEqual(['skip', 'start', 'progress', 'pass']); expect(testEvents[0]).toMatchObject({ type: 'replay-test', status: 'skip', @@ -239,6 +267,14 @@ test('test emits skip progress without synthetic duration', async () => { total: 2, message: 'missing platform metadata for --platform android', }); + expect(testEvents[2]).toMatchObject({ + type: 'replay-test', + status: 'progress', + index: 2, + total: 2, + stepIndex: 1, + stepTotal: 1, + }); expect(testEvents[0]?.durationMs).toBeUndefined(); }); diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index ee751ec03..ed0d8431c 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -4,6 +4,7 @@ import { type CommandFlags } from '../../core/dispatch.ts'; import { parseReplayInput } from '../../compat/replay-input.ts'; import { asAppError } from '../../utils/errors.ts'; import type { DaemonInvokeFn, DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; +import { emitRequestProgress, readReplayTestActionProgress } from '../request-progress.ts'; import { SessionStore } from '../session-store.ts'; import { type ReplayScriptMetadata, writeReplayScript } from '../../replay/script.ts'; import { healReplayAction } from './session-replay-heal.ts'; @@ -92,6 +93,7 @@ export async function runReplayScriptFile(params: { for (let index = 0; index < actions.length; index += 1) { const action = actions[index]; if (!action || action.command === 'replay') continue; + emitReplayTestActionProgress(resolved, index, actions.length); const sampleStart = readSessionSnapshotSampleCount(sessionStore, sessionName); let response = await invokeReplayAction({ @@ -233,6 +235,23 @@ function buildReplayBuiltinVars(params: { return builtins; } +function emitReplayTestActionProgress( + file: string, + actionIndex: number, + actionTotal: number, +): void { + const progress = readReplayTestActionProgress(); + if (!progress) return; + emitRequestProgress({ + type: 'replay-test', + ...progress, + file: progress.file || file, + status: 'progress', + stepIndex: actionIndex + 1, + stepTotal: actionTotal, + }); +} + function buildReplayMetadataFlags( flags: CommandFlags | undefined, metadata: ReplayScriptMetadata, diff --git a/src/daemon/handlers/session-test-attempt.ts b/src/daemon/handlers/session-test-attempt.ts index 286674010..c79a081be 100644 --- a/src/daemon/handlers/session-test-attempt.ts +++ b/src/daemon/handlers/session-test-attempt.ts @@ -108,7 +108,17 @@ async function runSingleReplayTestAttempt( context: ReplayTestCaseContext, attemptIndex: number, ): Promise { - const { entry, sessionName, suiteInvocationId, caseIndex, requestId, timeoutMs, shard } = params; + const { + entry, + sessionName, + suiteInvocationId, + caseIndex, + suiteIndex, + suiteTotal, + requestId, + timeoutMs, + shard, + } = params; const attempt = attemptIndex + 1; const startedAt = Date.now(); const testSessionName = buildReplayTestSessionName( @@ -139,6 +149,17 @@ async function runSingleReplayTestAttempt( target: entry.metadata.target, artifactsDir: attemptArtifactsDir, shard, + progress: { + file: entry.path, + title: entry.title, + index: suiteIndex, + total: suiteTotal, + attempt, + maxAttempts: context.maxAttempts, + session: testSessionName, + artifactsDir: context.testArtifactsDir, + ...replayTestProgressShardMetadata(shard), + }, runReplay: params.runReplay, cleanupSession: params.cleanupSession, finalizeAttempt: params.finalizeAttempt, diff --git a/src/daemon/handlers/session-test-runtime.ts b/src/daemon/handlers/session-test-runtime.ts index a03d1066f..0a619a337 100644 --- a/src/daemon/handlers/session-test-runtime.ts +++ b/src/daemon/handlers/session-test-runtime.ts @@ -9,6 +9,10 @@ import { markRequestCanceled, registerRequestAbort, } from '../request-cancel.ts'; +import { + type ReplayTestActionProgressContext, + withReplayTestActionProgress, +} from '../request-progress.ts'; import type { DaemonResponse } from '../types.ts'; import type { ReplayScriptMetadata } from '../../replay/script.ts'; import type { @@ -32,6 +36,7 @@ export async function runReplayTestAttempt( target?: ReplayScriptMetadata['target']; artifactsDir?: string; shard?: ReplayTestRunReplayParams['shard']; + progress?: ReplayTestActionProgressContext; } & ReplayTestRuntimeDependencies, ): Promise { const { @@ -44,6 +49,7 @@ export async function runReplayTestAttempt( target, artifactsDir, shard, + progress, runReplay, cleanupSession, finalizeAttempt, @@ -65,17 +71,21 @@ export async function runReplayTestAttempt( platform, target, }); - const replayPromise = runReplay({ - filePath, - sessionName, - platform, - target, - requestId, - artifactsDir, - artifactPaths, - tracePath, - shard, - }) + const replayPromise = withReplayTestActionProgress( + progress, + async () => + await runReplay({ + filePath, + sessionName, + platform, + target, + requestId, + artifactsDir, + artifactPaths, + tracePath, + shard, + }), + ) .catch((error) => { const appErr = normalizeError(error); return { diff --git a/src/daemon/request-progress.ts b/src/daemon/request-progress.ts index bd5a5fad1..90c950eec 100644 --- a/src/daemon/request-progress.ts +++ b/src/daemon/request-progress.ts @@ -15,9 +15,11 @@ export type ReplayTestProgressEvent = { type: 'replay-test'; file: string; title?: string; - status: 'start' | 'pass' | 'fail' | 'skip'; + status: 'start' | 'progress' | 'pass' | 'fail' | 'skip'; index: number; total: number; + stepIndex?: number; + stepTotal?: number; attempt?: number; maxAttempts?: number; durationMs?: number; @@ -32,8 +34,15 @@ export type ReplayTestProgressEvent = { export type RequestProgressEvent = ReplayTestSuiteProgressEvent | ReplayTestProgressEvent; export type RequestProgressSink = (event: RequestProgressEvent) => void; +export type ReplayTestActionProgressContext = Omit< + ReplayTestProgressEvent, + 'type' | 'status' | 'stepIndex' | 'stepTotal' | 'durationMs' | 'retrying' | 'message' +>; const requestProgress = new AsyncLocalStorage(); +const replayTestActionProgress = new AsyncLocalStorage< + ReplayTestActionProgressContext | undefined +>(); export async function withRequestProgressSink( sink: RequestProgressSink | undefined, @@ -45,3 +54,14 @@ export async function withRequestProgressSink( export function emitRequestProgress(event: RequestProgressEvent): void { requestProgress.getStore()?.(event); } + +export async function withReplayTestActionProgress( + context: ReplayTestActionProgressContext | undefined, + run: () => Promise, +): Promise { + return await replayTestActionProgress.run(context, run); +} + +export function readReplayTestActionProgress(): ReplayTestActionProgressContext | undefined { + return replayTestActionProgress.getStore(); +} diff --git a/src/utils/__tests__/daemon-client.test.ts b/src/utils/__tests__/daemon-client.test.ts index e550f6382..53e68af29 100644 --- a/src/utils/__tests__/daemon-client.test.ts +++ b/src/utils/__tests__/daemon-client.test.ts @@ -418,6 +418,46 @@ test('sendToDaemon reuses reachable local socket daemon metadata', async (t) => test('sendToDaemon prints replay test progress before the socket response', async () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-socket-progress-')); + const artifactsDir = path.join(stateDir, 'login-flow'); + const attemptDir = path.join(artifactsDir, 'attempt-2'); + fs.mkdirSync(attemptDir, { recursive: true }); + fs.writeFileSync( + path.join(attemptDir, 'replay-timing.ndjson'), + [ + { + type: 'replay_action_start', + step: 1, + line: 3, + command: 'open', + positionals: ['Demo'], + }, + { + type: 'replay_action_stop', + step: 1, + line: 3, + command: 'open', + ok: true, + durationMs: 250, + }, + { + type: 'replay_action_start', + step: 2, + line: 4, + command: '__maestroAssertVisible', + positionals: ['text="Home"', '3000'], + }, + { + type: 'replay_action_stop', + step: 2, + line: 4, + command: '__maestroAssertVisible', + ok: true, + durationMs: 750, + }, + ] + .map((entry) => JSON.stringify(entry)) + .join('\n'), + ); let stderr = ''; const originalStderrWrite = process.stderr.write.bind(process.stderr); const originalCreateConnection = net.createConnection; @@ -445,25 +485,23 @@ test('sendToDaemon prints replay test progress before the socket response', asyn socket.write = () => { if (createConnectionCalls === 2) { process.nextTick(() => { - socket.emit( - 'data', - `${JSON.stringify({ - type: 'progress', - event: { - type: 'replay-test', - file: '/tmp/01-login.ad', - title: 'Login flow', - status: 'fail', - index: 1, - total: 2, - attempt: 1, - maxAttempts: 2, - durationMs: 1234, - retrying: true, - message: 'first attempt failed', - }, - })}\n`, - ); + const progress = `${JSON.stringify({ + type: 'progress', + event: { + type: 'replay-test', + file: '/tmp/01-login.ad', + title: 'Login flow', + status: 'pass', + index: 1, + total: 2, + attempt: 2, + maxAttempts: 2, + durationMs: 1234, + artifactsDir, + }, + })}\n`; + socket.emit('data', progress); + socket.emit('data', progress); socket.emit( 'data', `${JSON.stringify({ @@ -491,13 +529,16 @@ test('sendToDaemon prints replay test progress before the socket response', asyn session: 'default', command: 'test', positionals: ['/tmp/replays'], - flags: { stateDir, daemonTransport: 'socket' }, + flags: { stateDir, daemonTransport: 'socket', verbose: true }, meta: { requestId: 'req-progress', requestProgress: 'replay-test' }, }); assert.deepEqual(response, { ok: true, data: { via: 'socket' } }); - assert.match(stderr, /\[1\/2] RETRY "Login flow" in 01-login\.ad attempt 1\/2 \(1\.23s\)/); - assert.match(stderr, / first attempt failed/); + assert.match(stderr, /✓ "Login flow" in 01-login\.ad \(1\.23s\)/); + assert.equal(stderr.match(/✓ "Login flow" in 01-login\.ad \(1\.23s\)/g)?.length, 1); + assert.match(stderr, /steps \(attempt 2\):/); + assert.match(stderr, /open "Demo" \(line 3, 0\.25s\)/); + assert.match(stderr, /assertVisible "text=\\"Home\\"" "3000" \(line 4, 0\.75s\)/); } finally { (net as unknown as { createConnection: typeof net.createConnection }).createConnection = originalCreateConnection; @@ -598,7 +639,7 @@ test('sendToDaemon prints replay test progress before the HTTP NDJSON response', assert.deepEqual(response, { ok: true, data: { via: 'http-progress' } }); }); assert.deepEqual(seenPaths, ['GET /agent-device/health', 'POST /agent-device/rpc']); - assert.match(stderr, /\[2\/3] PASS "Payments flow" in 02-payments\.ad \(2\.50s\)/); + assert.match(stderr, /✓ "Payments flow" in 02-payments\.ad \(2\.50s\)/); } finally { (http as unknown as { request: typeof http.request }).request = originalHttpRequest; process.stderr.write = originalStderrWrite; diff --git a/src/utils/output.ts b/src/utils/output.ts index 7d7d249b3..0f9636b84 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -585,7 +585,7 @@ function applyContextWindow(lines: SnapshotDiffLine[], contextWindow: number): S return lines.filter((_, index) => keep[index]); } -function supportsColor(): boolean { +export function supportsColor(stream: { isTTY?: boolean } = process.stdout): boolean { const forceColor = process.env.FORCE_COLOR; if (typeof forceColor === 'string') { return forceColor !== '0'; @@ -593,11 +593,15 @@ function supportsColor(): boolean { if (typeof process.env.NO_COLOR === 'string') { return false; } - return Boolean(process.stdout.isTTY); + return Boolean(stream.isTTY); } -function colorize(text: string, format: Parameters[0]): string { - return styleText(format, text); +export function colorize( + text: string, + format: Parameters[0], + options?: Parameters[2], +): string { + return styleText(format, text, options); } function formatMuted(text: string, useColor: boolean): string {