From 107381db10255f567e41dc9c18f6c6e3ba59be57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 19 Jun 2026 10:35:58 +0200 Subject: [PATCH 1/2] feat: add semantic web provider seam --- src/core/__tests__/web-interactor.test.ts | 95 +++++++++++++++++++ src/core/interactor-types.ts | 2 +- src/core/interactors.ts | 4 + src/core/interactors/web.ts | 50 ++++++++++ .../request-platform-providers.test.ts | 77 +++++++++++++++ src/daemon/request-platform-providers.ts | 20 ++++ src/platforms/web/provider.ts | 71 ++++++++++++++ src/utils/snapshot.ts | 2 +- 8 files changed, 319 insertions(+), 2 deletions(-) create mode 100644 src/core/__tests__/web-interactor.test.ts create mode 100644 src/core/interactors/web.ts create mode 100644 src/platforms/web/provider.ts diff --git a/src/core/__tests__/web-interactor.test.ts b/src/core/__tests__/web-interactor.test.ts new file mode 100644 index 000000000..7b40737c5 --- /dev/null +++ b/src/core/__tests__/web-interactor.test.ts @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createWebInteractor } from '../interactors/web.ts'; +import { AppError } from '../../utils/errors.ts'; +import { withWebProvider, type WebProvider } from '../../platforms/web/provider.ts'; + +test('web interactor delegates first-slice operations to the scoped provider', async () => { + const calls: string[] = []; + const interactor = createWebInteractor(); + const provider = makeWebProvider({ + async open(target, options) { + calls.push(`open:${target}:${options?.url ?? ''}`); + }, + async close(target) { + calls.push(`close:${target ?? ''}`); + }, + async snapshot(options) { + calls.push(`snapshot:${options?.scope ?? ''}`); + return { + nodes: [{ index: 0, role: 'button', label: 'Submit' }], + truncated: true, + }; + }, + async screenshot(outPath, options) { + calls.push(`screenshot:${outPath}:${options?.fullscreen === true}`); + }, + async click(x, y) { + calls.push(`click:${x}:${y}`); + }, + async fill(x, y, text, options) { + calls.push(`fill:${x}:${y}:${text}:${options?.delayMs ?? 0}`); + }, + async typeText(text, options) { + calls.push(`type:${text}:${options?.delayMs ?? 0}`); + }, + async scroll(direction, options) { + calls.push(`scroll:${direction}:${options?.pixels ?? options?.amount ?? ''}`); + }, + }); + + const snapshot = await withWebProvider(provider, async () => { + await interactor.open('https://example.test'); + await interactor.open('app-shell', { url: 'https://example.test/deep' }); + await interactor.close('app-shell'); + await interactor.tap(10, 20); + await interactor.focus(11, 21); + await interactor.fill(12, 22, 'hello', 5); + await interactor.type('world', 6); + await interactor.scroll('down', { pixels: 400 }); + await interactor.screenshot('/tmp/web.png', { fullscreen: true }); + return await interactor.snapshot({ scope: 'main' }); + }); + + assert.deepEqual(calls, [ + 'open:https://example.test:', + 'open:https://example.test/deep:https://example.test/deep', + 'close:app-shell', + 'click:10:20', + 'click:11:21', + 'fill:12:22:hello:5', + 'type:world:6', + 'scroll:down:400', + 'screenshot:/tmp/web.png:true', + 'snapshot:main', + ]); + assert.equal(snapshot.backend, 'web'); + assert.equal(snapshot.truncated, true); + assert.deepEqual(snapshot.nodes, [{ index: 0, role: 'button', label: 'Submit' }]); +}); + +test('web interactor reports unsupported operations explicitly', async () => { + const interactor = createWebInteractor(); + + await assert.rejects( + () => interactor.back(), + (error: unknown) => + error instanceof AppError && + error.code === 'UNSUPPORTED_OPERATION' && + error.message === 'back is not supported on web', + ); +}); + +function makeWebProvider(overrides: Partial = {}): WebProvider { + return { + open: async () => {}, + close: async () => {}, + snapshot: async () => ({ nodes: [] }), + screenshot: async () => {}, + click: async () => {}, + fill: async () => {}, + typeText: async () => {}, + scroll: async () => {}, + ...overrides, + }; +} diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index 2cb313d36..17154b934 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -57,7 +57,7 @@ export type SnapshotOptions = BaseSnapshotOptions & { export type SnapshotResult = Omit & { nodes?: RawSnapshotNode[]; - backend: Extract; + backend: Extract; }; export type Interactor = { diff --git a/src/core/interactors.ts b/src/core/interactors.ts index e6b9152cc..f8cb36b3c 100644 --- a/src/core/interactors.ts +++ b/src/core/interactors.ts @@ -15,6 +15,10 @@ export async function getInteractor( const { createLinuxInteractor } = await import('./interactors/linux.ts'); return createLinuxInteractor(); } + case 'web': { + const { createWebInteractor } = await import('./interactors/web.ts'); + return createWebInteractor(); + } case 'ios': case 'macos': { const { createAppleInteractor } = await import('./interactors/apple.ts'); diff --git a/src/core/interactors/web.ts b/src/core/interactors/web.ts new file mode 100644 index 000000000..753e0981b --- /dev/null +++ b/src/core/interactors/web.ts @@ -0,0 +1,50 @@ +import type { Interactor } from '../interactor-types.ts'; +import { AppError } from '../../utils/errors.ts'; +import { withDiagnosticTimer } from '../../utils/diagnostics.ts'; +import { resolveWebProvider } from '../../platforms/web/provider.ts'; + +export function createWebInteractor(): Interactor { + const provider = () => resolveWebProvider(); + return { + open: (target, options) => provider().open(options?.url ?? target, { url: options?.url }), + openDevice: () => provider().open('about:blank'), + close: (target) => provider().close(target), + tap: (x, y) => provider().click(x, y), + doubleTap: () => unsupportedWebOperation('doubleTap'), + swipe: () => unsupportedWebOperation('swipe'), + pan: () => unsupportedWebOperation('pan'), + fling: () => unsupportedWebOperation('fling'), + longPress: () => unsupportedWebOperation('longPress'), + focus: (x, y) => provider().click(x, y), + type: (text, delayMs) => provider().typeText(text, { delayMs }), + fill: (x, y, text, delayMs) => provider().fill(x, y, text, { delayMs }), + scroll: (direction, options) => provider().scroll(direction, options), + pinch: () => unsupportedWebOperation('pinch'), + screenshot: (outPath, options) => provider().screenshot(outPath, options), + snapshot: async (options) => { + const result = await withDiagnosticTimer( + 'snapshot_capture', + async () => await provider().snapshot(options), + { backend: 'web' }, + ); + return { + nodes: result.nodes, + truncated: result.truncated ?? false, + backend: 'web', + }; + }, + back: () => unsupportedWebOperation('back'), + home: () => unsupportedWebOperation('home'), + rotate: () => unsupportedWebOperation('rotate'), + rotateGesture: () => unsupportedWebOperation('rotateGesture'), + transformGesture: () => unsupportedWebOperation('transformGesture'), + appSwitcher: () => unsupportedWebOperation('appSwitcher'), + readClipboard: () => unsupportedWebOperation('readClipboard'), + writeClipboard: () => unsupportedWebOperation('writeClipboard'), + setSetting: () => unsupportedWebOperation('setSetting'), + }; +} + +async function unsupportedWebOperation(operation: string): Promise { + throw new AppError('UNSUPPORTED_OPERATION', `${operation} is not supported on web`); +} diff --git a/src/daemon/__tests__/request-platform-providers.test.ts b/src/daemon/__tests__/request-platform-providers.test.ts index 191c5c711..730cf4b31 100644 --- a/src/daemon/__tests__/request-platform-providers.test.ts +++ b/src/daemon/__tests__/request-platform-providers.test.ts @@ -3,11 +3,14 @@ import { test } from 'vitest'; import { ANDROID_EMULATOR, IOS_SIMULATOR, + WEB_DESKTOP_DEVICE, makeAndroidSession, makeIosSession, + makeSession, } from '../../__tests__/test-utils/index.ts'; import { withTargetDeviceResolutionScope } from '../../core/dispatch-resolve.ts'; import { createLocalAppleToolProvider, runXcrun } from '../../platforms/ios/tool-provider.ts'; +import { resolveWebProvider, type WebProvider } from '../../platforms/web/provider.ts'; import type { DeviceInfo } from '../../utils/device.ts'; import { withRequestPlatformProviderScope } from '../request-platform-providers.ts'; import type { DaemonRequest } from '../types.ts'; @@ -194,6 +197,66 @@ test('request platform provider scopes stay isolated across concurrent requests' assert.deepEqual(appleCalls, [`ios-session:${IOS_SIMULATOR.id}:list devices -j`]); }); +test('request platform provider scope applies web provider only for web sessions', async () => { + const calls: string[] = []; + const webProvider = makeWebProvider({ + async open(target) { + calls.push(`open:${target}`); + }, + }); + + await withRequestPlatformProviderScope( + { + req: request('open'), + existingSession: makeSession('web-session', { device: WEB_DESKTOP_DEVICE }), + providers: { + webProvider: ({ device, session }) => { + calls.push(`${session?.name}:${device.id}`); + return webProvider; + }, + linuxToolProvider: () => { + throw new Error('Linux provider should not apply to a web session'); + }, + }, + }, + async () => await resolveWebProvider().open('https://example.test'), + ); + + assert.deepEqual(calls, ['web-session:agent-browser-chrome', 'open:https://example.test']); +}); + +test('request platform provider scope follows explicit web selector', async () => { + const seenDevices: string[] = []; + + await withTargetDeviceResolutionScope( + async () => [WEB_DESKTOP_DEVICE], + async () => + await withRequestPlatformProviderScope( + { + req: { + ...request('snapshot'), + flags: { + platform: 'web', + }, + }, + existingSession: undefined, + providers: { + webProvider: ({ device, session }) => { + seenDevices.push(`${session?.name ?? 'none'}:${device.id}`); + return makeWebProvider(); + }, + appleToolProvider: () => { + throw new Error('Apple provider should not apply to a web request'); + }, + }, + }, + async () => await resolveWebProvider().snapshot(), + ), + ); + + assert.deepEqual(seenDevices, ['none:agent-browser-chrome']); +}); + function request(command: string): DaemonRequest { return { token: 'test-token', @@ -204,3 +267,17 @@ function request(command: string): DaemonRequest { meta: { requestId: `req-${command}` }, }; } + +function makeWebProvider(overrides: Partial = {}): WebProvider { + return { + open: async () => {}, + close: async () => {}, + snapshot: async () => ({ nodes: [] }), + screenshot: async () => {}, + click: async () => {}, + fill: async () => {}, + typeText: async () => {}, + scroll: async () => {}, + ...overrides, + }; +} diff --git a/src/daemon/request-platform-providers.ts b/src/daemon/request-platform-providers.ts index fc45e35c1..cec65c533 100644 --- a/src/daemon/request-platform-providers.ts +++ b/src/daemon/request-platform-providers.ts @@ -9,6 +9,7 @@ import type { AppleToolProvider, } from '../platforms/ios/tool-provider.ts'; import type { LinuxToolProvider } from '../platforms/linux/tool-provider.ts'; +import type { WebProvider } from '../platforms/web/provider.ts'; import { isApplePlatform, type DeviceInfo } from '../utils/device.ts'; import type { AppLogProvider } from './app-log.ts'; import { hasExplicitDeviceSelector } from './device-selector-intent.ts'; @@ -39,6 +40,8 @@ export type AppleToolProviderResolver = PlatformProviderResolver< export type LinuxToolProviderResolver = PlatformProviderResolver; +export type WebProviderResolver = PlatformProviderResolver; + export type AppLogProviderResolver = PlatformProviderResolver; export type RecordingProviderResolver = PlatformProviderResolver; @@ -48,6 +51,7 @@ export type PlatformProviderResolvers = { appleRunnerProvider?: AppleRunnerProviderResolver; appleToolProvider?: AppleToolProviderResolver; linuxToolProvider?: LinuxToolProviderResolver; + webProvider?: WebProviderResolver; appLogProvider?: AppLogProviderResolver; recordingProvider?: RecordingProviderResolver; }; @@ -85,6 +89,9 @@ type ResolvedRequestPlatformProviders = { linuxTool?: { provider?: LinuxToolProvider; }; + web?: { + provider?: WebProvider; + }; appLog?: { provider?: AppLogProvider; }; @@ -184,6 +191,19 @@ const REQUEST_PLATFORM_PROVIDER_DESCRIPTORS = [ appendRequestProviderWrapper(wrappers, scopedProviders.linuxTool, withLinuxToolProvider); }, }, + { + resolverKey: 'webProvider', + resolve(providers, context) { + const webProvider = providers.webProvider; + if (!webProvider || context.device.platform !== 'web') return {}; + return { web: { provider: webProvider(context) } }; + }, + async appendWrapper(scopedProviders, wrappers) { + if (!scopedProviders.web?.provider) return; + const { withWebProvider } = await import('../platforms/web/provider.ts'); + appendRequestProviderWrapper(wrappers, scopedProviders.web, withWebProvider); + }, + }, { resolverKey: 'appLogProvider', resolve(providers, context) { diff --git a/src/platforms/web/provider.ts b/src/platforms/web/provider.ts new file mode 100644 index 000000000..b62ff9d38 --- /dev/null +++ b/src/platforms/web/provider.ts @@ -0,0 +1,71 @@ +import type { ScrollDirection } from '../../core/scroll-gesture.ts'; +import type { SessionSurface } from '../../core/session-surface.ts'; +import { AppError } from '../../utils/errors.ts'; +import { createScopedProvider } from '../../utils/scoped-provider.ts'; +import type { RawSnapshotNode } from '../../utils/snapshot.ts'; + +export type WebOpenOptions = { + url?: string; +}; + +export type WebScreenshotOptions = { + fullscreen?: boolean; + stabilize?: boolean; + surface?: SessionSurface; +}; + +export type WebSnapshotOptions = { + interactiveOnly?: boolean; + depth?: number; + scope?: string; + raw?: boolean; + surface?: SessionSurface; +}; + +export type WebSnapshotResult = { + nodes: RawSnapshotNode[]; + truncated?: boolean; +}; + +export type WebProvider = { + open(target: string, options?: WebOpenOptions): Promise; + close(target?: string): Promise; + snapshot(options?: WebSnapshotOptions): Promise; + screenshot(outPath: string, options?: WebScreenshotOptions): Promise; + click(x: number, y: number): Promise; + fill(x: number, y: number, text: string, options?: { delayMs?: number }): Promise; + typeText(text: string, options?: { delayMs?: number }): Promise; + scroll(direction: ScrollDirection, options?: { amount?: number; pixels?: number }): Promise; + readText?(x: number, y: number): Promise; +}; + +const localWebProvider: WebProvider = { + open: () => unsupportedLocalWebProvider(), + close: () => unsupportedLocalWebProvider(), + snapshot: () => unsupportedLocalWebProvider(), + screenshot: () => unsupportedLocalWebProvider(), + click: () => unsupportedLocalWebProvider(), + fill: () => unsupportedLocalWebProvider(), + typeText: () => unsupportedLocalWebProvider(), + scroll: () => unsupportedLocalWebProvider(), +}; + +const webProviderScope = createScopedProvider(localWebProvider); + +export function resolveWebProvider(provider?: WebProvider): WebProvider { + return webProviderScope.resolve(provider); +} + +export async function withWebProvider( + provider: WebProvider | undefined, + fn: () => Promise, +): Promise { + return await webProviderScope.run(provider, fn); +} + +async function unsupportedLocalWebProvider(): Promise { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Web automation requires a request-scoped web provider.', + ); +} diff --git a/src/utils/snapshot.ts b/src/utils/snapshot.ts index 728cc4ec0..88c53f1bf 100644 --- a/src/utils/snapshot.ts +++ b/src/utils/snapshot.ts @@ -62,7 +62,7 @@ export type SnapshotNode = RawSnapshotNode & { ref: string; }; -export type SnapshotBackend = 'xctest' | 'android' | 'macos-helper' | 'linux-atspi'; +export type SnapshotBackend = 'xctest' | 'android' | 'macos-helper' | 'linux-atspi' | 'web'; export type SnapshotState = { nodes: SnapshotNode[]; From 77377d85afe29727100cfb5e68b0c46ddc128c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 19 Jun 2026 11:07:28 +0200 Subject: [PATCH 2/2] fix: forward web provider through request router --- src/daemon/request-router.ts | 4 + .../provider-scenarios/web-provider.test.ts | 114 ++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 test/integration/provider-scenarios/web-provider.test.ts diff --git a/src/daemon/request-router.ts b/src/daemon/request-router.ts index 5ddf4ebd8..08231ab42 100644 --- a/src/daemon/request-router.ts +++ b/src/daemon/request-router.ts @@ -14,6 +14,7 @@ import { type LinuxToolProviderResolver, type RequestPlatformProviderScope, type RecordingProviderResolver, + type WebProviderResolver, withRequestPlatformProviderScope, } from './request-platform-providers.ts'; import { @@ -46,6 +47,7 @@ export type RequestRouterDeps = { appleRunnerProvider?: AppleRunnerProviderResolver; appleToolProvider?: AppleToolProviderResolver; linuxToolProvider?: LinuxToolProviderResolver; + webProvider?: WebProviderResolver; appLogProvider?: AppLogProviderResolver; recordingProvider?: RecordingProviderResolver; deviceInventoryProvider?: DeviceInventoryProvider; @@ -64,6 +66,7 @@ export function createRequestHandler(deps: RequestRouterDeps): DaemonInvokeFn { appleRunnerProvider, appleToolProvider, linuxToolProvider, + webProvider, appLogProvider, recordingProvider, deviceInventoryProvider, @@ -132,6 +135,7 @@ export function createRequestHandler(deps: RequestRouterDeps): DaemonInvokeFn { appleRunnerProvider, appleToolProvider, linuxToolProvider, + webProvider, appLogProvider, recordingProvider, }, diff --git a/test/integration/provider-scenarios/web-provider.test.ts b/test/integration/provider-scenarios/web-provider.test.ts new file mode 100644 index 000000000..45d5b92ae --- /dev/null +++ b/test/integration/provider-scenarios/web-provider.test.ts @@ -0,0 +1,114 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { WEB_DESKTOP_DEVICE } from '../../../src/__tests__/test-utils/index.ts'; +import type { WebProvider } from '../../../src/platforms/web/provider.ts'; +import { createProviderScenarioHarness } from './harness.ts'; + +test('web provider is scoped through the request router and dispatch path', async () => { + const calls: string[] = []; + const webProvider: WebProvider = { + async open(target) { + calls.push(`open:${target}`); + }, + async close(target) { + calls.push(`close:${target ?? ''}`); + }, + async snapshot(options) { + calls.push(`snapshot:${options?.scope ?? ''}`); + return { + nodes: [ + { + index: 0, + type: 'section', + role: 'main', + label: 'main', + rect: { x: 0, y: 0, width: 320, height: 240 }, + depth: 0, + }, + { + index: 1, + type: 'button', + role: 'button', + label: 'Launch', + rect: { x: 10, y: 20, width: 80, height: 32 }, + hittable: true, + depth: 1, + parentIndex: 0, + }, + ], + }; + }, + async screenshot() { + calls.push('screenshot'); + }, + async click(x, y) { + calls.push(`click:${x}:${y}`); + }, + async fill(x, y, text) { + calls.push(`fill:${x}:${y}:${text}`); + }, + async typeText(text) { + calls.push(`type:${text}`); + }, + async scroll(direction) { + calls.push(`scroll:${direction}`); + }, + }; + + const harness = await createProviderScenarioHarness({ + deviceInventoryProvider: async () => [WEB_DESKTOP_DEVICE], + webProvider: ({ device, session }) => { + calls.push(`scope:${session?.name ?? 'none'}:${device.id}`); + return webProvider; + }, + }); + + try { + const open = await harness.callCommand( + 'open', + ['https://example.test'], + { platform: 'web' }, + { meta: { requestId: 'req-web-open' } }, + ); + assert.equal(open.json.error, undefined); + + const snapshot = await harness.callCommand( + 'snapshot', + [], + { platform: 'web', snapshotScope: 'main' }, + { meta: { requestId: 'req-web-snapshot' } }, + ); + + assert.deepEqual(snapshot.json.result.data.nodes, [ + { + index: 0, + type: 'section', + role: 'main', + label: 'main', + rect: { x: 0, y: 0, width: 320, height: 240 }, + depth: 0, + parentIndex: undefined, + ref: 'e1', + }, + { + index: 1, + type: 'button', + role: 'button', + label: 'Launch', + rect: { x: 10, y: 20, width: 80, height: 32 }, + hittable: true, + depth: 1, + parentIndex: 0, + ref: 'e2', + }, + ]); + assert.deepEqual(calls, [ + 'scope:none:agent-browser-chrome', + 'open:https://example.test', + 'scope:default:agent-browser-chrome', + 'snapshot:main', + ]); + } finally { + await harness.close(); + } +});