diff --git a/.gitignore b/.gitignore index 6197c849..117af42f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,20 @@ dist-ssr /*.ts /*.json *.tgz -example/wdio-*.json +examples/wdio/wdio-*.json +examples/wdio/wdio-*.webm +examples/nightwatch/logs/ + +# Adapter-encoded screencasts (written next to the project root by default) +selenium-video-*.webm +nightwatch-video-*.webm +packages/nightwatch-devtools/nightwatch-video-*.webm + +# trace.zip output (mode: 'trace') +trace-*.zip + +# vitest --coverage output +coverage/ # pnpm state, cache, logs, and debug files /packages/**/*.mjs diff --git a/README.md b/README.md index 62a6448f..d88f2dfb 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,46 @@ Real-time capture of browser-side events through the WebDriver BiDi protocol โ€” When BiDi is active in Selenium or Nightwatch, the per-command Chrome performance-log network-capture path is gated off so requests don't appear twice in the dashboard. The attach + sink logic lives in `@wdio/devtools-core`'s `bidi.ts` โ€” same module both adapters consume. +### ๐Ÿ“ฆ Trace mode (trace.zip) + +Headless capture path โ€” no DevTools UI window opens. At session end the adapter writes a `trace-.zip` next to the user's spec / config file, suitable for offline replay, AI-agent diffing, or any consumer that prefers a portable artifact over a live UI. + +| Adapter | How to enable | +|---|---| +| **WebdriverIO** | `services: [['devtools', { mode: 'trace' }]]` | +| **Selenium** | `DevTools.configure({ mode: 'trace' })` (before importing `selenium-webdriver`) | +| **Nightwatch** | `globals: nightwatchDevtools({ mode: 'trace' })` | + +The zip contains: +- `trace.trace` โ€” NDJSON `context-options` + `before`/`after` action events +- `trace.network` โ€” HAR-style network entries derived from the existing capture +- `resources/page@-.jpeg` โ€” screenshot per user-facing action +- `resources/elements-page@-.json` โ€” flat interactable element list extracted by the page-injected scripts in `@wdio/devtools-core/element-scripts` +- `resources/snapshot-page@-.txt` โ€” depth-indented accessibility-tree snapshot (AI-friendly) + +What counts as a user-facing action is filtered through an allow-list in `@wdio/devtools-core/action-mapping.ts` (`url`, `click`, `setValue`, `sendKeys`, `get`, etc.). Internal commands like `findElement`/`waitUntil`/`executeScript` don't produce trace entries. + +Trace mode and live mode are **mutually exclusive** โ€” `screencast` options are ignored in trace mode (live-mode feature). Live and trace serve different audiences (humans debugging vs. agents diffing), and stacking them only costs perf. + +#### Output layout โ€” `traceFormat` + +`{ mode: 'trace', traceFormat: 'zip' | 'ndjson-directory' }`. Default `'zip'` writes a single `trace-.zip`; `'ndjson-directory'` unpacks the same `trace.trace` + `trace.network` + `resources/` files into a `trace-/` folder. Both render in `npx playwright show-trace `. The unpacked form skips the unzip step for scripted / agentic consumers. + +#### ๐Ÿ“ฑ Mobile testing + +Adapters detect mobile sessions via `platformName: 'android' | 'ios'` (case-insensitive) and adjust the per-action snapshot to extract elements from the mobile XML tree instead of the DOM. The trace's `context-options` records `title: 'android' โ€” ` / `'ios' โ€” ` so the viewer labels frames correctly. + +A reference WDIO config is at [examples/wdio/wdio.mobile.conf.ts](examples/wdio/wdio.mobile.conf.ts). Prereqs to run it end-to-end with a local emulator: + +1. **Java JDK** โ€” `brew install --cask temurin` +2. **Android SDK** โ€” `brew install --cask android-commandlinetools` then `yes | sdkmanager --licenses && sdkmanager "platform-tools" "emulator" "system-images;android-34;google_apis_playstore;arm64-v8a"`. The brew cask installs sdkmanager under `/opt/homebrew/share/android-commandlinetools/`, and sdkmanager downloads other SDK pieces alongside it โ€” set `ANDROID_HOME` to that path (not `~/Library/Android/sdk/`). +3. **AVD + emulator** โ€” `avdmanager create avd -n devtools-test -k "system-images;android-34;google_apis_playstore;arm64-v8a" -d "pixel_7"`, then `emulator -avd devtools-test &` + `adb wait-for-device`. +4. **Appium + UiAutomator2 driver** โ€” `sudo npm i -g appium && appium driver install uiautomator2`. +5. **Chromedriver pinning** โ€” Appium's autodownload doesn't reach back far enough for the Chrome version that ships with most Android system images (e.g. Chrome 113 on Android 14). Manually download the matching Chromedriver and start Appium with `--default-capabilities '{"appium:chromedriverExecutableDir": ""}'` plus `--allow-insecure=uiautomator2:chromedriver_autodownload`. +6. **Classic WebDriver protocol** โ€” Appium 3's BiDi shim for UiAutomator2 doesn't implement every BiDi command (e.g. `script.addPreloadScript`). Set `'wdio:enforceWebDriverClassic': true` in the capability block so WDIO doesn't attempt the BiDi handshake. + +These are emulator-specific issues; on a physical phone with USB debugging only steps 1, 4, 6 (and the Chromedriver pin if Chrome on the device is old) apply. + ### ๐Ÿ”๏ธŽ TestLens - **Code Intelligence**: View test definitions directly in your editor - **Run/Debug Actions**: Execute individual tests or suites with inline CodeLens actions diff --git a/examples/nightwatch/nightwatch.conf.cjs b/examples/nightwatch/nightwatch.conf.cjs index a817bf57..7f14af67 100644 --- a/examples/nightwatch/nightwatch.conf.cjs +++ b/examples/nightwatch/nightwatch.conf.cjs @@ -45,6 +45,7 @@ module.exports = { // off to avoid duplicate entries. globals: nightwatchDevtools({ port: 3000, + mode: 'live', screencast: { enabled: true, pollIntervalMs: 200 }, bidi: true }) diff --git a/examples/selenium/jest-test/test/example.js b/examples/selenium/jest-test/test/example.js index 15e9d21a..a59b252b 100644 --- a/examples/selenium/jest-test/test/example.js +++ b/examples/selenium/jest-test/test/example.js @@ -12,7 +12,7 @@ const VALID_USERNAME = 'tomsmith' const VALID_PASSWORD = 'SuperSecretPassword!' DevTools.configure({ - screencast: { enabled: true, quality: 70, maxWidth: 1280, maxHeight: 720 }, + mode: 'trace', headless: true }) diff --git a/examples/wdio/package.json b/examples/wdio/package.json index d3cc2104..a3c3644c 100644 --- a/examples/wdio/package.json +++ b/examples/wdio/package.json @@ -16,6 +16,7 @@ "typescript": "^6.0.3" }, "scripts": { - "wdio": "wdio run ./wdio.conf.ts" + "wdio": "wdio run ./wdio.conf.ts", + "mobile": "wdio run ./wdio.mobile.conf.ts" } } diff --git a/examples/wdio/wdio.conf.ts b/examples/wdio/wdio.conf.ts index 73e88052..00abe28a 100644 --- a/examples/wdio/wdio.conf.ts +++ b/examples/wdio/wdio.conf.ts @@ -128,19 +128,13 @@ export const config: Options.Testrunner = { // your test setup with almost no effort. Unlike plugins, they don't add new // commands. Instead, they hook themselves up into the test process. services: [ - 'devtools' - // [ - // 'devtools', - // { - // screencast: { - // enabled: true, - // captureFormat: 'jpeg', // 'jpeg' or 'png' โ€” frame format sent by Chrome over CDP - // quality: 70, // JPEG quality 0โ€“100 - // maxWidth: 1280, // max frame width in px - // maxHeight: 720 // max frame height in px - // } - // } - // ] + [ + 'devtools', + { + mode: 'live' as const, + screencast: { enabled: true, pollIntervalMs: 200 } + } + ] ], // // Framework you want to run your specs with. diff --git a/examples/wdio/wdio.mobile.conf.ts b/examples/wdio/wdio.mobile.conf.ts new file mode 100644 index 00000000..7f693ab4 --- /dev/null +++ b/examples/wdio/wdio.mobile.conf.ts @@ -0,0 +1,79 @@ +// Mobile-web (Android Chrome via Appium) variant of wdio.conf.ts. +// +// Prerequisites: +// 1. Appium 2.x running locally: `appium --address 127.0.0.1 --port 4723` +// 2. UiAutomator2 driver installed: `appium driver install uiautomator2` +// 3. An Android emulator running with Chrome installed +// (or a real device with USB debugging on and `adb devices` listing it). +// +// Run (from inside examples/wdio): +// pnpm mobile +// +// The DevTools service detects `platformName: Android|iOS` via shared +// capabilities and adjusts the action-snapshot probe (mobile XML element +// extraction) and the trace's context naming accordingly. + +import path from 'node:path' +import type { Options } from '@wdio/types' + +const __dirname = path.resolve(path.dirname(new URL(import.meta.url).pathname)) + +export const config: Options.Testrunner = { + runner: 'local', + + specs: ['./features/**/*.feature'], + exclude: [], + + hostname: '127.0.0.1', + port: 4723, + path: '/', + + maxInstances: 1, + // `wdio:enforceWebDriverClassic` isn't in @wdio/types yet but is honored + // at runtime โ€” needed because Appium's BiDi shim for UiAutomator2 doesn't + // implement every BiDi command (e.g. script.addPreloadScript). + capabilities: [ + { + platformName: 'Android', + 'appium:automationName': 'UiAutomator2', + 'appium:deviceName': 'emulator-5554', + browserName: 'Chrome', + 'appium:chromedriverAutodownload': true, + 'wdio:enforceWebDriverClassic': true + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ] as any, + + logLevel: 'info', + bail: 0, + baseUrl: 'http://localhost', + waitforTimeout: 15000, + connectionRetryTimeout: 120000, + connectionRetryCount: 3, + services: [ + [ + 'devtools', + { + mode: 'trace' as const, + screencast: { enabled: true, pollIntervalMs: 250 } + } + ] + ], + framework: 'cucumber', + reporters: ['spec'], + cucumberOpts: { + require: [ + path.resolve(__dirname, 'features', 'step-definitions', 'steps.ts') + ], + backtrace: false, + requireModule: [], + dryRun: false, + failFast: false, + snippets: true, + source: true, + strict: false, + tagExpression: '', + timeout: 90000, + ignoreUndefinedDefinitions: false + } +} diff --git a/packages/core/package.json b/packages/core/package.json index b637cd88..2e2325ae 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -18,6 +18,22 @@ "./*": { "types": "./src/*.ts", "default": "./src/*.ts" + }, + "./locators": { + "types": "./src/locators/index.ts", + "default": "./src/locators/index.ts" + }, + "./element-snapshot": { + "types": "./src/element-snapshot.ts", + "default": "./src/element-snapshot.ts" + }, + "./element-scripts": { + "types": "./src/element-scripts.ts", + "default": "./src/element-scripts.ts" + }, + "./element-types": { + "types": "./src/element-types.ts", + "default": "./src/element-types.ts" } }, "types": "./src/index.ts", @@ -27,9 +43,13 @@ "license": "MIT", "devDependencies": { "@types/ws": "^8.18.1", + "@types/yazl": "^2.4.6", "@wdio/devtools-script": "workspace:*", "@wdio/devtools-shared": "workspace:^", + "@xmldom/xmldom": "^0.9.8", "stacktrace-parser": "^0.1.11", - "ws": "^8.21.0" + "ws": "^8.21.0", + "xpath": "^0.0.34", + "yazl": "^2.5.1" } } diff --git a/packages/core/src/action-mapping.ts b/packages/core/src/action-mapping.ts new file mode 100644 index 00000000..43a39802 --- /dev/null +++ b/packages/core/src/action-mapping.ts @@ -0,0 +1,65 @@ +// Allow-list mapping from runner-native command names to trace +// vocabulary. Ported from Vince Graics' PR #209 (`@wdio/tracing-service`); the +// existing devtools UI uses its own denylist (`INTERNAL_COMMANDS`) โ€” this map +// is for the trace.zip exporter to filter + rename in one step. + +export interface TraceAction { + class: string + method: string +} + +const ACTION_MAP: Record = { + // WDIO browser-level + url: { class: 'Page', method: 'navigate' }, + navigateTo: { class: 'Page', method: 'navigate' }, + back: { class: 'Page', method: 'goBack' }, + forward: { class: 'Page', method: 'goForward' }, + refresh: { class: 'Page', method: 'reload' }, + newWindow: { class: 'Page', method: 'goto' }, + // Selenium WebDriver navigation (driver.get, driver.navigate().to/back/forward/refresh) + get: { class: 'Page', method: 'navigate' }, + to: { class: 'Page', method: 'navigate' }, + // WDIO element-level + click: { class: 'Element', method: 'click' }, + doubleClick: { class: 'Element', method: 'dblclick' }, + setValue: { class: 'Element', method: 'fill' }, + selectByVisibleText: { class: 'Element', method: 'selectOption' }, + moveTo: { class: 'Element', method: 'hover' }, + scrollIntoView: { class: 'Element', method: 'scrollIntoViewIfNeeded' }, + dragAndDrop: { class: 'Element', method: 'dragTo' }, + // Selenium WebElement actions + sendKeys: { class: 'Element', method: 'fill' }, + clear: { class: 'Element', method: 'clear' }, + submit: { class: 'Element', method: 'submit' }, + // Cross-runner + keys: { class: 'Keyboard', method: 'press' }, + execute: { class: 'Page', method: 'evaluate' }, + executeAsync: { class: 'Page', method: 'evaluate' }, + switchToFrame: { class: 'Frame', method: 'goto' }, + touchAction: { class: 'Element', method: 'tap' } +} + +// Excluded by design: +// clearValue / addValue โ€” WDIO fires these inside setValue (duplicate events). +// executeScript โ€” Selenium's `until` polling fires it ~50ms; also recurses +// because @wdio/elements uses executeScript inside captureActionSnapshot. +// WDIO's user-facing `execute`/`executeAsync` are still captured. + +export function mapCommandToAction(command: string): TraceAction | null { + return ACTION_MAP[command] ?? null +} + +export function formatActionTitle( + action: TraceAction, + args: unknown[], + params?: Record +): string { + const firstArg = args[0] ?? params?.selector + if (firstArg === undefined) { + return `${action.class}.${action.method}()` + } + const label = ( + typeof firstArg === 'object' ? JSON.stringify(firstArg) : String(firstArg) + ).slice(0, 80) + return `${action.class}.${action.method}("${label}")` +} diff --git a/packages/core/src/action-snapshot.ts b/packages/core/src/action-snapshot.ts new file mode 100644 index 00000000..43affebd --- /dev/null +++ b/packages/core/src/action-snapshot.ts @@ -0,0 +1,66 @@ +// Adapter-agnostic per-action snapshot capture. Each adapter wires its own +// `runScript`, `takeScreenshot`, etc. shim so the actual capture pipeline +// (timeouts, fallbacks, snapshot serialization) lives in one place. + +import { accessibilityTreeScript, elementsScript } from './element-scripts.js' +import { serializeWebSnapshot } from './element-snapshot.js' +import type { AccessibilityNode, BrowserElementInfo } from './element-types.js' +import { SNAPSHOT_PROBE_TIMEOUT_MS, withTimeout } from './with-timeout.js' +import type { ActionSnapshot } from '@wdio/devtools-shared' + +export type ScriptRunner = (scriptSrc: string) => Promise + +export interface CaptureActionSnapshotInput { + command: string + runScript: ScriptRunner + takeScreenshot?: () => Promise + getUrl?: () => Promise + getTitle?: () => Promise +} + +async function runWith( + runScript: ScriptRunner, + scriptSrc: string, + fallback: T +): Promise { + return withTimeout( + runScript(scriptSrc).then((r) => r as T), + SNAPSHOT_PROBE_TIMEOUT_MS, + fallback + ).catch(() => fallback) +} + +export async function captureActionSnapshot( + input: CaptureActionSnapshotInput +): Promise { + try { + const timestamp = Date.now() + const [shot, url, title, tree, elements] = await Promise.all([ + input.takeScreenshot?.().catch(() => null) ?? Promise.resolve(null), + input.getUrl?.().catch(() => undefined) ?? Promise.resolve(undefined), + input.getTitle?.().catch(() => undefined) ?? Promise.resolve(undefined), + runWith( + input.runScript, + accessibilityTreeScript(true), + [] + ), + runWith( + input.runScript, + elementsScript(false, true), + [] + ) + ]) + const snapshotText = serializeWebSnapshot(tree, { url, title }) + return { + timestamp, + command: input.command, + url, + title, + screenshot: shot ?? undefined, + elements, + snapshotText + } + } catch { + return null + } +} diff --git a/packages/core/src/element-scripts.ts b/packages/core/src/element-scripts.ts new file mode 100644 index 00000000..9b9b33d9 --- /dev/null +++ b/packages/core/src/element-scripts.ts @@ -0,0 +1,377 @@ +/** + * Browser-injectable script strings for element extraction. + * + * Each function returns a self-contained JavaScript string designed to run + * inside a browser page via `browser.execute(script)`. The scripts have no + * external dependencies and must be ES5-compatible. + * + * WDIO-dependent wrappers that call `browser.execute(script)` live in + * `@wdio/elements` โ€” these are just the script bodies. + */ + +/** + * Accessibility tree walk โ€” returns a flat array of AccessibilityNode. + * + * Walks the DOM from `document.body`, assigning semantic roles (button, link, + * textbox, heading, img, statictext, โ€ฆ) based on tag name, ARIA attributes, + * and visibility. Each node carries a unique CSS selector. + */ +export function accessibilityTreeScript(inViewportOnly: boolean): string { + return `(function () { + var INPUT_TYPE_ROLES = { + text: 'textbox', search: 'searchbox', email: 'textbox', url: 'textbox', + tel: 'textbox', password: 'textbox', number: 'spinbutton', + checkbox: 'checkbox', radio: 'radio', range: 'slider', + submit: 'button', reset: 'button', image: 'button', file: 'button', color: 'button' + } + + var CONTAINER_ROLES = new Set([ + 'navigation', 'banner', 'contentinfo', 'complementary', 'main', + 'form', 'region', 'group', 'list', 'listitem', 'table', 'row', 'rowgroup', 'generic' + ]) + + function getRole(el) { + var explicit = el.getAttribute('role') + if (explicit) { return explicit.split(' ')[0] } + var tag = el.tagName.toLowerCase() + switch (tag) { + case 'button': return 'button' + case 'a': return el.hasAttribute('href') ? 'link' : null + case 'input': { + var type = (el.getAttribute('type') || 'text').toLowerCase() + if (type === 'hidden') { return null } + return INPUT_TYPE_ROLES[type] || 'textbox' + } + case 'select': return 'combobox' + case 'textarea': return 'textbox' + case 'h1': case 'h2': case 'h3': case 'h4': case 'h5': case 'h6': return 'heading' + case 'img': return 'img' + case 'nav': return 'navigation' + case 'main': return 'main' + case 'header': return !el.closest('article,aside,main,nav,section') ? 'banner' : null + case 'footer': return !el.closest('article,aside,main,nav,section') ? 'contentinfo' : null + case 'aside': return 'complementary' + case 'dialog': return 'dialog' + case 'form': return 'form' + case 'section': return el.hasAttribute('aria-label') || el.hasAttribute('aria-labelledby') ? 'region' : null + case 'summary': return 'button' + case 'details': return 'group' + case 'progress': return 'progressbar' + case 'meter': return 'meter' + case 'ul': case 'ol': return 'list' + case 'li': return 'listitem' + case 'table': return 'table' + } + if (el.contentEditable === 'true') { return 'textbox' } + if (el.hasAttribute('tabindex') && parseInt(el.getAttribute('tabindex') || '-1', 10) >= 0) { return 'generic' } + if (getDirectText(el)) { return 'statictext' } + return null + } + + function getAccessibleName(el, role) { + var ariaLabel = el.getAttribute('aria-label') + if (ariaLabel) { return ariaLabel.trim() } + var labelledBy = el.getAttribute('aria-labelledby') + if (labelledBy) { + var texts = labelledBy.split(/\\s+/).map(function(id) { return (document.getElementById(id)?.textContent || '').trim() }).filter(Boolean) + if (texts.length > 0) { return texts.join(' ').slice(0, 200) } + } + var tag = el.tagName.toLowerCase() + if (tag === 'img' || (tag === 'input' && el.getAttribute('type') === 'image')) { + var alt = el.getAttribute('alt') + if (alt !== null) { return alt.trim() } + } + if (['input', 'select', 'textarea'].indexOf(tag) !== -1) { + var id = el.getAttribute('id') + if (id) { + var label = document.querySelector('label[for="' + CSS.escape(id) + '"]') + if (label) { return (label.textContent || '').trim() } + } + var parentLabel = el.closest('label') + if (parentLabel) { + var clone = parentLabel.cloneNode(true) + clone.querySelectorAll('input,select,textarea').forEach(function(n) { n.remove() }) + var lt = (clone.textContent || '').trim() + if (lt) { return lt } + } + } + var ph = el.getAttribute('placeholder') + if (ph) { return ph.trim() } + var title = el.getAttribute('title') + if (title) { return title.trim() } + var childImg = el.querySelector('img') + if (childImg) { + var imgAlt = childImg.getAttribute('alt') + if (imgAlt) { return imgAlt.trim() } + } + if (role && CONTAINER_ROLES.has(role)) { return '' } + return ((el.textContent || '').trim().replace(/\\s+/g, ' ') || '').slice(0, 200) + } + + function getSelector(element) { + var tag = element.tagName.toLowerCase() + var text = (element.textContent || '').trim().replace(/\\s+/g, ' ') + if (text && text.length > 0 && text.length <= 120) { + var sameTagElements = document.querySelectorAll(tag) + var matchCount = 0 + sameTagElements.forEach(function(el) { if (el.textContent.includes(text)) { matchCount++ } }) + if (matchCount === 1) { return tag + '*=' + text } + } + var ariaLabel = element.getAttribute('aria-label') + if (ariaLabel && ariaLabel.length <= 200) { + var sel = '[aria-label="' + CSS.escape(ariaLabel) + '"]' + if (document.querySelectorAll(sel).length === 1) { return sel } + } + var testId = element.getAttribute('data-testid') + if (testId) { + var testSel = '[data-testid="' + CSS.escape(testId) + '"]' + if (document.querySelectorAll(testSel).length === 1) { return testSel } + } + if (element.id) { return '#' + CSS.escape(element.id) } + var nameAttr = element.getAttribute('name') + if (nameAttr) { + var nameSel = tag + '[name="' + CSS.escape(nameAttr) + '"]' + if (document.querySelectorAll(nameSel).length === 1) { return nameSel } + } + if (element.className && typeof element.className === 'string') { + var classes = element.className.trim().split(/\\s+/).filter(Boolean) + for (var i = 0; i < classes.length; i++) { + var clsSel = tag + '.' + CSS.escape(classes[i]) + if (document.querySelectorAll(clsSel).length === 1) { return clsSel } + } + if (classes.length >= 2) { + var twoClsSel = tag + classes.slice(0, 2).map(function(c) { return '.' + CSS.escape(c) }).join('') + if (document.querySelectorAll(twoClsSel).length === 1) { return twoClsSel } + } + } + var current = element + var path = [] + while (current && current !== document.documentElement) { + var seg = current.tagName.toLowerCase() + if (current.id) { path.unshift('#' + CSS.escape(current.id)); break } + var parent = current.parentElement + if (parent) { + var siblings = Array.from(parent.children).filter(function(c) { return c.tagName === current.tagName }) + if (siblings.length > 1) { seg += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')' } + } + path.unshift(seg) + current = current.parentElement + if (path.length >= 4) { break } + } + return path.join(' > ') + } + + function getDirectText(el) { + var text = '' + for (var i = 0; i < el.childNodes.length; i++) { + if (el.childNodes[i].nodeType === 3) { text += el.childNodes[i].textContent } + } + return text.trim().replace(/\\s+/g, ' ') + } + + function isVisible(el) { + if (typeof el.checkVisibility === 'function') { + return el.checkVisibility({ opacityProperty: true, visibilityProperty: true, contentVisibilityAuto: true }) + } + var style = window.getComputedStyle(el) + return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0' && el.offsetWidth > 0 && el.offsetHeight > 0 + } + + function isInViewport(el) { + var rect = el.getBoundingClientRect() + return rect.top >= 0 && rect.left >= 0 && rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) && rect.right <= (window.innerWidth || document.documentElement.clientWidth) + } + + function getLevel(el) { + var m = el.tagName.toLowerCase().match(/^h([1-6])$/) + if (m) { return parseInt(m[1], 10) } + var ariaLevel = el.getAttribute('aria-level') + if (ariaLevel) { return parseInt(ariaLevel, 10) } + return undefined + } + + function getState(el) { + var inputEl = el + var isCheckable = ['input', 'menuitemcheckbox', 'menuitemradio'].indexOf(el.tagName.toLowerCase()) !== -1 || ['checkbox', 'radio', 'switch'].indexOf(el.getAttribute('role') || '') !== -1 + return { + disabled: el.getAttribute('aria-disabled') === 'true' || inputEl.disabled ? 'true' : '', + checked: isCheckable && inputEl.checked ? 'true' : el.getAttribute('aria-checked') || '', + expanded: el.getAttribute('aria-expanded') || '', + selected: el.getAttribute('aria-selected') || '', + pressed: el.getAttribute('aria-pressed') || '', + required: inputEl.required || el.getAttribute('aria-required') === 'true' ? 'true' : '', + readonly: inputEl.readOnly || el.getAttribute('aria-readonly') === 'true' ? 'true' : '' + } + } + + var result = [] + + function walk(el, depth) { + if (depth > 200) { return } + if (!isVisible(el)) { return } + var role = getRole(el) + var inViewport = isInViewport(el) + if (!role) { + for (var i = 0; i < el.children.length; i++) { walk(el.children[i], depth + 1) } + return + } + if (${inViewportOnly} && !inViewport) { + for (var i = 0; i < el.children.length; i++) { walk(el.children[i], depth + 1) } + return + } + var name = getAccessibleName(el, role) + var selector = getSelector(el) + var node = { role: role, name: name, selector: selector, depth: depth, level: getLevel(el) ?? '', isInViewport: inViewport } + var state = getState(el) + for (var k in state) { node[k] = state[k] } + result.push(node) + for (var i = 0; i < el.children.length; i++) { walk(el.children[i], depth + 1) } + } + + for (var i = 0; i < document.body.children.length; i++) { walk(document.body.children[i], 0) } + return result + })()` +} + +/** + * Interactable element query โ€” returns a flat array of BrowserElementInfo. + * + * Uses `querySelectorAll` with a broad interactable-selector list, then + * filters by visibility and (optionally) viewport containment. Each element + * gets a computed accessible name and a unique CSS selector. + */ +export function elementsScript( + includeBounds: boolean, + inViewportOnly: boolean +): string { + return `(function () { + var interactableSelectors = [ + 'a[href]', 'button', 'input:not([type="hidden"])', 'select', 'textarea', + '[role="button"]', '[role="link"]', '[role="checkbox"]', '[role="radio"]', + '[role="tab"]', '[role="menuitem"]', '[role="combobox"]', '[role="option"]', + '[role="switch"]', '[role="slider"]', '[role="textbox"]', '[role="searchbox"]', + '[role="spinbutton"]', '[contenteditable="true"]', '[tabindex]:not([tabindex="-1"])' + ].join(',') + + function isVisible(element) { + if (typeof element.checkVisibility === 'function') { + return element.checkVisibility({ opacityProperty: true, visibilityProperty: true, contentVisibilityAuto: true }) + } + var style = window.getComputedStyle(element) + return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0' && element.offsetWidth > 0 && element.offsetHeight > 0 + } + + function getAccessibleName(el) { + var ariaLabel = el.getAttribute('aria-label') + if (ariaLabel) { return ariaLabel.trim() } + var labelledBy = el.getAttribute('aria-labelledby') + if (labelledBy) { + var texts = labelledBy.split(/\\s+/).map(function(id) { return (document.getElementById(id)?.textContent || '').trim() }).filter(Boolean) + if (texts.length > 0) { return texts.join(' ').slice(0, 200) } + } + var tag = el.tagName.toLowerCase() + if (tag === 'img' || (tag === 'input' && el.getAttribute('type') === 'image')) { + var alt = el.getAttribute('alt') + if (alt !== null) { return alt.trim() } + } + if (['input', 'select', 'textarea'].indexOf(tag) !== -1) { + var id = el.getAttribute('id') + if (id) { + var label = document.querySelector('label[for="' + CSS.escape(id) + '"]') + if (label) { return (label.textContent || '').trim() } + } + var parentLabel = el.closest('label') + if (parentLabel) { + var clone = parentLabel.cloneNode(true) + clone.querySelectorAll('input,select,textarea').forEach(function(n) { n.remove() }) + var lt = (clone.textContent || '').trim() + if (lt) { return lt } + } + } + var ph = el.getAttribute('placeholder') + if (ph) { return ph.trim() } + var title = el.getAttribute('title') + if (title) { return title.trim() } + return ((el.textContent || '').trim().replace(/\\s+/g, ' ') || '').slice(0, 200) + } + + function getSelector(element) { + var tag = element.tagName.toLowerCase() + var text = (element.textContent || '').trim().replace(/\\s+/g, ' ') + if (text && text.length > 0 && text.length <= 120) { + var sameTagElements = document.querySelectorAll(tag) + var matchCount = 0 + sameTagElements.forEach(function(el) { if (el.textContent.includes(text)) { matchCount++ } }) + if (matchCount === 1) { return tag + '*=' + text } + } + var ariaLabel = element.getAttribute('aria-label') + if (ariaLabel && ariaLabel.length <= 200) { + var sel = '[aria-label="' + CSS.escape(ariaLabel) + '"]' + if (document.querySelectorAll(sel).length === 1) { return sel } + } + var testId = element.getAttribute('data-testid') + if (testId) { + var testSel = '[data-testid="' + CSS.escape(testId) + '"]' + if (document.querySelectorAll(testSel).length === 1) { return testSel } + } + if (element.id) { return '#' + CSS.escape(element.id) } + var nameAttr = element.getAttribute('name') + if (nameAttr) { + var nameSel = tag + '[name="' + CSS.escape(nameAttr) + '"]' + if (document.querySelectorAll(nameSel).length === 1) { return nameSel } + } + if (element.className && typeof element.className === 'string') { + var classes = element.className.trim().split(/\\s+/).filter(Boolean) + for (var i = 0; i < classes.length; i++) { + var clsSel = tag + '.' + CSS.escape(classes[i]) + if (document.querySelectorAll(clsSel).length === 1) { return clsSel } + } + if (classes.length >= 2) { + var twoClsSel = tag + classes.slice(0, 2).map(function(c) { return '.' + CSS.escape(c) }).join('') + if (document.querySelectorAll(twoClsSel).length === 1) { return twoClsSel } + } + } + var current = element + var path = [] + while (current && current !== document.documentElement) { + var seg = current.tagName.toLowerCase() + if (current.id) { path.unshift('#' + CSS.escape(current.id)); break } + var parent = current.parentElement + if (parent) { + var siblings = Array.from(parent.children).filter(function(c) { return c.tagName === current.tagName }) + if (siblings.length > 1) { seg += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')' } + } + path.unshift(seg) + current = current.parentElement + if (path.length >= 4) { break } + } + return path.join(' > ') + } + + var elements = [] + var seen = new Set() + + document.querySelectorAll(interactableSelectors).forEach(function(el) { + if (seen.has(el)) { return } + seen.add(el) + var htmlEl = el + if (!isVisible(htmlEl)) { return } + var inputEl = htmlEl + var rect = htmlEl.getBoundingClientRect() + var isInVp = rect.top >= 0 && rect.left >= 0 && rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) && rect.right <= (window.innerWidth || document.documentElement.clientWidth) + if (${inViewportOnly} && !isInVp) { return } + var entry = { + tagName: htmlEl.tagName.toLowerCase(), + name: getAccessibleName(htmlEl), + type: htmlEl.getAttribute('type') || '', + value: inputEl.value || '', + href: htmlEl.getAttribute('href') || '', + selector: getSelector(htmlEl), + isInViewport: isInVp + } + ${includeBounds ? 'entry.boundingBox = { x: rect.x + window.scrollX, y: rect.y + window.scrollY, width: rect.width, height: rect.height }' : ''} + elements.push(entry) + }) + return elements + })()` +} diff --git a/packages/core/src/element-snapshot.ts b/packages/core/src/element-snapshot.ts new file mode 100644 index 00000000..c4e1b2bb --- /dev/null +++ b/packages/core/src/element-snapshot.ts @@ -0,0 +1,758 @@ +/** + * AI-readable snapshot serializers + * + * Converts accessibility trees and mobile element trees into depth-indented + * text files that LLMs can consume without any parsing. + */ + +import type { AccessibilityNode } from './element-types.js' +import type { JSONElement } from './locators/types.js' +import { parseAndroidBounds, parseIOSBounds } from './locators/xml-parsing.js' +import { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS +} from './locators/constants.js' +import { getSuggestedLocators } from './locators/locator-generation.js' + +/** + * Roles that can be interacted with โ€” rendered with `โ†’ selector`. + * Structural roles (heading, img, form, nav, โ€ฆ) are intentionally excluded. + */ +const INTERACTIVE_ROLES = new Set([ + 'button', + 'link', + 'textbox', + 'checkbox', + 'radio', + 'combobox', + 'slider', + 'searchbox', + 'spinbutton', + 'switch', + 'tab', + 'menuitem', + 'option' +]) + +/** + * Walk backwards from `index` to find the nearest ancestor or preceding + * structural sibling with a non-empty name. Same-depth nodes are only + * used when they are structural (img, heading, statictext, โ€ฆ) โ€” never + * another interactive element. + */ +function inferPurpose( + nodes: AccessibilityNode[], + index: number +): string | undefined { + const myDepth = nodes[index].depth + for (let i = index - 1; i >= 0; i--) { + if (nodes[i].depth <= myDepth && nodes[i].name) { + // Same-depth sibling: only structural elements count + if (nodes[i].depth === myDepth && INTERACTIVE_ROLES.has(nodes[i].role)) { + continue + } + return nodes[i].name + } + } + return undefined +} + +export interface WebSnapshotOptions { + /** Only include nodes whose bounding rect intersects the viewport (default true). */ + inViewportOnly?: boolean +} + +/** + * Serialize a web accessibility tree into a depth-indented text snapshot. + * + * @param nodes Flat ordered node list from getBrowserAccessibilityTree() + * @param context Optional page context for the header line + * @param options {@link WebSnapshotOptions} + */ +export function serializeWebSnapshot( + nodes: AccessibilityNode[], + context?: { url?: string; title?: string }, + options: WebSnapshotOptions = {} +): string { + const { inViewportOnly = true } = options + + let header = '[Page' + if (context?.title) { + header += `: ${context.title}` + } + if (context?.url) { + header += ` โ€” ${context.url}` + } + header += ']' + + const lines: string[] = [header] + + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + + // When viewport filtering is on, skip nodes that are known to be off-screen. + // Nodes from a tree captured with inViewportOnly=false will have + // isInViewport populated; nodes from a pre-filtered tree all have + // isInViewport=true (or undefined for pre-existing data). + if (inViewportOnly && node.isInViewport === false) { + continue + } + + const indent = ' '.repeat(node.depth + 1) // +1 indents everything under the header + const isInteractive = INTERACTIVE_ROLES.has(node.role) + + // Skip statictext that merely echoes the parent link/button name. + // Example: link "Highlights" โ†’ a*=Highlights doesn't need + // statictext "Highlights" as a child because it adds no information. + if (node.role === 'statictext' && node.name) { + let echoedByParent = false + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + const parentRole = nodes[j].role + const parentName = nodes[j].name + if ( + INTERACTIVE_ROLES.has(parentRole) && + parentName && + parentName.includes(node.name) + ) { + echoedByParent = true + } + break // only check the immediate structural parent + } + } + if (echoedByParent) { + continue + } + } + + // Heading gets level suffix: heading[2] + const roleLabel = + node.role === 'heading' && node.level + ? `heading[${node.level}]` + : node.role + + if (isInteractive) { + // No selector โ†’ agent can't act on this node; skip entirely + if (!node.selector) { + continue + } + const purpose = inferPurpose(nodes, i) + if (node.name) { + // Show parent context when available โ€” disambiguates + // duplicate selectors like six "Add to Wishlist" buttons. + lines.push( + purpose + ? `${indent}${roleLabel} "${node.name}" โˆˆ "${purpose}" โ†’ ${node.selector}` + : `${indent}${roleLabel} "${node.name}" โ†’ ${node.selector}` + ) + } else if (purpose) { + lines.push(`${indent}${roleLabel} โˆˆ "${purpose}" โ†’ ${node.selector}`) + } else { + lines.push(`${indent}${roleLabel} โ†’ ${node.selector}`) + } + } else { + // Container / structural: show role + name when present, no selector + lines.push( + node.name + ? `${indent}${roleLabel} "${node.name}"` + : `${indent}${roleLabel}` + ) + } + } + + return lines.join('\n') +} + +// --------------------------------------------------------------------------- +// Mobile snapshot helpers +// --------------------------------------------------------------------------- + +/** Shorten fully-qualified Android/iOS class names to the last segment. */ +function simplifyTag(tagName: string): string { + const dot = tagName.lastIndexOf('.') + if (dot !== -1) { + return tagName.slice(dot + 1) + } + return tagName.replace(/^XCUIElementType/, '') +} + +// --------------------------------------------------------------------------- +// Mobile role classification โ€” maps raw Android/iOS class names to semantic +// roles so the snapshot reads like the web version (button, textbox, img, โ€ฆ). +// --------------------------------------------------------------------------- + +const ANDROID_ROLE_MAP: Record = { + 'android.widget.Button': 'button', + 'android.widget.ImageButton': 'button', + 'android.widget.ToggleButton': 'button', + 'android.widget.FloatingActionButton': 'button', + 'com.google.android.material.button.MaterialButton': 'button', + 'com.google.android.material.floatingactionbutton.FloatingActionButton': + 'button', + 'android.widget.EditText': 'textbox', + 'android.widget.AutoCompleteTextView': 'textbox', + 'android.widget.MultiAutoCompleteTextView': 'textbox', + 'android.widget.SearchView': 'searchbox', + 'android.widget.ImageView': 'img', + 'android.widget.QuickContactBadge': 'img', + 'android.widget.CheckBox': 'checkbox', + 'android.widget.RadioButton': 'radio', + 'android.widget.Switch': 'switch', + 'android.widget.Spinner': 'combobox', + 'android.widget.SeekBar': 'slider', + 'android.widget.RatingBar': 'slider', + 'android.widget.ProgressBar': 'progressbar', + 'android.widget.TextView': 'statictext', + 'android.widget.CheckedTextView': 'statictext', + 'android.widget.RecyclerView': 'list', + 'android.widget.ListView': 'list', + 'android.widget.GridView': 'list', + 'android.webkit.WebView': 'webview' +} + +const IOS_ROLE_MAP: Record = { + XCUIElementTypeButton: 'button', + XCUIElementTypeLink: 'link', + XCUIElementTypeTextField: 'textbox', + XCUIElementTypeSecureTextField: 'textbox', + XCUIElementTypeTextView: 'textbox', + XCUIElementTypeSearchField: 'searchbox', + XCUIElementTypeImage: 'img', + XCUIElementTypeIcon: 'img', + XCUIElementTypeSwitch: 'switch', + XCUIElementTypeSlider: 'slider', + XCUIElementTypeStepper: 'slider', + XCUIElementTypeCheckBox: 'checkbox', + XCUIElementTypeRadioButton: 'radio', + XCUIElementTypePicker: 'combobox', + XCUIElementTypePickerWheel: 'combobox', + XCUIElementTypeDatePicker: 'combobox', + XCUIElementTypeSegmentedControl: 'combobox', + XCUIElementTypeStaticText: 'statictext', + XCUIElementTypeCell: 'listitem', + XCUIElementTypeTable: 'list', + XCUIElementTypeCollectionView: 'list' +} + +function classifyMobileRole( + tagName: string, + platform: 'android' | 'ios' +): string { + if (platform === 'android') { + return ANDROID_ROLE_MAP[tagName] || simplifyTag(tagName) + } + return IOS_ROLE_MAP[tagName] || simplifyTag(tagName) +} + +// --------------------------------------------------------------------------- +// Locator generation +// --------------------------------------------------------------------------- + +function getBestAndroidLocator( + attrs: JSONElement['attributes'] +): string | undefined { + // Pre-computed by the full locator pipeline (generateAllElementLocators). + // Takes priority over the simplified fallback logic below. + if (attrs._selector) { + return attrs._selector + } + // ~ prefix = accessibility-id shorthand in WebdriverIO ($('~foo')) + if (attrs['content-desc']) { + return `~${attrs['content-desc']}` + } + if (attrs['resource-id']) { + return `id:${attrs['resource-id']}` + } + if (attrs.text) { + return `~${attrs.text}` + } + // Fallback: class-based locator (only useful with :nth-of-type or index) + if (attrs.class) { + return `class:${simplifyTag(attrs.class)}` + } + return undefined +} + +function getBestIOSLocator( + attrs: JSONElement['attributes'] +): string | undefined { + // Pre-computed by the full locator pipeline. + if (attrs._selector) { + return attrs._selector + } + // ~ prefix = accessibility-id shorthand (maps to `name` on iOS) + if (attrs.name) { + return `~${attrs.name}` + } + if (attrs.label) { + return `~${attrs.label}` + } + if (attrs.value) { + return `~${attrs.value}` + } + // Fallback: class-based locator + if (attrs.type) { + return `class:${simplifyTag(attrs.type)}` + } + return undefined +} + +// --------------------------------------------------------------------------- +// Identity +// --------------------------------------------------------------------------- + +function getMobileNodeIdentity( + attrs: JSONElement['attributes'], + platform: 'android' | 'ios' +): string { + if (platform === 'android') { + const contentDesc = attrs['content-desc'] + if (contentDesc) { + return contentDesc + } + if (attrs.text) { + return attrs.text + } + // Fall back to the last segment of the resource-id (e.g. "search_action_bar") + const rid = attrs['resource-id'] + if (rid) { + const slash = rid.lastIndexOf('/') + return slash !== -1 ? rid.slice(slash + 1) : rid + } + return '' + } + return attrs.name || attrs.label || attrs.value || attrs.text || '' +} + +// --------------------------------------------------------------------------- +// Interactivity +// --------------------------------------------------------------------------- + +const ANDROID_INTERACTABLE_SET = new Set(ANDROID_INTERACTABLE_TAGS) +const IOS_INTERACTABLE_SET = new Set(IOS_INTERACTABLE_TAGS) + +/** An element is *explicitly* interactive when it carries a click/focus/check + * attribute โ€” as opposed to being interactive only because its tag is in the + * interactable-tag list. Explicit parents should carry the โ†’ selector, not + * their tag-interactive children. */ +function isExplicitlyInteractive( + attrs: JSONElement['attributes'], + platform: 'android' | 'ios' +): boolean { + if (platform === 'android') { + return ( + attrs.clickable === 'true' || + attrs.focusable === 'true' || + attrs.checkable === 'true' || + attrs['long-clickable'] === 'true' + ) + } + return attrs.accessible === 'true' +} + +function isMobileInteractive( + element: JSONElement, + platform: 'android' | 'ios' +): boolean { + const attrs = element.attributes + if (platform === 'android') { + if (ANDROID_INTERACTABLE_SET.has(element.tagName)) { + return true + } + return ( + attrs.clickable === 'true' || + attrs['long-clickable'] === 'true' || + attrs.focusable === 'true' || + attrs.checkable === 'true' + ) + } + if (IOS_INTERACTABLE_SET.has(element.tagName)) { + return true + } + return attrs.accessible === 'true' +} + +// --------------------------------------------------------------------------- +// Viewport +// --------------------------------------------------------------------------- + +interface WalkMobileOptions { + inViewportOnly: boolean + viewport: { width: number; height: number } + /** Raw page-source XML. When provided, the full locator pipeline is used. */ + sourceXML?: string + /** 'uiautomator2' or 'xcuitest'. Required when sourceXML is set. */ + automationName?: string +} + +function isMobileInViewport( + element: JSONElement, + platform: 'android' | 'ios', + viewport: { width: number; height: number } +): boolean { + const bounds = + platform === 'android' + ? parseAndroidBounds(element.attributes.bounds || '') + : parseIOSBounds(element.attributes) + + if (bounds.width === 0 && bounds.height === 0) { + return true + } + + return ( + bounds.x >= 0 && + bounds.y >= 0 && + bounds.width > 0 && + bounds.height > 0 && + bounds.x + bounds.width <= viewport.width && + bounds.y + bounds.height <= viewport.height + ) +} + +// --------------------------------------------------------------------------- +// Flat-node representation (mirrors AccessibilityNode so both pipelines share +// inferPurpose, dedup, and rendering logic). +// --------------------------------------------------------------------------- + +interface MobileFlatNode { + role: string + name: string + selector: string + depth: number + isInteractive: boolean + /** True when the element has clickable/focusable/checkable โ€” the intended tap target. */ + isExplicitInteractive: boolean + isInViewport: boolean +} + +/** + * First pass: walk the JSONElement tree, apply viewport filtering and + * collect every node into a flat array with semantic roles and selectors. + */ +function collectMobileNodes( + element: JSONElement, + platform: 'android' | 'ios', + depth: number, + nodes: MobileFlatNode[], + walkOpts: WalkMobileOptions +): void { + const attrs = element.attributes + const role = classifyMobileRole(element.tagName, platform) + const name = getMobileNodeIdentity(attrs, platform) + const explicit = isExplicitlyInteractive(attrs, platform) + const interactive = isMobileInteractive(element, platform) + const inViewport = isMobileInViewport(element, platform, walkOpts.viewport) + + // Viewport filtering + if (walkOpts.inViewportOnly) { + if (interactive && !inViewport) { + // Skip this node but still recurse (scroll children may be in view). + for (const child of element.children || []) { + collectMobileNodes(child, platform, depth + 1, nodes, walkOpts) + } + return + } + if (!interactive && !inViewport) { + // Collapse off-screen container to a placeholder. + nodes.push({ + role: 'generic', + name: name ? `${role} "${name}"` : role, + selector: '', + depth, + isInteractive: false, + isExplicitInteractive: false, + isInViewport: false + }) + return + } + } + + // Generate a selector for every interactive element. + // Use the full locator pipeline when source XML is available; + // otherwise fall back to the simplified attribute-based heuristics. + let locator = '' + if (interactive) { + if (walkOpts.sourceXML && walkOpts.automationName) { + // Full pipeline: accessible-id, id, text, uiautomator, xpath, class-name + const suggested = getSuggestedLocators( + element, + walkOpts.sourceXML, + walkOpts.automationName, + { + sourceXML: walkOpts.sourceXML, + parsedDOM: null, + isAndroid: platform === 'android' + } + ) + if (suggested.length > 0) { + locator = suggested[0][1] // first = best priority + } + } + if (!locator) { + // Simplified fallback + locator = + (platform === 'android' + ? getBestAndroidLocator(attrs) + : getBestIOSLocator(attrs)) ?? '' + } + } + + nodes.push({ + role, + name, + selector: locator, + depth, + isInteractive: interactive, + isExplicitInteractive: explicit, + isInViewport: inViewport + }) + + for (const child of element.children || []) { + collectMobileNodes(child, platform, depth + 1, nodes, walkOpts) + } +} + +// --------------------------------------------------------------------------- +// Context inference โ€” shared with the web pipeline. +// Same-depth structural siblings (img, statictext, heading, โ€ฆ) provide +// context for following interactive nodes. +// --------------------------------------------------------------------------- + +const MOBILE_STRUCTURAL_ROLES = new Set([ + 'img', + 'heading', + 'list', + 'listitem', + 'webview', + 'progressbar', + 'slider', + 'switch', + 'generic' +]) + +function mobileInferPurpose( + nodes: MobileFlatNode[], + index: number +): string | undefined { + const myDepth = nodes[index].depth + for (let i = index - 1; i >= 0; i--) { + if (nodes[i].depth <= myDepth && nodes[i].name) { + if ( + nodes[i].depth === myDepth && + !MOBILE_STRUCTURAL_ROLES.has(nodes[i].role) + ) { + continue + } + return nodes[i].name + } + } + return undefined +} + +// --------------------------------------------------------------------------- +// When a tag-only-interactive child (e.g. a statictext TextView) sits +// directly under an explicitly-interactive parent (e.g. a clickable +// LinearLayout row), the *parent* should carry the โ†’ selector โ€” the +// child is just a label. Suppress the child's interactivity so the +// parent renders as the actionable element. +// --------------------------------------------------------------------------- + +function suppressTagOnlyChildren(nodes: MobileFlatNode[]): void { + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + if (!node.isInteractive || node.isExplicitInteractive) { + continue + } + // Walk up through ALL ancestors looking for an explicitly-interactive + // parent. The immediate depth-1 parent may just be a layout wrapper; + // the real clickable row could be 2-3 levels up. + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + if (nodes[j].isExplicitInteractive) { + node.isInteractive = false + break // found โ€” suppress and stop + } + // keep looking upward through the ancestor chain + } + } + } +} + +// --------------------------------------------------------------------------- +// Render pass: flat nodes into lines with โˆˆ context, dedup, noise filter, +// and class-instance indexing. +// --------------------------------------------------------------------------- + +/** Layout roles that carry no semantic meaning by themselves. */ +const NOISY_ROLES = new Set([ + 'FrameLayout', + 'LinearLayout', + 'ViewGroup', + 'RelativeLayout', + 'View', + 'CardView', + 'ConstraintLayout', + 'ScrollView' +]) + +/** + * Pre-count selector occurrences so we can attach .instance(N) suffixes + * to duplicate selectors. + */ +function countSelectors(nodes: MobileFlatNode[]): Map { + const counts = new Map() + for (const node of nodes) { + if (node.selector) { + counts.set(node.selector, (counts.get(node.selector) ?? 0) + 1) + } + } + return counts +} + +function renderMobileNodes(nodes: MobileFlatNode[]): string[] { + const lines: string[] = [] + const selectorCounts = countSelectors(nodes) + const selectorIndex = new Map() + + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + const indent = ' '.repeat(node.depth + 1) + + // Collapse anonymous layout containers at depth โ‰ฅ 2. + // Keep depth 0-1 structural chrome and any named container. + if ( + NOISY_ROLES.has(node.role) && + !node.name && + node.depth > 1 && + !node.isInteractive + ) { + continue + } + + // Off-screen containers rendered as collapsed placeholders + if (node.isInViewport === false && !node.isInteractive) { + lines.push(`${indent}โ‹ฏ ${node.name} (off-screen)`) + continue + } + + // Dedup: skip statictext whose text is echoed by the parent interactive element + if (node.role === 'statictext' && node.name) { + let echoedByParent = false + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + if ( + nodes[j].isInteractive && + nodes[j].name && + nodes[j].name.includes(node.name) + ) { + echoedByParent = true + } + break + } + } + if (echoedByParent) { + continue + } + } + + if (node.isInteractive && node.selector) { + // Append .instance(N) when the same selector repeats + let selector = node.selector + const total = selectorCounts.get(selector) ?? 1 + if (total > 1) { + const idx = selectorIndex.get(selector) ?? 0 + selectorIndex.set(selector, idx + 1) + selector = `${selector}.instance(${idx})` + } + + const purpose = mobileInferPurpose(nodes, i) + if (node.name) { + lines.push( + purpose + ? `${indent}${node.role} "${node.name}" โˆˆ "${purpose}" โ†’ ${selector}` + : `${indent}${node.role} "${node.name}" โ†’ ${selector}` + ) + } else if (purpose) { + lines.push(`${indent}${node.role} โˆˆ "${purpose}" โ†’ ${selector}`) + } else { + lines.push(`${indent}${node.role} โ†’ ${selector}`) + } + } else { + // Container / structural / non-locatable + lines.push( + node.name + ? `${indent}${node.role} "${node.name}"` + : `${indent}${node.role}` + ) + } + } + + return lines +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +export interface MobileSnapshotOptions { + /** Only include elements whose bounds intersect the viewport (default true). */ + inViewportOnly?: boolean + /** + * Raw XML page source string. When provided the full locator pipeline + * (getSuggestedLocators) runs on every interactive node, producing the same + * selectors that getElements() returns. Omit to use simplified heuristics. + */ + sourceXML?: string +} + +/** + * Serialize a mobile element tree into a depth-indented text snapshot. + * + * @param root Root JSONElement from the page source XML parse + * @param context Platform, optional device name, viewport, and source XML. + * Include `sourceXML` to use the full locator pipeline. + * @param options {@link MobileSnapshotOptions} + */ +export function serializeMobileSnapshot( + root: JSONElement, + context: { + platform: 'android' | 'ios' + deviceName?: string + viewport?: { width: number; height: number } + /** Raw page-source XML. When set, selectors match getElements() output. */ + sourceXML?: string + }, + options: MobileSnapshotOptions = {} +): string { + const { platform, deviceName, viewport, sourceXML } = context + const { inViewportOnly = true } = options + + // Auto-detect source XML stashed by getMobileVisibleElementsWithTree + const effectiveXML = sourceXML || root.attributes._sourceXML + + const effectiveViewport = viewport ?? { width: 9999, height: 9999 } + const automationName = platform === 'android' ? 'uiautomator2' : 'xcuitest' + + let header = `[${platform}` + if (deviceName) { + header += ` โ€” ${deviceName}` + } + if (viewport) { + header += ` (${viewport.width}ร—${viewport.height})` + } + header += ']' + + const nodes: MobileFlatNode[] = [] + collectMobileNodes(root, platform, 0, nodes, { + inViewportOnly, + viewport: effectiveViewport, + sourceXML: effectiveXML, + automationName: effectiveXML ? automationName : undefined + }) + + // Let explicitly-interactive parents carry the โ†’ selector + suppressTagOnlyChildren(nodes) + + const lines = renderMobileNodes(nodes) + return [header, ...lines].join('\n') +} diff --git a/packages/core/src/element-types.ts b/packages/core/src/element-types.ts new file mode 100644 index 00000000..f4b3fcc6 --- /dev/null +++ b/packages/core/src/element-types.ts @@ -0,0 +1,45 @@ +/** + * Framework-agnostic element types used by element extraction scripts, + * snapshot serializers, and locator generation. + * + * These types describe the data structures returned by browser-injectable + * scripts and mobile page-source parsing. They have no WebdriverIO dependency. + */ + +export interface AccessibilityNode { + role: string + name: string + selector: string + depth: number + level: number | string + disabled: string + checked: string + expanded: string + selected: string + pressed: string + required: string + readonly: string + /** Whether the element's bounding rect intersects the viewport. */ + isInViewport?: boolean +} + +export interface BrowserElementInfo { + tagName: string + name: string // computed accessible name (ARIA spec) + type: string + value: string + href: string + selector: string + isInViewport: boolean + boundingBox?: { x: number; y: number; width: number; height: number } +} + +export interface GetBrowserElementsOptions { + includeBounds?: boolean + /** Only return elements whose bounding rect intersects the viewport (default true). */ + inViewportOnly?: boolean +} + +// Re-export mobile types from locators for convenience. +// Downstream consumers can also import directly from @wdio/devtools-core/locators. +export type { JSONElement } from './locators/types.js' diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e34b6ac4..5f165d86 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,7 +1,16 @@ // Framework-agnostic capture/reporter logic shared by @wdio/devtools-* // adapters. See ARCHITECTURE.md ยง2 and CLAUDE.md ยง2.2. +export * from './action-mapping.js' +export * from './action-snapshot.js' +export * from './with-timeout.js' export * from './assert-patcher.js' +export * from './element-snapshot.js' +export * from './element-scripts.js' +export * from './element-types.js' +export * from './trace-exporter.js' +export * from './trace-har.js' +export * from './trace-zip-writer.js' export * from './bidi.js' export * from './console.js' export * from './uid.js' diff --git a/packages/core/src/locators/constants.ts b/packages/core/src/locators/constants.ts new file mode 100644 index 00000000..540784b3 --- /dev/null +++ b/packages/core/src/locators/constants.ts @@ -0,0 +1,169 @@ +/** + * Platform-specific element tag constants for mobile automation + */ + +export const ANDROID_INTERACTABLE_TAGS = [ + // Input elements + 'android.widget.EditText', + 'android.widget.AutoCompleteTextView', + 'android.widget.MultiAutoCompleteTextView', + 'android.widget.SearchView', + + // Button-like elements + 'android.widget.Button', + 'android.widget.ImageButton', + 'android.widget.ToggleButton', + 'android.widget.CompoundButton', + 'android.widget.RadioButton', + 'android.widget.CheckBox', + 'android.widget.Switch', + 'android.widget.FloatingActionButton', + 'com.google.android.material.button.MaterialButton', + 'com.google.android.material.floatingactionbutton.FloatingActionButton', + + // Text elements (often tappable) + 'android.widget.TextView', + 'android.widget.CheckedTextView', + + // Image elements (often tappable) + 'android.widget.ImageView', + 'android.widget.QuickContactBadge', + + // Selection elements + 'android.widget.Spinner', + 'android.widget.SeekBar', + 'android.widget.RatingBar', + 'android.widget.ProgressBar', + 'android.widget.DatePicker', + 'android.widget.TimePicker', + 'android.widget.NumberPicker', + + // List/grid items + 'android.widget.AdapterView' +] + +export const ANDROID_LAYOUT_CONTAINERS = [ + // Core ViewGroup classes + 'android.view.ViewGroup', + 'android.view.View', + 'android.widget.FrameLayout', + 'android.widget.LinearLayout', + 'android.widget.RelativeLayout', + 'android.widget.GridLayout', + 'android.widget.TableLayout', + 'android.widget.TableRow', + 'android.widget.AbsoluteLayout', + + // AndroidX layout classes + 'androidx.constraintlayout.widget.ConstraintLayout', + 'androidx.coordinatorlayout.widget.CoordinatorLayout', + 'androidx.appcompat.widget.LinearLayoutCompat', + 'androidx.cardview.widget.CardView', + 'androidx.appcompat.widget.ContentFrameLayout', + 'androidx.appcompat.widget.FitWindowsFrameLayout', + + // Scrolling containers + 'android.widget.ScrollView', + 'android.widget.HorizontalScrollView', + 'android.widget.NestedScrollView', + 'androidx.core.widget.NestedScrollView', + 'androidx.recyclerview.widget.RecyclerView', + 'android.widget.ListView', + 'android.widget.GridView', + 'android.widget.AbsListView', + + // App chrome / system elements + 'android.widget.ActionBarContainer', + 'android.widget.ActionBarOverlayLayout', + 'android.view.ViewStub', + 'androidx.appcompat.widget.ActionBarContainer', + 'androidx.appcompat.widget.ActionBarContextView', + 'androidx.appcompat.widget.ActionBarOverlayLayout', + + // Decor views + 'com.android.internal.policy.DecorView', + 'android.widget.DecorView' +] + +export const IOS_INTERACTABLE_TAGS = [ + // Input elements + 'XCUIElementTypeTextField', + 'XCUIElementTypeSecureTextField', + 'XCUIElementTypeTextView', + 'XCUIElementTypeSearchField', + + // Button-like elements + 'XCUIElementTypeButton', + 'XCUIElementTypeLink', + + // Text elements (often tappable) + 'XCUIElementTypeStaticText', + + // Image elements + 'XCUIElementTypeImage', + 'XCUIElementTypeIcon', + + // Selection elements + 'XCUIElementTypeSwitch', + 'XCUIElementTypeSlider', + 'XCUIElementTypeStepper', + 'XCUIElementTypeSegmentedControl', + 'XCUIElementTypePicker', + 'XCUIElementTypePickerWheel', + 'XCUIElementTypeDatePicker', + 'XCUIElementTypePageIndicator', + + // Table/list items + 'XCUIElementTypeCell', + 'XCUIElementTypeMenuItem', + 'XCUIElementTypeMenuBarItem', + + // Toggle elements + 'XCUIElementTypeCheckBox', + 'XCUIElementTypeRadioButton', + 'XCUIElementTypeToggle', + + // Other interactive + 'XCUIElementTypeKey', + 'XCUIElementTypeKeyboard', + 'XCUIElementTypeAlert', + 'XCUIElementTypeSheet' +] + +export const IOS_LAYOUT_CONTAINERS = [ + // Generic containers + 'XCUIElementTypeOther', + 'XCUIElementTypeGroup', + 'XCUIElementTypeLayoutItem', + + // Scroll containers + 'XCUIElementTypeScrollView', + 'XCUIElementTypeTable', + 'XCUIElementTypeCollectionView', + 'XCUIElementTypeScrollBar', + + // Navigation chrome + 'XCUIElementTypeNavigationBar', + 'XCUIElementTypeTabBar', + 'XCUIElementTypeToolbar', + 'XCUIElementTypeStatusBar', + 'XCUIElementTypeMenuBar', + + // Windows and views + 'XCUIElementTypeWindow', + 'XCUIElementTypeSheet', + 'XCUIElementTypeDrawer', + 'XCUIElementTypeDialog', + 'XCUIElementTypePopover', + 'XCUIElementTypePopUpButton', + + // Outline elements + 'XCUIElementTypeOutline', + 'XCUIElementTypeOutlineRow', + 'XCUIElementTypeBrowser', + 'XCUIElementTypeSplitGroup', + 'XCUIElementTypeSplitter', + + // Application root + 'XCUIElementTypeApplication' +] diff --git a/packages/core/src/locators/element-filter.ts b/packages/core/src/locators/element-filter.ts new file mode 100644 index 00000000..d249f3a2 --- /dev/null +++ b/packages/core/src/locators/element-filter.ts @@ -0,0 +1,234 @@ +/** + * Element filtering logic for mobile automation + */ + +import type { JSONElement, FilterOptions } from './types.js' +import { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS, + ANDROID_LAYOUT_CONTAINERS, + IOS_LAYOUT_CONTAINERS +} from './constants.js' + +/** + * Check if element tag matches any in the list (handles partial matches) + */ +function matchesTagList(tagName: string, tagList: string[]): boolean { + if (tagList.includes(tagName)) { + return true + } + + for (const tag of tagList) { + if (tagName.endsWith(tag) || tagName.includes(tag)) { + return true + } + } + + return false +} + +/** + * Check if element matches tag name filters + */ +function matchesTagFilters( + element: JSONElement, + includeTagNames: string[], + excludeTagNames: string[] +): boolean { + if ( + includeTagNames.length > 0 && + !matchesTagList(element.tagName, includeTagNames) + ) { + return false + } + + if (matchesTagList(element.tagName, excludeTagNames)) { + return false + } + + return true +} + +/** + * Check if element matches attribute-based filters + */ +function matchesAttributeFilters( + element: JSONElement, + requireAttributes: string[], + minAttributeCount: number +): boolean { + if (requireAttributes.length > 0) { + const hasRequiredAttr = requireAttributes.some( + (attr) => element.attributes?.[attr] + ) + if (!hasRequiredAttr) { + return false + } + } + + if (element.attributes && minAttributeCount > 0) { + const attrCount = Object.values(element.attributes).filter( + (v) => v !== undefined && v !== null && v !== '' + ).length + if (attrCount < minAttributeCount) { + return false + } + } + + return true +} + +/** + * Check if element is interactable based on platform + */ +export function isInteractableElement( + element: JSONElement, + _isNative: boolean, + automationName: string +): boolean { + const isAndroid = automationName.toLowerCase().includes('uiautomator') + const interactableTags = isAndroid + ? ANDROID_INTERACTABLE_TAGS + : IOS_INTERACTABLE_TAGS + + if (matchesTagList(element.tagName, interactableTags)) { + return true + } + + if (isAndroid) { + if ( + element.attributes?.clickable === 'true' || + element.attributes?.focusable === 'true' || + element.attributes?.checkable === 'true' || + element.attributes?.['long-clickable'] === 'true' + ) { + return true + } + } + + if (!isAndroid) { + if (element.attributes?.accessible === 'true') { + return true + } + } + + return false +} + +/** + * Check if element is a layout container + */ +export function isLayoutContainer( + element: JSONElement, + platform: 'android' | 'ios' +): boolean { + const containerList = + platform === 'android' ? ANDROID_LAYOUT_CONTAINERS : IOS_LAYOUT_CONTAINERS + return matchesTagList(element.tagName, containerList) +} + +/** + * Check if element has meaningful content (text, accessibility info) + */ +export function hasMeaningfulContent( + element: JSONElement, + platform: 'android' | 'ios' +): boolean { + const attrs = element.attributes + + if (attrs.text && attrs.text.trim() !== '' && attrs.text !== 'null') { + return true + } + + if (platform === 'android') { + if ( + attrs['content-desc'] && + attrs['content-desc'].trim() !== '' && + attrs['content-desc'] !== 'null' + ) { + return true + } + } else { + if (attrs.label && attrs.label.trim() !== '' && attrs.label !== 'null') { + return true + } + if (attrs.name && attrs.name.trim() !== '' && attrs.name !== 'null') { + return true + } + } + + return false +} + +/** + * Determine if an element should be included based on all filter criteria + */ +export function shouldIncludeElement( + element: JSONElement, + filters: FilterOptions, + isNative: boolean, + automationName: string +): boolean { + const { + includeTagNames = [], + excludeTagNames = ['hierarchy'], + requireAttributes = [], + minAttributeCount = 0, + fetchableOnly = false, + clickableOnly = false, + visibleOnly = true + } = filters + + if (!matchesTagFilters(element, includeTagNames, excludeTagNames)) { + if (element.attributes?.clickable !== 'true') { + return false + } + } + + if (!matchesAttributeFilters(element, requireAttributes, minAttributeCount)) { + return false + } + + if (clickableOnly && element.attributes?.clickable !== 'true') { + return false + } + + if (visibleOnly) { + const isAndroid = automationName.toLowerCase().includes('uiautomator') + if (isAndroid && element.attributes?.displayed === 'false') { + return false + } + if (!isAndroid && element.attributes?.visible === 'false') { + return false + } + } + + if ( + fetchableOnly && + !isInteractableElement(element, isNative, automationName) + ) { + return false + } + + return true +} + +/** + * Get default filter options for a platform + */ +export function getDefaultFilters( + platform: 'android' | 'ios', + includeContainers: boolean = false +): FilterOptions { + const layoutContainers = + platform === 'android' ? ANDROID_LAYOUT_CONTAINERS : IOS_LAYOUT_CONTAINERS + + return { + excludeTagNames: includeContainers + ? ['hierarchy'] + : ['hierarchy', ...layoutContainers], + fetchableOnly: !includeContainers, + visibleOnly: true, + clickableOnly: false + } +} diff --git a/packages/core/src/locators/index.ts b/packages/core/src/locators/index.ts new file mode 100644 index 00000000..06e88e22 --- /dev/null +++ b/packages/core/src/locators/index.ts @@ -0,0 +1,282 @@ +/** + * Mobile element locator generation + * + * Main orchestrator module that coordinates XML parsing, element filtering, + * and locator generation for mobile automation. + */ + +// Types +export type { + ElementAttributes, + JSONElement, + Bounds, + FilterOptions, + UniquenessResult, + LocatorStrategy, + LocatorContext, + ElementWithLocators, + GenerateLocatorsOptions +} from './types.js' + +// Constants +export { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS, + ANDROID_LAYOUT_CONTAINERS, + IOS_LAYOUT_CONTAINERS +} from './constants.js' + +// XML Parsing +export { + xmlToJSON, + xmlToDOM, + evaluateXPath, + checkXPathUniqueness, + findDOMNodeByPath, + parseAndroidBounds, + parseIOSBounds, + flattenElementTree, + countAttributeOccurrences, + isAttributeUnique +} from './xml-parsing.js' + +// Element Filtering +export { + isInteractableElement, + isLayoutContainer, + hasMeaningfulContent, + shouldIncludeElement, + getDefaultFilters +} from './element-filter.js' + +// Locator Generation +export { + getSuggestedLocators, + getBestLocator, + locatorsToObject +} from './locator-generation.js' + +import type { + JSONElement, + FilterOptions, + LocatorStrategy, + ElementWithLocators, + GenerateLocatorsOptions, + XMLDocument +} from './types.js' + +import { + xmlToJSON, + xmlToDOM, + parseAndroidBounds, + parseIOSBounds, + findDOMNodeByPath +} from './xml-parsing.js' +import { + shouldIncludeElement, + isLayoutContainer, + hasMeaningfulContent +} from './element-filter.js' +import { getSuggestedLocators, locatorsToObject } from './locator-generation.js' + +interface ProcessingContext { + sourceXML: string + platform: 'android' | 'ios' + automationName: string + isNative: boolean + viewportSize: { width: number; height: number } + filters: FilterOptions + inViewportOnly: boolean + results: ElementWithLocators[] + parsedDOM: XMLDocument | null +} + +/** + * Parse element bounds based on platform + */ +function parseBounds( + element: JSONElement, + platform: 'android' | 'ios' +): { x: number; y: number; width: number; height: number } { + return platform === 'android' + ? parseAndroidBounds(element.attributes.bounds || '') + : parseIOSBounds(element.attributes) +} + +/** + * Check if bounds are within viewport + */ +function isWithinViewport( + bounds: { x: number; y: number; width: number; height: number }, + viewport: { width: number; height: number } +): boolean { + return ( + bounds.x >= 0 && + bounds.y >= 0 && + bounds.width > 0 && + bounds.height > 0 && + bounds.x + bounds.width <= viewport.width && + bounds.y + bounds.height <= viewport.height + ) +} + +/** + * Transform JSONElement to ElementWithLocators + */ +function transformElement( + element: JSONElement, + locators: [LocatorStrategy, string][], + ctx: ProcessingContext +): ElementWithLocators { + const attrs = element.attributes + const bounds = parseBounds(element, ctx.platform) + + return { + tagName: element.tagName, + locators: locatorsToObject(locators), + text: attrs.text || attrs.label || '', + contentDesc: attrs['content-desc'] || '', + resourceId: attrs['resource-id'] || '', + accessibilityId: attrs.name || attrs['content-desc'] || '', + label: attrs.label || '', + value: attrs.value || '', + className: attrs.class || element.tagName, + clickable: + attrs.clickable === 'true' || + attrs.accessible === 'true' || + attrs['long-clickable'] === 'true', + enabled: attrs.enabled !== 'false', + displayed: + ctx.platform === 'android' + ? attrs.displayed !== 'false' + : attrs.visible !== 'false', + bounds, + isInViewport: isWithinViewport(bounds, ctx.viewportSize) + } +} + +/** + * Check if element should be processed + */ +function shouldProcess(element: JSONElement, ctx: ProcessingContext): boolean { + if ( + shouldIncludeElement(element, ctx.filters, ctx.isNative, ctx.automationName) + ) { + return true + } + return ( + isLayoutContainer(element, ctx.platform) && + hasMeaningfulContent(element, ctx.platform) + ) +} + +/** + * Process a single element and add to results if valid + */ +function processElement(element: JSONElement, ctx: ProcessingContext): void { + if (!shouldProcess(element, ctx)) { + return + } + + // Skip off-screen elements early when viewport filtering is on โ€” + // avoids expensive locator generation for elements the caller doesn't want. + if (ctx.inViewportOnly) { + const b = parseBounds(element, ctx.platform) + if (!isWithinViewport(b, ctx.viewportSize)) { + return + } + } + + try { + const targetNode = ctx.parsedDOM + ? findDOMNodeByPath(ctx.parsedDOM, element.path) + : undefined + + const locators = getSuggestedLocators( + element, + ctx.sourceXML, + ctx.automationName, + { + sourceXML: ctx.sourceXML, + parsedDOM: ctx.parsedDOM, + isAndroid: ctx.platform === 'android' + }, + targetNode || undefined + ) + if (locators.length === 0) { + return + } + + // Stash the best locator on the tree node so serializeMobileSnapshot + // can reuse the full locator pipeline instead of recomputing. + element.attributes._selector = locators[0][1] + + const transformed = transformElement(element, locators, ctx) + if (Object.keys(transformed.locators).length === 0) { + return + } + + ctx.results.push(transformed) + } catch (error) { + // Core is logger-free; console.error provides the required + // "enough detail to debug" per the error-handling convention. + // A single bad element never fails the entire page-source walk. + console.error(`[processElement] Error at path ${element.path}:`, error) + } +} + +/** + * Recursively traverse and process element tree + */ +function traverseTree( + element: JSONElement | null, + ctx: ProcessingContext +): void { + if (!element) { + return + } + + processElement(element, ctx) + + for (const child of element.children || []) { + traverseTree(child, ctx) + } +} + +/** + * Generate locators for all elements from page source XML + */ +export function generateAllElementLocators( + sourceXML: string, + options: GenerateLocatorsOptions +): ElementWithLocators[] { + const sourceJSON = xmlToJSON(sourceXML) + + if (!sourceJSON) { + // Core is logger-free; console.error is the only signal that XML + // parsing failed โ€” the caller receives an empty result silently otherwise. + console.error( + '[generateAllElementLocators] Failed to parse page source XML' + ) + return [] + } + + const parsedDOM = xmlToDOM(sourceXML) + + const ctx: ProcessingContext = { + sourceXML, + platform: options.platform, + automationName: + options.platform === 'android' ? 'uiautomator2' : 'xcuitest', + isNative: options.isNative ?? true, + viewportSize: options.viewportSize ?? { width: 9999, height: 9999 }, + filters: options.filters ?? {}, + inViewportOnly: options.inViewportOnly ?? true, + results: [], + parsedDOM + } + + traverseTree(sourceJSON, ctx) + + return ctx.results +} diff --git a/packages/core/src/locators/locator-generation.ts b/packages/core/src/locators/locator-generation.ts new file mode 100644 index 00000000..a66bdf49 --- /dev/null +++ b/packages/core/src/locators/locator-generation.ts @@ -0,0 +1,648 @@ +/** + * Locator strategy generation for mobile elements + */ + +import type { + JSONElement, + LocatorStrategy, + LocatorContext, + UniquenessResult, + XMLNode, + XMLDocument +} from './types.js' +import type { Element as XMLElement } from '@xmldom/xmldom' +import { + checkXPathUniqueness, + evaluateXPath, + isAttributeUnique +} from './xml-parsing.js' + +/** + * Check if a string value is valid for use in a locator + */ +function isValidValue(value: string | undefined): value is string { + return ( + value !== undefined && + value !== null && + value !== 'null' && + value.trim() !== '' + ) +} + +/** + * Escape special characters in text for use in selectors + */ +function escapeText(text: string): string { + return text.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n') +} + +/** + * Escape value for use in XPath expressions + */ +function escapeXPathValue(value: string): string { + if (!value.includes("'")) { + return `'${value}'` + } + if (!value.includes('"')) { + return `"${value}"` + } + const parts: string[] = [] + let current = '' + for (const char of value) { + if (char === "'") { + if (current) { + parts.push(`'${current}'`) + } + parts.push('"\'"') + current = '' + } else { + current += char + } + } + if (current) { + parts.push(`'${current}'`) + } + return `concat(${parts.join(',')})` +} + +/** + * Wrap non-unique XPath with index + */ +function generateIndexedXPath(baseXPath: string, index: number): string { + return `(${baseXPath})[${index}]` +} + +/** + * Add .instance(n) for UiAutomator (0-based) + */ +function generateIndexedUiAutomator( + baseSelector: string, + index: number +): string { + return `${baseSelector}.instance(${index - 1})` +} + +/** + * Check uniqueness, falling back to regex if no DOM available + */ +function checkUniqueness( + ctx: LocatorContext, + xpath: string, + targetNode?: XMLNode +): UniquenessResult { + if (ctx.parsedDOM) { + return checkXPathUniqueness(ctx.parsedDOM, xpath, targetNode) + } + + // Bounded quantifiers ({1,200} for attr names, {1,2000} for values) prevent + // polynomial-time backtracking on near-misses like `//[@<` repeated โ€” CodeQL's + // js/polynomial-redos query flagged the unbounded form. The bounds are well + // above any realistic XPath attribute name/value length. + const match = xpath.match(/\/\/\*\[@([^=]{1,200})="([^"]{1,2000})"\]/) + if (match) { + const [, attr, value] = match + return { isUnique: isAttributeUnique(ctx.sourceXML, attr, value) } + } + return { isUnique: false } +} + +/** + * Get sibling index (1-based) among same-tag siblings + */ +function getSiblingIndex(element: XMLElement): number { + const parent = element.parentNode + if (!parent) { + return 1 + } + + const tagName = element.nodeName + let index = 0 + + for (let i = 0; i < parent.childNodes.length; i++) { + const child = parent.childNodes.item(i) + if (child?.nodeType === 1 && child.nodeName === tagName) { + index++ + if (child === element) { + return index + } + } + } + + return 1 +} + +/** + * Count siblings with same tag name + */ +function countSiblings(element: XMLElement): number { + const parent = element.parentNode + if (!parent) { + return 1 + } + + const tagName = element.nodeName + let count = 0 + + for (let i = 0; i < parent.childNodes.length; i++) { + const child = parent.childNodes.item(i) + if (child?.nodeType === 1 && child.nodeName === tagName) { + count++ + } + } + + return count +} + +/** + * Find unique attribute for element in XPath format + */ +function findUniqueAttribute( + element: XMLElement, + ctx: LocatorContext +): string | null { + const attrs = ctx.isAndroid + ? ['resource-id', 'content-desc', 'text'] + : ['name', 'label', 'value'] + + for (const attr of attrs) { + const value = element.getAttribute(attr) + if (value && value.trim()) { + const xpath = `//*[@${attr}=${escapeXPathValue(value)}]` + const result = ctx.parsedDOM + ? checkXPathUniqueness(ctx.parsedDOM, xpath) + : { isUnique: isAttributeUnique(ctx.sourceXML, attr, value) } + + if (result.isUnique) { + return `@${attr}=${escapeXPathValue(value)}` + } + } + } + + return null +} + +/** + * Build hierarchical XPath by traversing up the DOM tree + */ +function buildHierarchicalXPath( + ctx: LocatorContext, + element: XMLElement, + maxDepth: number = 3 +): string | null { + if (!ctx.parsedDOM) { + return null + } + + const pathParts: string[] = [] + let current: XMLElement | null = element + let depth = 0 + + while (current && depth < maxDepth) { + const tagName = current.nodeName + const uniqueAttr = findUniqueAttribute(current, ctx) + + if (uniqueAttr) { + pathParts.unshift(`//${tagName}[${uniqueAttr}]`) + break + } else { + const siblingIndex = getSiblingIndex(current) + const siblingCount = countSiblings(current) + + if (siblingCount > 1) { + pathParts.unshift(`${tagName}[${siblingIndex}]`) + } else { + pathParts.unshift(tagName) + } + } + + const parent = current.parentNode as XMLElement | null + current = parent && parent.nodeType === 1 ? parent : null + depth++ + } + + if (pathParts.length === 0) { + return null + } + + let result = pathParts[0] + for (let i = 1; i < pathParts.length; i++) { + result += '/' + pathParts[i] + } + + if (!result.startsWith('//')) { + result = '//' + result + } + + return result +} + +/** + * Add XPath locator with uniqueness checking and fallbacks + */ +function addXPathLocator( + results: [LocatorStrategy, string][], + xpath: string, + ctx: LocatorContext, + targetNode?: XMLNode +): void { + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + if (uniqueness.isUnique) { + results.push(['xpath', xpath]) + } else if (uniqueness.index) { + results.push(['xpath', generateIndexedXPath(xpath, uniqueness.index)]) + } else { + if (targetNode && ctx.parsedDOM) { + // @xmldom/xmldom 0.9+ XMLNode doesn't satisfy global Node; safe at runtime + const hierarchical = buildHierarchicalXPath( + ctx, + targetNode as unknown as XMLElement + ) + if (hierarchical) { + results.push(['xpath', hierarchical]) + } + } + results.push(['xpath', xpath]) + } +} + +/** + * Check if element is within UiAutomator scope + */ +function isInUiAutomatorScope( + element: JSONElement, + doc: XMLDocument | null +): boolean { + if (!doc) { + return true + } + + const hierarchyNodes = evaluateXPath(doc, '/hierarchy/*') + if (hierarchyNodes.length === 0) { + return true + } + + const lastIndex = hierarchyNodes.length + const pathParts = element.path.split('.') + if (pathParts.length === 0 || pathParts[0] === '') { + return true + } + + const firstIndex = parseInt(pathParts[0], 10) + return firstIndex === lastIndex - 1 +} + +/** + * Build Android UiAutomator selector with multiple attributes + */ +function buildUiAutomatorSelector(element: JSONElement): string | null { + const attrs = element.attributes + const parts: string[] = [] + + if (isValidValue(attrs['resource-id'])) { + parts.push(`resourceId("${attrs['resource-id']}")`) + } + if (isValidValue(attrs.text) && attrs.text!.length < 100) { + parts.push(`text("${escapeText(attrs.text!)}")`) + } + if (isValidValue(attrs['content-desc'])) { + parts.push(`description("${attrs['content-desc']}")`) + } + if (isValidValue(attrs.class)) { + parts.push(`className("${attrs.class}")`) + } + + if (parts.length === 0) { + return null + } + + return `android=new UiSelector().${parts.join('.')}` +} + +/** + * Build iOS predicate string with multiple conditions + */ +function buildPredicateString(element: JSONElement): string | null { + const attrs = element.attributes + const conditions: string[] = [] + + if (isValidValue(attrs.name)) { + conditions.push(`name == "${escapeText(attrs.name!)}"`) + } + if (isValidValue(attrs.label)) { + conditions.push(`label == "${escapeText(attrs.label!)}"`) + } + if (isValidValue(attrs.value)) { + conditions.push(`value == "${escapeText(attrs.value!)}"`) + } + if (attrs.visible === 'true') { + conditions.push('visible == 1') + } + if (attrs.enabled === 'true') { + conditions.push('enabled == 1') + } + + if (conditions.length === 0) { + return null + } + + return `-ios predicate string:${conditions.join(' AND ')}` +} + +/** + * Build iOS class chain selector + */ +function buildClassChain(element: JSONElement): string | null { + const attrs = element.attributes + const tagName = element.tagName + + if (!tagName.startsWith('XCUI')) { + return null + } + + let selector = `**/${tagName}` + + if (isValidValue(attrs.label)) { + selector += `[\`label == "${escapeText(attrs.label!)}"\`]` + } else if (isValidValue(attrs.name)) { + selector += `[\`name == "${escapeText(attrs.name!)}"\`]` + } + + return `-ios class chain:${selector}` +} + +/** + * Build XPath for element with unique identification + */ +function buildXPath( + element: JSONElement, + _sourceXML: string, + isAndroid: boolean +): string | null { + const attrs = element.attributes + const tagName = element.tagName + const conditions: string[] = [] + + if (isAndroid) { + if (isValidValue(attrs['resource-id'])) { + conditions.push(`@resource-id="${attrs['resource-id']}"`) + } + if (isValidValue(attrs['content-desc'])) { + conditions.push(`@content-desc="${attrs['content-desc']}"`) + } + if (isValidValue(attrs.text) && attrs.text!.length < 100) { + conditions.push(`@text="${escapeText(attrs.text!)}"`) + } + } else { + if (isValidValue(attrs.name)) { + conditions.push(`@name="${attrs.name}"`) + } + if (isValidValue(attrs.label)) { + conditions.push(`@label="${attrs.label}"`) + } + if (isValidValue(attrs.value)) { + conditions.push(`@value="${attrs.value}"`) + } + } + + if (conditions.length === 0) { + return `//${tagName}` + } + + return `//${tagName}[${conditions.join(' and ')}]` +} + +/** + * Get simple locators based on single attributes + */ +function getSimpleSuggestedLocators( + element: JSONElement, + ctx: LocatorContext, + automationName: string, + targetNode?: XMLNode +): [LocatorStrategy, string][] { + const results: [LocatorStrategy, string][] = [] + const isAndroid = automationName.toLowerCase().includes('uiautomator') + const attrs = element.attributes + const inUiAutomatorScope = isAndroid + ? isInUiAutomatorScope(element, ctx.parsedDOM) + : true + + if (isAndroid) { + // Resource ID + const resourceId = attrs['resource-id'] + if (isValidValue(resourceId)) { + const xpath = `//*[@resource-id="${resourceId}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique && inUiAutomatorScope) { + results.push([ + 'id', + `android=new UiSelector().resourceId("${resourceId}")` + ]) + } else if (uniqueness.index && inUiAutomatorScope) { + const base = `android=new UiSelector().resourceId("${resourceId}")` + results.push(['id', generateIndexedUiAutomator(base, uniqueness.index)]) + } + } + + // Content Description + const contentDesc = attrs['content-desc'] + if (isValidValue(contentDesc)) { + const xpath = `//*[@content-desc="${contentDesc}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push(['accessibility-id', `~${contentDesc}`]) + } + } + + // Text + const text = attrs.text + if (isValidValue(text) && text.length < 100) { + const xpath = `//*[@text="${escapeText(text)}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique && inUiAutomatorScope) { + results.push([ + 'text', + `android=new UiSelector().text("${escapeText(text)}")` + ]) + } else if (uniqueness.index && inUiAutomatorScope) { + const base = `android=new UiSelector().text("${escapeText(text)}")` + results.push([ + 'text', + generateIndexedUiAutomator(base, uniqueness.index) + ]) + } + } + } else { + // iOS: Accessibility ID (name) + const name = attrs.name + if (isValidValue(name)) { + const xpath = `//*[@name="${name}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push(['accessibility-id', `~${name}`]) + } + } + + // iOS: Label + const label = attrs.label + if (isValidValue(label) && label !== attrs.name) { + const xpath = `//*[@label="${escapeText(label)}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push([ + 'predicate-string', + `-ios predicate string:label == "${escapeText(label)}"` + ]) + } + } + + // iOS: Value + const value = attrs.value + if (isValidValue(value)) { + const xpath = `//*[@value="${escapeText(value)}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push([ + 'predicate-string', + `-ios predicate string:value == "${escapeText(value)}"` + ]) + } + } + } + + return results +} + +/** + * Get complex locators (combinations, XPath, etc.) + */ +function getComplexSuggestedLocators( + element: JSONElement, + ctx: LocatorContext, + automationName: string, + targetNode?: XMLNode +): [LocatorStrategy, string][] { + const results: [LocatorStrategy, string][] = [] + const isAndroid = automationName.toLowerCase().includes('uiautomator') + const inUiAutomatorScope = isAndroid + ? isInUiAutomatorScope(element, ctx.parsedDOM) + : true + + if (isAndroid) { + if (inUiAutomatorScope) { + const uiAutomator = buildUiAutomatorSelector(element) + if (uiAutomator) { + results.push(['uiautomator', uiAutomator]) + } + } + + const xpath = buildXPath(element, ctx.sourceXML, true) + if (xpath) { + addXPathLocator(results, xpath, ctx, targetNode) + } + + if (inUiAutomatorScope && isValidValue(element.attributes.class)) { + results.push([ + 'class-name', + `android=new UiSelector().className("${element.attributes.class}")` + ]) + } + } else { + const predicate = buildPredicateString(element) + if (predicate) { + results.push(['predicate-string', predicate]) + } + + const classChain = buildClassChain(element) + if (classChain) { + results.push(['class-chain', classChain]) + } + + const xpath = buildXPath(element, ctx.sourceXML, false) + if (xpath) { + addXPathLocator(results, xpath, ctx, targetNode) + } + + const type = element.tagName + if (type.startsWith('XCUIElementType')) { + results.push(['class-name', `-ios class chain:**/${type}`]) + } + } + + return results +} + +/** + * Get all suggested locators for an element + */ +export function getSuggestedLocators( + element: JSONElement, + sourceXML: string, + automationName: string, + ctx?: LocatorContext, + targetNode?: XMLNode +): [LocatorStrategy, string][] { + const locatorCtx = ctx ?? { + sourceXML, + parsedDOM: null, + isAndroid: automationName.toLowerCase().includes('uiautomator') + } + + const simpleLocators = getSimpleSuggestedLocators( + element, + locatorCtx, + automationName, + targetNode + ) + const complexLocators = getComplexSuggestedLocators( + element, + locatorCtx, + automationName, + targetNode + ) + + const seen = new Set() + const results: [LocatorStrategy, string][] = [] + + for (const locator of [...simpleLocators, ...complexLocators]) { + if (!seen.has(locator[1])) { + seen.add(locator[1]) + results.push(locator) + } + } + + return results +} + +/** + * Get the best (first priority) locator for an element + */ +export function getBestLocator( + element: JSONElement, + sourceXML: string, + automationName: string +): string | null { + const locators = getSuggestedLocators(element, sourceXML, automationName) + return locators.length > 0 ? locators[0][1] : null +} + +/** + * Convert locator array to object format + */ +export function locatorsToObject( + locators: [LocatorStrategy, string][] +): Record { + const result: Record = {} + for (const [strategy, value] of locators) { + if (!result[strategy]) { + result[strategy] = value + } + } + return result +} diff --git a/packages/core/src/locators/types.ts b/packages/core/src/locators/types.ts new file mode 100644 index 00000000..28f5a497 --- /dev/null +++ b/packages/core/src/locators/types.ts @@ -0,0 +1,110 @@ +/** + * Type definitions for mobile element locator generation + */ + +import type { Document as XMLDocument, Node as XMLNode } from '@xmldom/xmldom' +export type { XMLDocument, XMLNode } + +export interface ElementAttributes { + // Android attributes + 'resource-id'?: string + 'content-desc'?: string + text?: string + class?: string + package?: string + clickable?: string + 'long-clickable'?: string + focusable?: string + checkable?: string + scrollable?: string + enabled?: string + displayed?: string + bounds?: string // Format: "[x1,y1][x2,y2]" + + // iOS attributes + type?: string + name?: string + label?: string + value?: string + accessible?: string + visible?: string + x?: string + y?: string + width?: string + height?: string + + // Generic + [key: string]: string | undefined +} + +export interface JSONElement { + children: JSONElement[] + tagName: string + attributes: ElementAttributes + path: string // Dot-separated index path for tree traversal +} + +export interface Bounds { + x: number + y: number + width: number + height: number +} + +export interface FilterOptions { + includeTagNames?: string[] // Only include these tags (whitelist) + excludeTagNames?: string[] // Exclude these tags (blacklist) + requireAttributes?: string[] // Must have at least one of these attributes + minAttributeCount?: number // Minimum number of non-empty attributes + fetchableOnly?: boolean // Only interactable elements + clickableOnly?: boolean // Only elements with clickable="true" + visibleOnly?: boolean // Only visible/displayed elements +} + +export interface UniquenessResult { + isUnique: boolean + index?: number // 1-based index if not unique + totalMatches?: number +} + +export type LocatorStrategy = + | 'accessibility-id' + | 'id' + | 'class-name' + | 'xpath' + | 'predicate-string' + | 'class-chain' + | 'uiautomator' + | 'text' + +export interface LocatorContext { + sourceXML: string + parsedDOM: XMLDocument | null + isAndroid: boolean +} + +export interface ElementWithLocators { + tagName: string + locators: Record + text: string + contentDesc: string + resourceId: string + accessibilityId: string + label: string + value: string + className: string + clickable: boolean + enabled: boolean + displayed: boolean + bounds: Bounds + isInViewport: boolean +} + +export interface GenerateLocatorsOptions { + platform: 'android' | 'ios' + viewportSize?: { width: number; height: number } + filters?: FilterOptions + isNative?: boolean + /** Only return elements whose bounds intersect the viewport (default true). */ + inViewportOnly?: boolean +} diff --git a/packages/core/src/locators/xml-parsing.ts b/packages/core/src/locators/xml-parsing.ts new file mode 100644 index 00000000..a100a042 --- /dev/null +++ b/packages/core/src/locators/xml-parsing.ts @@ -0,0 +1,329 @@ +/** + * XML parsing utilities for mobile element source + */ + +import { DOMParser } from '@xmldom/xmldom' +import type { + Document as XMLDocument, + Element as XMLElement, + Node as XMLNode +} from '@xmldom/xmldom' +import xpath from 'xpath' +import type { + ElementAttributes, + JSONElement, + Bounds, + UniquenessResult +} from './types.js' + +/** + * Get child nodes that are elements (not text nodes, comments, etc.) + */ +function childNodesOf(node: XMLNode): XMLNode[] { + const children: XMLNode[] = [] + if (node.childNodes) { + for (let i = 0; i < node.childNodes.length; i++) { + const child = node.childNodes.item(i) + if (child?.nodeType === 1) { + children.push(child) + } + } + } + return children +} + +/** + * Recursively translate DOM node to JSONElement + */ +function translateRecursively( + domNode: XMLNode, + parentPath: string = '', + index: number | null = null +): JSONElement { + const attributes: ElementAttributes = {} + + const element = domNode as XMLElement + if (element.attributes) { + for (let attrIdx = 0; attrIdx < element.attributes.length; attrIdx++) { + const attr = element.attributes.item(attrIdx) + if (attr) { + attributes[attr.name] = attr.value.replace(/(\n)/gm, '\\n') + } + } + } + + const path = + index === null ? '' : `${parentPath ? parentPath + '.' : ''}${index}` + + return { + children: childNodesOf(domNode).map((childNode, childIndex) => + translateRecursively(childNode as XMLNode, path, childIndex) + ), + tagName: domNode.nodeName, + attributes, + path + } +} + +/** + * Compare two nodes for equality by platform-specific attributes + * (reference equality via === may fail when nodes come from different traversals) + */ +function isSameElement(node1: XMLNode, node2: XMLNode): boolean { + if (node1.nodeType !== 1 || node2.nodeType !== 1) { + return false + } + const el1 = node1 as XMLElement + const el2 = node2 as XMLElement + + if (el1.nodeName !== el2.nodeName) { + return false + } + + // For Android, compare by bounds (unique per element) + const bounds1 = el1.getAttribute('bounds') + const bounds2 = el2.getAttribute('bounds') + if (bounds1 && bounds2) { + return bounds1 === bounds2 + } + + // For iOS, compare by x, y, width, height + const x1 = el1.getAttribute('x') + const y1 = el1.getAttribute('y') + const x2 = el2.getAttribute('x') + const y2 = el2.getAttribute('y') + if (x1 && y1 && x2 && y2) { + return ( + x1 === x2 && + y1 === y2 && + el1.getAttribute('width') === el2.getAttribute('width') && + el1.getAttribute('height') === el2.getAttribute('height') + ) + } + + return false +} + +/** + * Convert XML page source to JSON tree structure + */ +export function xmlToJSON(sourceXML: string): JSONElement | null { + try { + const parser = new DOMParser() + const sourceDoc = parser.parseFromString(sourceXML, 'text/xml') + + // xmldom 0.9+ throws ParseError for fatal errors (caught below); this catches non-fatal cases + const parseErrors = sourceDoc.getElementsByTagName('parsererror') + if (parseErrors.length > 0) { + console.error( + '[xmlToJSON] XML parsing error:', + parseErrors[0].textContent + ) + return null + } + + const children = childNodesOf(sourceDoc) + const firstChild = + children[0] || + (sourceDoc.documentElement + ? childNodesOf(sourceDoc.documentElement)[0] + : null) + + return firstChild + ? translateRecursively(firstChild) + : { children: [], tagName: '', attributes: {}, path: '' } + } catch (e) { + console.error('[xmlToJSON] Failed to parse XML:', e) + return null + } +} + +/** + * Parse XML source to DOM Document for XPath evaluation + */ +export function xmlToDOM(sourceXML: string): XMLDocument | null { + try { + const parser = new DOMParser() + const doc = parser.parseFromString(sourceXML, 'text/xml') + + // xmldom 0.9+ throws ParseError for fatal errors (caught below); this catches non-fatal cases + const parseErrors = doc.getElementsByTagName('parsererror') + if (parseErrors.length > 0) { + console.error('[xmlToDOM] XML parsing error:', parseErrors[0].textContent) + return null + } + + return doc + } catch (e) { + console.error('[xmlToDOM] Failed to parse XML:', e) + return null + } +} + +/** + * Execute XPath query on DOM document + */ +export function evaluateXPath(doc: XMLDocument, xpathExpr: string): XMLNode[] { + try { + // @xmldom/xmldom 0.9+ types don't satisfy global Node; xpath still works at runtime + const nodes = xpath.select(xpathExpr, doc as unknown as Node) + if (Array.isArray(nodes)) { + return nodes as unknown as XMLNode[] + } + return [] + } catch (e) { + console.error(`[evaluateXPath] Failed to evaluate "${xpathExpr}":`, e) + return [] + } +} + +/** + * Check if an XPath selector is unique and get index if not + */ +export function checkXPathUniqueness( + doc: XMLDocument, + xpathExpr: string, + targetNode?: XMLNode +): UniquenessResult { + try { + const nodes = evaluateXPath(doc, xpathExpr) + const totalMatches = nodes.length + + if (totalMatches === 0) { + return { isUnique: false } + } + + if (totalMatches === 1) { + return { isUnique: true } + } + + // Not unique - find index of target node if provided + if (targetNode) { + for (let i = 0; i < nodes.length; i++) { + if (nodes[i] === targetNode || isSameElement(nodes[i], targetNode)) { + return { + isUnique: false, + index: i + 1, // 1-based index for XPath + totalMatches + } + } + } + } + + return { isUnique: false, totalMatches } + } catch (e) { + console.error(`[checkXPathUniqueness] Error checking "${xpathExpr}":`, e) + return { isUnique: false } + } +} + +/** + * Find DOM node by JSONElement path (e.g., "0.2.1") + */ +export function findDOMNodeByPath( + doc: XMLDocument, + path: string +): XMLNode | null { + if (!path) { + return doc.documentElement + } + + const indices = path.split('.').map(Number) + let current: XMLNode | null = doc.documentElement + + for (const index of indices) { + if (!current) { + return null + } + + const children: XMLNode[] = [] + if (current.childNodes) { + for (let i = 0; i < current.childNodes.length; i++) { + const child = current.childNodes.item(i) + if (child?.nodeType === 1) { + children.push(child) + } + } + } + + current = children[index] || null + } + + return current +} + +/** + * Parse Android bounds string "[x1,y1][x2,y2]" to coordinates + */ +export function parseAndroidBounds(bounds: string): Bounds { + const match = bounds.match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/) + if (!match) { + return { x: 0, y: 0, width: 0, height: 0 } + } + + const x1 = parseInt(match[1], 10) + const y1 = parseInt(match[2], 10) + const x2 = parseInt(match[3], 10) + const y2 = parseInt(match[4], 10) + + return { + x: x1, + y: y1, + width: x2 - x1, + height: y2 - y1 + } +} + +/** + * Parse iOS element bounds from individual x, y, width, height attributes + */ +export function parseIOSBounds(attributes: ElementAttributes): Bounds { + return { + x: parseInt(attributes.x || '0', 10), + y: parseInt(attributes.y || '0', 10), + width: parseInt(attributes.width || '0', 10), + height: parseInt(attributes.height || '0', 10) + } +} + +/** + * Flatten JSON element tree to array (depth-first) + */ +export function flattenElementTree(root: JSONElement): JSONElement[] { + const result: JSONElement[] = [] + + function traverse(element: JSONElement) { + result.push(element) + for (const child of element.children) { + traverse(child) + } + } + + traverse(root) + return result +} + +/** + * Count occurrences of an attribute value in the source XML + */ +export function countAttributeOccurrences( + sourceXML: string, + attribute: string, + value: string +): number { + const escapedValue = value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const pattern = new RegExp(`${attribute}=["']${escapedValue}["']`, 'g') + const matches = sourceXML.match(pattern) + return matches ? matches.length : 0 +} + +/** + * Check if an attribute value is unique in the source (fast regex-based check) + */ +export function isAttributeUnique( + sourceXML: string, + attribute: string, + value: string +): boolean { + return countAttributeOccurrences(sourceXML, attribute, value) === 1 +} diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts new file mode 100644 index 00000000..27374a58 --- /dev/null +++ b/packages/core/src/trace-exporter.ts @@ -0,0 +1,418 @@ +// Converts a captured TraceLog into a trace.zip Buffer. +// Stays runner-agnostic so the three adapters can call this directly. + +import fs from 'node:fs/promises' +import path from 'node:path' +import type { + ActionSnapshot, + CommandLog, + ConsoleLog, + Metadata, + NetworkRequest, + TraceFormat, + TraceLog, + TraceMutation +} from '@wdio/devtools-shared' +import { formatActionTitle, mapCommandToAction } from './action-mapping.js' +import { networkRequestToHar } from './trace-har.js' +import { buildTraceZip, type TraceZipResource } from './trace-zip-writer.js' + +const TRACE_VERSION = 8 +const LIBRARY_NAME = '@wdio/devtools-core' +const LIBRARY_VERSION = '1.0.0' + +interface ContextOptionsEvent { + version: number + type: 'context-options' + origin: 'library' + libraryName: string + libraryVersion: string + browserName: string + platform: string + wallTime: number + monotonicTime: number + sdkLanguage: string + title: string + contextId: string + options: { viewport: { width: number; height: number } } +} + +interface BeforeEvent { + type: 'before' + callId: string + startTime: number + class: string + method: string + pageId: string + params: Record + title: string +} + +interface AfterEvent { + type: 'after' + callId: string + endTime: number + error?: { message: string } +} + +interface ScreencastFrameEvent { + type: 'screencast-frame' + pageId: string + sha1: string + elements?: string + width: number + height: number + timestamp: number +} + +type TraceEvent = + | ContextOptionsEvent + | BeforeEvent + | AfterEvent + | ScreencastFrameEvent + +function shortId(sessionId?: string): string { + return (sessionId ?? Math.random().toString(36).slice(2, 10)).slice(0, 8) +} + +function resolveContextNaming(caps: Record | undefined): { + browserName: string + title: string +} { + const platformName = + typeof caps?.platformName === 'string' + ? caps.platformName.toLowerCase() + : undefined + const deviceName = + typeof caps?.['appium:deviceName'] === 'string' + ? (caps['appium:deviceName'] as string) + : undefined + if (platformName === 'android' || platformName === 'ios') { + return { + browserName: 'chromium', + title: deviceName ? `${platformName} โ€” ${deviceName}` : platformName + } + } + const browserName = + typeof caps?.browserName === 'string' ? caps.browserName : 'chromium' + return { browserName, title: browserName } +} + +function buildContextOptions( + trace: TraceLog, + contextId: string, + wallTime: number +): ContextOptionsEvent { + const caps = trace.metadata.capabilities as + | Record + | undefined + const { browserName, title } = resolveContextNaming(caps) + const viewport = trace.metadata.viewport ?? { width: 1280, height: 720 } + return { + version: TRACE_VERSION, + type: 'context-options', + origin: 'library', + libraryName: LIBRARY_NAME, + libraryVersion: LIBRARY_VERSION, + browserName, + platform: process.platform, + wallTime, + monotonicTime: 0, + sdkLanguage: 'javascript', + title, + contextId, + options: { + viewport: { width: viewport.width, height: viewport.height } + } + } +} + +function buildActionEvents( + commands: CommandLog[], + pageId: string, + wallTime: number +): TraceEvent[] { + const events: TraceEvent[] = [] + // cmd.timestamp records command completion, so the *previous* mapped + // command's timestamp is a usable startTime for the next one. + let prevEndMs = 0 + let callCounter = 0 + for (const cmd of commands) { + const action = mapCommandToAction(cmd.command) + if (!action) { + continue + } + callCounter++ + const callId = `call@${callCounter}` + // +1ms minimum duration guarantees endTime > startTime so the viewer + // never sees an `after` whose matching `before` hasn't been parsed yet + // (its action-map lookup crashes on undefined and aborts trace load). + const endMs = Math.max(prevEndMs + 1, cmd.timestamp - wallTime) + const params: Record = Object.fromEntries( + cmd.args.map((a, i) => [String(i), a]) + ) + events.push({ + type: 'before', + callId, + startTime: prevEndMs, + class: action.class, + method: action.method, + pageId, + params, + title: formatActionTitle(action, cmd.args, params) + }) + const afterEvent: AfterEvent = { + type: 'after', + callId, + endTime: endMs + } + if (cmd.error) { + const err = cmd.error as { message?: string } + afterEvent.error = { message: err.message ?? String(cmd.error) } + } + events.push(afterEvent) + prevEndMs = endMs + } + return events +} + +function buildNetworkNdjson(requests: NetworkRequest[]): Buffer { + if (!requests.length) { + return Buffer.alloc(0) + } + const lines = requests.map((r) => JSON.stringify(networkRequestToHar(r))) + return Buffer.from(lines.join('\n'), 'utf8') +} + +function buildSnapshotResources( + snapshots: ActionSnapshot[], + pageId: string +): TraceZipResource[] { + const out: TraceZipResource[] = [] + for (const snap of snapshots) { + const base = `${pageId}-${snap.timestamp}` + if (snap.screenshot) { + out.push({ + resourceName: `${base}.jpeg`, + data: Buffer.from(snap.screenshot, 'base64') + }) + } + if (snap.elements && snap.elements.length) { + out.push({ + resourceName: `elements-${base}.json`, + data: Buffer.from(JSON.stringify(snap.elements), 'utf8') + }) + } + if (snap.snapshotText) { + out.push({ + resourceName: `snapshot-${base}.txt`, + data: Buffer.from(snap.snapshotText, 'utf8') + }) + } + } + return out +} + +function buildScreencastFrames( + snapshots: ActionSnapshot[], + pageId: string, + wallTime: number, + viewport: { width: number; height: number } +): ScreencastFrameEvent[] { + return snapshots + .filter((s) => s.screenshot) + .map((s) => { + const base = `${pageId}-${s.timestamp}` + const frame: ScreencastFrameEvent = { + type: 'screencast-frame', + pageId, + sha1: `${base}.jpeg`, + width: viewport.width, + height: viewport.height, + timestamp: Math.max(0, s.timestamp - wallTime) + } + if (s.elements && s.elements.length) { + frame.elements = `elements-${base}.json` + } + return frame + }) +} + +/** + * Build a trace.zip buffer from the captured TraceLog. + * Filters commands through ACTION_MAP and renames to trace vocabulary; + * network entries become HAR resource-snapshots; per-action screenshots, + * element JSON, and snapshot text are written under `resources/`. + */ +/** Chronological sort key โ€” see `compareEvents` for the tie-breaker rationale. */ +function eventTime(e: TraceEvent): number { + switch (e.type) { + case 'context-options': + return -Infinity + case 'before': + return e.startTime + case 'after': + return e.endTime + case 'screencast-frame': + return e.timestamp + } +} + +/** At the same timestamp T: an action's `after` ends first, then the + * snapshot captured at the action boundary, then the next action's `before`. + * Matches the viewer's expectation that the screencast frame shows the + * state between the previous action's completion and the next one's start. */ +function eventOrder(e: TraceEvent): number { + switch (e.type) { + case 'context-options': + return 0 + case 'after': + return 1 + case 'screencast-frame': + return 2 + case 'before': + return 3 + } +} + +function compareEvents(a: TraceEvent, b: TraceEvent): number { + const dt = eventTime(a) - eventTime(b) + return dt !== 0 ? dt : eventOrder(a) - eventOrder(b) +} + +interface TraceBundle { + traceNdjson: string + networkNdjson: Buffer + resources: TraceZipResource[] +} + +function buildTraceBundle( + trace: TraceLog, + opts: { sessionId?: string; wallTimeOverride?: number } = {} +): TraceBundle { + // wallTime anchors monotonic offsets at the first captured command so + // subsequent actions render at positive deltas in the trace viewer. + const firstCommandTs = trace.commands[0]?.timestamp + const wallTime = opts.wallTimeOverride ?? firstCommandTs ?? Date.now() + const idPrefix = shortId(opts.sessionId) + const contextId = `context@${idPrefix}` + const pageId = `page@${idPrefix}` + const viewport = trace.metadata.viewport ?? { width: 1280, height: 720 } + const snapshots = trace.actionSnapshots ?? [] + const events: TraceEvent[] = [ + buildContextOptions(trace, contextId, wallTime), + ...buildScreencastFrames(snapshots, pageId, wallTime, viewport), + ...buildActionEvents(trace.commands, pageId, wallTime) + ].sort(compareEvents) + return { + traceNdjson: events.map((e) => JSON.stringify(e)).join('\n'), + networkNdjson: buildNetworkNdjson(trace.networkRequests), + resources: buildSnapshotResources(snapshots, pageId) + } +} + +export async function exportTraceZip( + trace: TraceLog, + opts: { sessionId?: string; wallTimeOverride?: number } = {} +): Promise { + return buildTraceZip(buildTraceBundle(trace, opts)) +} + +async function exportTraceDirectory( + trace: TraceLog, + targetDir: string, + opts: { sessionId?: string; wallTimeOverride?: number } = {} +): Promise { + const bundle = buildTraceBundle(trace, opts) + await fs.mkdir(path.join(targetDir, 'resources'), { recursive: true }) + await Promise.all([ + fs.writeFile(path.join(targetDir, 'trace.trace'), bundle.traceNdjson), + bundle.networkNdjson.length + ? fs.writeFile( + path.join(targetDir, 'trace.network'), + bundle.networkNdjson + ) + : Promise.resolve(), + ...bundle.resources.map((r) => + fs.writeFile(path.join(targetDir, 'resources', r.resourceName), r.data) + ) + ]) +} + +/** Minimum capturer surface needed to assemble a TraceLog. */ +export interface TraceCapturer { + mutations: TraceMutation[] + traceLogs: string[] + consoleLogs: ConsoleLog[] + networkRequests: NetworkRequest[] + commandsLog: CommandLog[] + sources: Map + metadata?: Metadata +} + +export interface WriteTraceZipOptions { + outputDir: string + sessionId: string + capabilities?: unknown + /** + * Per-action snapshots from a Phase-3-style hook. When omitted, snapshots + * are synthesized from CommandLog entries that carry a screenshot so the + * viewer still renders thumbnails for adapters without an action hook. + */ + actionSnapshots?: ActionSnapshot[] + /** Output layout โ€” `zip` (default) writes a single archive, `directory` + * unpacks the same files into `trace-/`. */ + format?: TraceFormat +} + +/** + * Build a TraceLog from a SessionCapturer-shaped source and write the trace + * artifact (zip file or directory). Returns the absolute path written. + */ +export async function writeTraceZip( + capturer: TraceCapturer, + opts: WriteTraceZipOptions +): Promise { + const baseMetadata = capturer.metadata ?? ({} as Metadata) + const actionSnapshots = + opts.actionSnapshots ?? + synthesizeSnapshotsFromCommands(capturer.commandsLog) + const traceLog: TraceLog = { + mutations: capturer.mutations, + logs: capturer.traceLogs, + consoleLogs: capturer.consoleLogs, + networkRequests: capturer.networkRequests, + metadata: { + ...baseMetadata, + ...(opts.capabilities + ? { capabilities: opts.capabilities as Metadata['capabilities'] } + : {}) + }, + commands: capturer.commandsLog, + sources: Object.fromEntries(capturer.sources), + ...(actionSnapshots.length ? { actionSnapshots } : {}) + } + await fs.mkdir(opts.outputDir, { recursive: true }) + if (opts.format === 'ndjson-directory') { + const dir = path.join(opts.outputDir, `trace-${opts.sessionId}`) + await fs.mkdir(dir, { recursive: true }) + await exportTraceDirectory(traceLog, dir, { sessionId: opts.sessionId }) + return dir + } + const zip = await exportTraceZip(traceLog, { sessionId: opts.sessionId }) + const zipPath = path.join(opts.outputDir, `trace-${opts.sessionId}.zip`) + await fs.writeFile(zipPath, zip) + return zipPath +} + +function synthesizeSnapshotsFromCommands( + commands: CommandLog[] +): ActionSnapshot[] { + return commands + .filter((c) => c.screenshot && mapCommandToAction(c.command)) + .map((c) => ({ + timestamp: c.timestamp, + command: c.command, + screenshot: c.screenshot + })) +} diff --git a/packages/core/src/trace-har.ts b/packages/core/src/trace-har.ts new file mode 100644 index 00000000..565053ff --- /dev/null +++ b/packages/core/src/trace-har.ts @@ -0,0 +1,100 @@ +// Convert the existing NetworkRequest shape into trace format +// `resource-snapshot` NDJSON entries (HAR-flavoured) for trace.zip. + +import type { NetworkRequest } from '@wdio/devtools-shared' + +export interface ResourceSnapshotEntry { + type: 'resource-snapshot' + snapshot: { + startedDateTime: string + time: number + request: { + method: string + url: string + httpVersion: string + cookies: unknown[] + headers: { name: string; value: string }[] + queryString: { name: string; value: string }[] + headersSize: number + bodySize: number + } + response: { + status: number + statusText: string + httpVersion: string + cookies: unknown[] + headers: { name: string; value: string }[] + content: { size: number; mimeType: string } + redirectURL: string + headersSize: number + bodySize: number + } + cache: Record + timings: { send: number; wait: number; receive: number } + } +} + +function toHeaderArray( + h: Record | undefined +): { name: string; value: string }[] { + if (!h) { + return [] + } + return Object.entries(h).map(([name, value]) => ({ name, value })) +} + +function toQueryString(url: string): { name: string; value: string }[] { + try { + const u = new URL(url) + const out: { name: string; value: string }[] = [] + u.searchParams.forEach((value, name) => out.push({ name, value })) + return out + } catch { + return [] + } +} + +export function networkRequestToHar( + entry: NetworkRequest +): ResourceSnapshotEntry { + const startedDateTime = new Date(entry.timestamp).toISOString() + const duration = + entry.time ?? (entry.endTime ?? entry.startTime) - entry.startTime + const status = entry.response?.status ?? entry.status ?? 0 + const mimeType = entry.response?.mimeType ?? '' + const responseHeaders = entry.response?.headers ?? entry.responseHeaders + return { + type: 'resource-snapshot', + snapshot: { + startedDateTime, + time: Math.max(0, duration), + request: { + method: entry.method, + url: entry.url, + httpVersion: 'HTTP/1.1', + cookies: [], + headers: toHeaderArray(entry.requestHeaders ?? entry.headers), + queryString: toQueryString(entry.url), + headersSize: -1, + bodySize: entry.requestBody ? entry.requestBody.length : -1 + }, + response: { + status, + statusText: entry.statusText ?? '', + httpVersion: 'HTTP/1.1', + cookies: [], + headers: toHeaderArray(responseHeaders), + content: { size: entry.size ?? 0, mimeType }, + redirectURL: '', + headersSize: -1, + bodySize: entry.size ?? -1 + }, + cache: {}, + timings: { + send: -1, + wait: Math.max(0, duration), + receive: -1 + } + } + } +} diff --git a/packages/core/src/trace-zip-writer.ts b/packages/core/src/trace-zip-writer.ts new file mode 100644 index 00000000..f3dfac69 --- /dev/null +++ b/packages/core/src/trace-zip-writer.ts @@ -0,0 +1,35 @@ +// Thin yazl wrapper that packages a trace into a single Buffer. +// Ported from Vince Graics' PR #209. + +import yazl from 'yazl' + +export interface TraceZipResource { + /** Path inside the zip, e.g. `resources/page@xxx-12345.jpeg`. */ + resourceName: string + data: Buffer +} + +export interface TraceZipInputs { + /** NDJSON action events (one JSON object per line). */ + traceNdjson: string + /** NDJSON HAR resource-snapshot entries. Empty buffer when omitted. */ + networkNdjson: Buffer + /** Files written under `resources/` โ€” typically screenshots + element snapshots. */ + resources: TraceZipResource[] +} + +export function buildTraceZip(inputs: TraceZipInputs): Promise { + return new Promise((resolve, reject) => { + const zipFile = new yazl.ZipFile() + zipFile.addBuffer(Buffer.from(inputs.traceNdjson, 'utf8'), 'trace.trace') + zipFile.addBuffer(inputs.networkNdjson, 'trace.network') + for (const resource of inputs.resources) { + zipFile.addBuffer(resource.data, `resources/${resource.resourceName}`) + } + zipFile.end() + const chunks: Buffer[] = [] + zipFile.outputStream.on('data', (chunk: Buffer) => chunks.push(chunk)) + zipFile.outputStream.on('end', () => resolve(Buffer.concat(chunks))) + zipFile.outputStream.on('error', reject) + }) +} diff --git a/packages/core/src/with-timeout.ts b/packages/core/src/with-timeout.ts new file mode 100644 index 00000000..feb840e2 --- /dev/null +++ b/packages/core/src/with-timeout.ts @@ -0,0 +1,26 @@ +// Resolves `promise` if it settles before `ms`; otherwise resolves to the +// supplied fallback value. Used by per-action snapshot capture to guard +// against hung in-page scripts (heavy pages, infinite-render loops) without +// stalling the user's test. + +export function withTimeout( + promise: Promise, + ms: number, + fallback: T +): Promise { + let timer: ReturnType | undefined + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(fallback), ms) + }) + // .finally on the race result clears the timer in BOTH the resolved AND + // rejected branch โ€” earlier code only cleared on fulfillment, leaving a + // dangling timer per failed probe that delayed process exit by up to `ms`. + return Promise.race([promise, timeout]).finally(() => { + if (timer) { + clearTimeout(timer) + } + }) +} + +/** Default ceiling for a single in-page snapshot probe. */ +export const SNAPSHOT_PROBE_TIMEOUT_MS = 2500 diff --git a/packages/elements/.npmignore b/packages/elements/.npmignore new file mode 100644 index 00000000..344b429f --- /dev/null +++ b/packages/elements/.npmignore @@ -0,0 +1,7 @@ +src +node_modules +tests +index.html +tsconfig.json +vite.config.ts +*.tgz diff --git a/packages/elements/ROADMAP.md b/packages/elements/ROADMAP.md new file mode 100644 index 00000000..f387fb4b --- /dev/null +++ b/packages/elements/ROADMAP.md @@ -0,0 +1,81 @@ +# @wdio/elements Roadmap + +## Current state (May 2026) + +The package delivers LLM-readable element snapshots for both web and mobile: + +| Capability | Web | Mobile | +|---|---|---| +| Interactable element list | `getInteractableBrowserElements()` | `getMobileVisibleElements()` | +| Semantic tree | `getBrowserAccessibilityTree()` | *(raw `JSONElement` only)* | +| Snapshot serialization | `serializeWebSnapshot()` | `serializeMobileSnapshot()` | +| Unified API | `getElements()` returns both | `getElements()` returns both | +| Viewport filtering | `inViewportOnly` (default true) | `inViewportOnly` (default true) | +| Role classification | Computed in-browser from tag/ARIA | `ANDROID_ROLE_MAP` / `IOS_ROLE_MAP` in snapshot.ts | +| Locator generation | CSS selectors in browser script | `getSuggestedLocators()` from locator-generation.ts | +| Context disambiguation | `โˆˆ` via `inferPurpose()` | `โˆˆ` via `mobileInferPurpose()` | +| Duplicate selector indexing | N/A (selectors are unique) | `.instance(N)` suffix | + +## Architectural concerns + +### 1. Two independent mobile pipelines + +`serializeMobileSnapshot` in `snapshot.ts` has its own copies of: + +- **Role classification** โ€” `ANDROID_ROLE_MAP` / `IOS_ROLE_MAP` duplicate logic from `locators/constants.ts` and `locators/element-filter.ts`. +- **Interactivity detection** โ€” `isMobileInteractive()` shadows `isInteractableElement()` from `element-filter.ts`. They use different criteria (tag-based vs attribute-based) and can disagree. +- **Locator generation** โ€” `getBestAndroidLocator()` / `getBestIOSLocator()` are simplified fallbacks. The full pipeline (`getSuggestedLocators()`) is now wired in when source XML is available, but the fallback still exists and the two paths can produce different selectors for the same element. + +These should be collapsed: `serializeMobileSnapshot` should consume pre-computed roles, interactivity flags, and selectors from the locator pipeline, not recompute them. + +### 2. No mobile equivalent of `getBrowserAccessibilityTree()` + +The web path returns a flat `AccessibilityNode[]` with roles, names, selectors, depths, and state. The mobile path returns a raw `JSONElement` tree โ€” the snapshot does all enrichment internally via `collectMobileNodes()` โ†’ `MobileFlatNode[]` (a private interface). There is no public function to get an enriched flat node list for mobile. + +**Proposal:** Extract `collectMobileNodes()` into a public `getMobileAccessibilityTree()` that returns `MobileFlatNode[]` (or a shared type). `serializeMobileSnapshot()` becomes a pure formatting pass โ€” like `serializeWebSnapshot()` already is. + +### 3. Layout noise in mobile snapshots + +The Android view hierarchy includes every layout container (`FrameLayout`, `LinearLayout`, `ViewGroup`, etc.). The current noise filter (`NOISY_ROLES`) collapses anonymous containers at depth โ‰ฅ 2, but named containers and depth 0-1 scaffolding still appear. The web a11y tree doesn't have this problem because the browser's accessibility computation already skips layout-only `
`s. + +**Proposal:** A `collapseContainers` option on the snapshot (default `true`) that skips any container without an interactive descendant. Alternatively, the tree collection pass could flag "informative" vs "structural" containers and let the renderer decide. + +### 4. Selector format for mobile + +Mobile selectors are Appium/WDIO-specific strings (`~Accessibility`, `android=new UiSelector()...`, `id:com.example:id/foo`). The web path outputs CSS selectors (`a*=Highlights`, `#cart-icon-bubble`). An LLM/agent needs different selector parsing logic per platform. There's no common selector abstraction. + +**Proposal:** A `SelectorString` type with platform-aware parsing, or at minimum consistent prefix conventions documented for LLM consumption. + +### 5. The raw tree doesn't carry locators unless processed + +`getMobileVisibleElementsWithTree()` returns `{ elements, tree }` where `tree` is the raw `xmlToJSON()` output. Locators are only on `elements` (from `generateAllElementLocators()`). The snapshot reads locators by running `getSuggestedLocators()` again (or falling back). If a consumer wants to annotate the tree themselves, they must re-run the locator pipeline. + +**Proposal:** Enrich the tree in-place during `generateAllElementLocators()` โ€” attach `_selector`, `_role`, and `_interactive` attributes to each `JSONElement` node that passes the filter. The raw tree becomes self-describing. + +## Improvement backlog + +| Priority | What | Effort | +|---|---|---| +| P0 | Merge `isMobileInteractive` + role classification into `generateAllElementLocators` โ€” one source of truth | Medium | +| P1 | Extract `getMobileAccessibilityTree()` as a public API returning enriched flat nodes | Medium | +| P1 | Enrich `JSONElement` tree nodes with locators during `generateAllElementLocators()` | Small | +| P2 | `collapseContainers` option on `serializeMobileSnapshot` | Small | +| P2 | Unify web + mobile serialization into a single `serializeSnapshot()` function | Large | +| P3 | Document selector format conventions for LLM consumption | Small | +| P3 | Add `checked`/`selected`/`expanded` state rendering to mobile snapshot (parity with web) | Small | + +## Verified capabilities + +- [x] Web: viewport-only snapshot with semantic roles and unique CSS selectors +- [x] Web: `โˆˆ` disambiguation for duplicate selectors (6 "Add to Wishlist" buttons โ†’ each with book title context) +- [x] Web: `statictext` role capturing visible text (book titles, promo copy, cookie text) +- [x] Web: deduplication of echoed text (child text already in parent name โ†’ skipped) +- [x] Mobile: semantic role mapping (TextViewโ†’statictext, ImageViewโ†’img, Buttonโ†’button, etc.) +- [x] Mobile: full-pipeline selectors via `getSuggestedLocators()` wired into snapshot +- [x] Mobile: `~` prefix for accessibility-id, `id:` for resource-id, `android=new UiSelector()...` for compound +- [x] Mobile: `.instance(N)` indexing for duplicate selectors +- [x] Mobile: explicit tap-target promotion (clickable parent carries `โ†’`, label children provide `โˆˆ` context) +- [x] Mobile: layout noise collapse for anonymous containers +- [x] Mobile: `โˆˆ` context from actual parent, not previous list-item sibling +- [x] Unified `getElements()` API returning `{ elements, tree }` for both platforms +- [x] `inViewportOnly` default `true` across all entry points with per-function toggles diff --git a/packages/elements/package.json b/packages/elements/package.json new file mode 100644 index 00000000..a334c3fb --- /dev/null +++ b/packages/elements/package.json @@ -0,0 +1,40 @@ +{ + "name": "@wdio/elements", + "version": "1.0.0", + "description": "Element detection scripts for WebdriverIO", + "author": "Vince Graics", + "license": "MIT", + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./locators": { + "types": "./dist/locators/index.d.ts", + "import": "./dist/locators/index.js" + } + }, + "types": "./dist/index.d.ts", + "repository": { + "type": "git", + "url": "git+https://github.com/webdriverio/devtools.git", + "directory": "packages/elements" + }, + "scripts": { + "build": "tsc -p ./tsconfig.json", + "lint": "eslint . --fix", + "test": "vitest run" + }, + "dependencies": {}, + "devDependencies": { + "@types/node": "25.5.2", + "@wdio/devtools-core": "workspace:^", + "@wdio/globals": "9.27.0", + "typescript": "6.0.2", + "vitest": "^4.0.16" + }, + "peerDependencies": { + "webdriverio": "^9.0.0" + } +} diff --git a/packages/elements/src/accessibility-tree.ts b/packages/elements/src/accessibility-tree.ts new file mode 100644 index 00000000..01c6a81f --- /dev/null +++ b/packages/elements/src/accessibility-tree.ts @@ -0,0 +1,27 @@ +/** + * Browser accessibility tree + * Single browser.execute() call: DOM walk โ†’ flat accessibility node list. + * + * The injected script lives in @wdio/devtools-core/element-scripts so it is + * the single source of truth for both the @wdio/elements wrappers and the + * framework-agnostic trace/snapshot pipeline. + */ + +import type { AccessibilityNode } from '@wdio/devtools-core/element-types' +import { accessibilityTreeScript as _accessibilityTreeScript } from '@wdio/devtools-core/element-scripts' + +export type { AccessibilityNode } + +/** + * Get browser accessibility tree via a single DOM walk. + */ +export async function getBrowserAccessibilityTree( + browser: WebdriverIO.Browser, + options: { inViewportOnly?: boolean } = {} +): Promise { + const { inViewportOnly = true } = options + const fn = new Function( + `return (${_accessibilityTreeScript(inViewportOnly)})` + ) as () => unknown + return browser.execute(fn) as unknown as Promise +} diff --git a/packages/elements/src/browser-elements.ts b/packages/elements/src/browser-elements.ts new file mode 100644 index 00000000..a08ed3bd --- /dev/null +++ b/packages/elements/src/browser-elements.ts @@ -0,0 +1,34 @@ +/** + * Browser element detection + * Single browser.execute() call: querySelectorAll โ†’ flat interactable element list. + * + * The injected script lives in @wdio/devtools-core/element-scripts so it is + * the single source of truth for both the @wdio/elements wrappers and the + * framework-agnostic trace/snapshot pipeline. + */ + +import type { + BrowserElementInfo, + GetBrowserElementsOptions +} from '@wdio/devtools-core/element-types' +import { elementsScript as _elementsScript } from '@wdio/devtools-core/element-scripts' + +export type { BrowserElementInfo, GetBrowserElementsOptions } + +/** + * Get interactable browser elements via querySelectorAll. + * + * The script body lives in core but is converted back to a function for + * WDIO's `browser.execute(fn, args)` serialization. Passing a raw string + * to execute() invokes a different code path that may not preserve scope. + */ +export async function getInteractableBrowserElements( + browser: WebdriverIO.Browser, + options: GetBrowserElementsOptions = {} +): Promise { + const { includeBounds = false, inViewportOnly = true } = options + const fn = new Function( + `return (${_elementsScript(includeBounds, inViewportOnly)})` + ) as () => unknown + return browser.execute(fn) as unknown as Promise +} diff --git a/packages/elements/src/get-elements.ts b/packages/elements/src/get-elements.ts new file mode 100644 index 00000000..4aedbf8b --- /dev/null +++ b/packages/elements/src/get-elements.ts @@ -0,0 +1,67 @@ +import { getInteractableBrowserElements } from './browser-elements.js' +import { getMobileVisibleElementsWithTree } from './mobile-elements.js' +import type { JSONElement } from './locators/index.js' + +export type VisibleElementsResult = { + total: number + showing: number + hasMore: boolean + elements: unknown[] + /** Raw JSON element tree โ€” only present for mobile (android/ios) sessions */ + tree?: JSONElement +} + +export async function getElements( + browser: WebdriverIO.Browser, + params: { + inViewportOnly?: boolean + includeContainers?: boolean + includeBounds?: boolean + limit?: number + offset?: number + } +): Promise { + const { + inViewportOnly = true, + includeContainers = false, + includeBounds = false, + limit = 0, + offset = 0 + } = params + + let elements: { isInViewport?: boolean }[] + let tree: JSONElement | undefined + + if (browser.isAndroid || browser.isIOS) { + const platform = browser.isAndroid ? 'android' : 'ios' + const result = await getMobileVisibleElementsWithTree(browser, platform, { + includeContainers, + includeBounds, + inViewportOnly + }) + elements = result.elements + tree = result.tree ?? undefined + } else { + elements = await getInteractableBrowserElements(browser, { + includeBounds, + inViewportOnly + }) + } + + const total = elements.length + + if (offset > 0) { + elements = elements.slice(offset) + } + if (limit > 0) { + elements = elements.slice(0, limit) + } + + return { + total, + showing: elements.length, + hasMore: offset + elements.length < total, + elements, + ...(tree !== undefined ? { tree } : {}) + } +} diff --git a/packages/elements/src/index.ts b/packages/elements/src/index.ts new file mode 100644 index 00000000..37e495bd --- /dev/null +++ b/packages/elements/src/index.ts @@ -0,0 +1,25 @@ +// WDIO-dependent element extraction wrappers. +// Framework-agnostic types, serializers, scripts, and locator generation live +// in @wdio/devtools-core and are re-exported here for backward compatibility. + +export { getInteractableBrowserElements } from './browser-elements.js' +export type { + BrowserElementInfo, + GetBrowserElementsOptions +} from './browser-elements.js' + +export { getBrowserAccessibilityTree } from './accessibility-tree.js' +export type { AccessibilityNode } from './accessibility-tree.js' + +export { getMobileVisibleElements } from './mobile-elements.js' +export type { + MobileElementInfo, + GetMobileElementsOptions +} from './mobile-elements.js' + +export { getElements } from './get-elements.js' +export type { VisibleElementsResult } from './get-elements.js' + +export { serializeWebSnapshot, serializeMobileSnapshot } from './snapshot.js' +export type { WebSnapshotOptions, MobileSnapshotOptions } from './snapshot.js' +export type { JSONElement } from './locators/index.js' diff --git a/packages/elements/src/locators/index.ts b/packages/elements/src/locators/index.ts new file mode 100644 index 00000000..09c12bc2 --- /dev/null +++ b/packages/elements/src/locators/index.ts @@ -0,0 +1,41 @@ +/** + * Mobile element locator generation โ€” re-exported from @wdio/devtools-core. + */ + +export type { + ElementAttributes, + JSONElement, + Bounds, + FilterOptions, + UniquenessResult, + LocatorStrategy, + LocatorContext, + ElementWithLocators, + GenerateLocatorsOptions +} from '@wdio/devtools-core/locators' + +export { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS, + ANDROID_LAYOUT_CONTAINERS, + IOS_LAYOUT_CONTAINERS, + xmlToJSON, + xmlToDOM, + evaluateXPath, + checkXPathUniqueness, + findDOMNodeByPath, + parseAndroidBounds, + parseIOSBounds, + flattenElementTree, + countAttributeOccurrences, + isAttributeUnique, + isInteractableElement, + isLayoutContainer, + hasMeaningfulContent, + shouldIncludeElement, + getDefaultFilters, + getSuggestedLocators, + getBestLocator, + locatorsToObject, + generateAllElementLocators +} from '@wdio/devtools-core/locators' diff --git a/packages/elements/src/mobile-elements.ts b/packages/elements/src/mobile-elements.ts new file mode 100644 index 00000000..1eabbc53 --- /dev/null +++ b/packages/elements/src/mobile-elements.ts @@ -0,0 +1,199 @@ +/** + * Mobile element detection utilities for iOS and Android + * + * Uses page source parsing for optimal performance (2 HTTP calls vs 600+ for 50 elements) + */ + +import type { + ElementWithLocators, + FilterOptions, + JSONElement, + LocatorStrategy +} from './locators/index.js' +import { + generateAllElementLocators, + getDefaultFilters, + xmlToJSON +} from './locators/index.js' + +/** + * Element info returned by getMobileVisibleElements + * Uses uniform fields (all elements have same keys) to enable TOON tabular format + */ +export interface MobileElementInfo { + selector: string + tagName: string + isInViewport: boolean + text: string + resourceId: string + accessibilityId: string + isEnabled: boolean + altSelector: string // Single alternative selector (flattened for tabular format) + // Only present when includeBounds=true + bounds?: { x: number; y: number; width: number; height: number } +} + +/** + * Options for getMobileVisibleElements + */ +export interface GetMobileElementsOptions { + includeContainers?: boolean + includeBounds?: boolean + /** Only return elements whose bounds intersect the viewport (default true). */ + inViewportOnly?: boolean + filterOptions?: FilterOptions +} + +/** + * Locator strategy priority order for selecting best selector + * Earlier = higher priority + */ +const LOCATOR_PRIORITY: LocatorStrategy[] = [ + 'accessibility-id', // Most stable, cross-platform + 'id', // Android resource-id + 'text', // Text-based (can be fragile but readable) + 'predicate-string', // iOS predicate + 'class-chain', // iOS class chain + 'uiautomator', // Android UiAutomator compound + 'xpath' // XPath (last resort, brittle) + // 'class-name' intentionally excluded - too generic +] + +/** + * Select best locators from available strategies + * Returns [primarySelector, ...alternativeSelectors] + */ +function selectBestLocators(locators: Record): string[] { + const selected: string[] = [] + + // Find primary selector based on priority + for (const strategy of LOCATOR_PRIORITY) { + if (locators[strategy]) { + selected.push(locators[strategy]) + break + } + } + + // Add one alternative if available (different strategy) + for (const strategy of LOCATOR_PRIORITY) { + if (locators[strategy] && !selected.includes(locators[strategy])) { + selected.push(locators[strategy]) + break + } + } + + return selected +} + +/** + * Convert ElementWithLocators to MobileElementInfo + * Uses uniform fields (all elements have same keys) to enable CSV tabular format + */ +function toMobileElementInfo( + element: ElementWithLocators, + includeBounds: boolean +): MobileElementInfo { + const selectedLocators = selectBestLocators(element.locators) + + // Use contentDesc for accessibilityId on Android, or name on iOS + const accessId = element.accessibilityId || element.contentDesc + + // Build object with ALL fields for uniform schema (enables CSV tabular format) + // Empty string '' used for missing values to keep schema consistent + const info: MobileElementInfo = { + selector: selectedLocators[0] || '', + tagName: element.tagName, + isInViewport: element.isInViewport, + text: element.text || '', + resourceId: element.resourceId || '', + accessibilityId: accessId || '', + isEnabled: element.enabled !== false, + altSelector: selectedLocators[1] || '' // Single alternative (flattened for tabular) + } + + // Only include bounds if explicitly requested (adds 4 extra columns) + if (includeBounds) { + info.bounds = element.bounds + } + + return info +} + +/** + * Get viewport size from browser + */ +async function getViewportSize( + browser: WebdriverIO.Browser +): Promise<{ width: number; height: number }> { + try { + const size = await browser.getWindowSize() + return { width: size.width, height: size.height } + } catch { + return { width: 9999, height: 9999 } + } +} + +/** + * Get all visible elements from a mobile app, also returning the raw JSON element tree. + * Single parse of page source: tree and flat list share one xmlToJSON call. + * + * Performance: 2 HTTP calls (getWindowSize + getPageSource) vs 12+ per element with legacy approach + */ +export async function getMobileVisibleElementsWithTree( + browser: WebdriverIO.Browser, + platform: 'ios' | 'android', + options: GetMobileElementsOptions = {} +): Promise<{ elements: MobileElementInfo[]; tree: JSONElement | null }> { + const { + includeContainers = false, + includeBounds = false, + inViewportOnly = true, + filterOptions + } = options + + const viewportSize = await getViewportSize(browser) + const pageSource = await browser.getPageSource() + + const filters: FilterOptions = { + ...getDefaultFilters(platform, includeContainers), + ...filterOptions + } + + const tree = xmlToJSON(pageSource) + + // Stash the source XML on the root element so serializeMobileSnapshot + // can use the full locator pipeline without requiring it as a separate arg. + if (tree) { + tree.attributes._sourceXML = pageSource + } + + const elementLocators = generateAllElementLocators(pageSource, { + platform, + viewportSize, + filters, + inViewportOnly + }) + + const elements = elementLocators.map((el) => + toMobileElementInfo(el, includeBounds) + ) + return { elements, tree } +} + +/** + * Get all visible elements from a mobile app + * + * Performance: 2 HTTP calls (getWindowSize + getPageSource) vs 12+ per element with legacy approach + */ +export async function getMobileVisibleElements( + browser: WebdriverIO.Browser, + platform: 'ios' | 'android', + options: GetMobileElementsOptions = {} +): Promise { + const { elements } = await getMobileVisibleElementsWithTree( + browser, + platform, + options + ) + return elements +} diff --git a/packages/elements/src/snapshot.ts b/packages/elements/src/snapshot.ts new file mode 100644 index 00000000..ed83ae34 --- /dev/null +++ b/packages/elements/src/snapshot.ts @@ -0,0 +1,12 @@ +/** + * AI-readable snapshot serializers โ€” re-exported from @wdio/devtools-core. + */ + +export { + serializeWebSnapshot, + serializeMobileSnapshot +} from '@wdio/devtools-core/element-snapshot' +export type { + WebSnapshotOptions, + MobileSnapshotOptions +} from '@wdio/devtools-core/element-snapshot' diff --git a/packages/elements/tests/accessibility-tree.test.ts b/packages/elements/tests/accessibility-tree.test.ts new file mode 100644 index 00000000..6fd7be13 --- /dev/null +++ b/packages/elements/tests/accessibility-tree.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect, vi } from 'vitest' +import { getBrowserAccessibilityTree } from '../src/accessibility-tree.js' + +describe('getBrowserAccessibilityTree', () => { + it('calls browser.execute and returns result', async () => { + const nodes = [ + { + role: 'button', + name: 'Submit', + selector: 'button*=Submit', + depth: 0, + level: '', + disabled: '', + checked: '', + expanded: '', + selected: '', + pressed: '', + required: '', + readonly: '' + } + ] + const mockBrowser = { execute: vi.fn().mockResolvedValue(nodes) } as any + const result = await getBrowserAccessibilityTree(mockBrowser) + expect(mockBrowser.execute).toHaveBeenCalledTimes(1) + expect(result).toEqual(nodes) + }) +}) diff --git a/packages/elements/tests/browser-elements.test.ts b/packages/elements/tests/browser-elements.test.ts new file mode 100644 index 00000000..6fba3af5 --- /dev/null +++ b/packages/elements/tests/browser-elements.test.ts @@ -0,0 +1,22 @@ +import { describe, it, expect, vi } from 'vitest' +import { getInteractableBrowserElements } from '../src/browser-elements.js' + +describe('getInteractableBrowserElements', () => { + it('calls browser.execute with includeBounds=false by default', async () => { + const mockBrowser = { execute: vi.fn().mockResolvedValue([]) } as any + const result = await getInteractableBrowserElements(mockBrowser) + expect(mockBrowser.execute).toHaveBeenCalledTimes(1) + expect(result).toEqual([]) + }) + + it('passes includeBounds option to script', async () => { + const mockBrowser = { + execute: vi.fn().mockResolvedValue([{ tagName: 'button', name: 'OK' }]) + } as any + const result = await getInteractableBrowserElements(mockBrowser, { + includeBounds: true + }) + expect(result).toHaveLength(1) + expect(result[0].tagName).toBe('button') + }) +}) diff --git a/packages/elements/tests/locators/locator-generation.test.ts b/packages/elements/tests/locators/locator-generation.test.ts new file mode 100644 index 00000000..c5d2de4b --- /dev/null +++ b/packages/elements/tests/locators/locator-generation.test.ts @@ -0,0 +1,23 @@ +import { describe, it, expect } from 'vitest' +import { locatorsToObject } from '@wdio/elements/locators' + +describe('locatorsToObject', () => { + it('converts locator array to object', () => { + const locators: [any, string][] = [ + ['accessibility-id', '~Submit'], + ['xpath', '//XCUIElementTypeButton[@name="Submit"]'] + ] + const result = locatorsToObject(locators) + expect(result['accessibility-id']).toBe('~Submit') + expect(result['xpath']).toBe('//XCUIElementTypeButton[@name="Submit"]') + }) + + it('returns first value for duplicate strategies', () => { + const locators: [any, string][] = [ + ['xpath', '//first'], + ['xpath', '//second'] + ] + const result = locatorsToObject(locators) + expect(result['xpath']).toBe('//first') + }) +}) diff --git a/packages/elements/tests/mobile-elements.test.ts b/packages/elements/tests/mobile-elements.test.ts new file mode 100644 index 00000000..64f19861 --- /dev/null +++ b/packages/elements/tests/mobile-elements.test.ts @@ -0,0 +1,13 @@ +import { describe, it, expect, vi } from 'vitest' +import { getMobileVisibleElements } from '../src/mobile-elements.js' + +describe('getMobileVisibleElements', () => { + it('returns empty array for unparseable XML', async () => { + const mockBrowser = { + getWindowSize: vi.fn().mockResolvedValue({ width: 375, height: 812 }), + getPageSource: vi.fn().mockResolvedValue(' & { role: string; depth: number } +): AccessibilityNode { + return { + name: '', + selector: '', + level: '', + disabled: '', + checked: '', + expanded: '', + selected: '', + pressed: '', + required: '', + readonly: '', + ...overrides + } +} + +describe('serializeWebSnapshot', () => { + it('produces a page header', () => { + const out = serializeWebSnapshot([]) + expect(out).toBe('[Page]') + }) + + it('includes title and url in header', () => { + const out = serializeWebSnapshot([], { + title: 'Login', + url: 'https://example.com/login' + }) + expect(out).toMatch('[Page: Login โ€” https://example.com/login]') + }) + + it('renders interactive role with name and selector', () => { + const nodes = [ + node({ + role: 'button', + depth: 0, + name: 'Submit', + selector: 'button*=Submit' + }) + ] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('button "Submit" โ†’ button*=Submit') + }) + + it('renders interactive role with โˆˆ ancestor name when self has no name', () => { + const nodes = [ + node({ role: 'form', depth: 0, name: 'Login form' }), + node({ role: 'checkbox', depth: 1, name: '', selector: '#remember' }) + ] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('checkbox โˆˆ "Login form" โ†’ #remember') + }) + + it('omits interactive node with no selector regardless of name', () => { + const nodes = [ + node({ role: 'button', depth: 0, name: '', selector: '' }), + node({ + role: 'button', + depth: 0, + name: 'Named but unselector', + selector: '' + }) + ] + const out = serializeWebSnapshot(nodes) + // Only the header โ€” both nodes skipped due to missing selector + expect(out.split('\n').length).toBe(1) + }) + + it('omits interactive node with โˆˆ context but no selector', () => { + const nodes = [ + node({ role: 'form', depth: 0, name: 'Login form' }), + node({ role: 'combobox', depth: 1, name: '', selector: '' }) + ] + const out = serializeWebSnapshot(nodes) + // combobox has ancestor context but no selector โ€” must be dropped + expect(out).not.toContain('combobox') + expect(out).not.toContain('โ†’') + }) + + it('renders container role without selector', () => { + const nodes = [node({ role: 'navigation', depth: 0, name: 'Main' })] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('navigation "Main"') + expect(out).not.toContain('โ†’') + }) + + it('renders heading with level suffix', () => { + const nodes = [ + node({ role: 'heading', depth: 0, name: 'Sign in', level: 1 }) + ] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('heading[1] "Sign in"') + }) + + it('indents nodes according to depth', () => { + const nodes = [ + node({ role: 'navigation', depth: 0, name: 'Nav' }), + node({ role: 'link', depth: 1, name: 'Home', selector: 'a*=Home' }) + ] + const lines = serializeWebSnapshot(nodes).split('\n') + // depth 0 โ†’ 1 level of indent (' ' ร— 1), depth 1 โ†’ 2 levels (' ' ร— 2) + expect(lines[1]).toMatch(/^ navigation/) + expect(lines[2]).toMatch(/^ link/) + }) + + it('renders full login page example correctly', () => { + const nodes: AccessibilityNode[] = [ + node({ role: 'navigation', depth: 0, name: 'Main' }), + node({ role: 'link', depth: 1, name: 'Home', selector: 'a*=Home' }), + node({ role: 'main', depth: 0, name: '' }), + node({ role: 'heading', depth: 1, name: 'Sign in', level: 1 }), + node({ role: 'form', depth: 1, name: 'Login' }), + node({ + role: 'textbox', + depth: 2, + name: 'Email address', + selector: '#email' + }), + node({ + role: 'button', + depth: 2, + name: 'Sign in', + selector: 'button*=Sign in' + }) + ] + const out = serializeWebSnapshot(nodes, { + title: 'Login', + url: 'https://example.com/login' + }) + expect(out).toContain('[Page: Login โ€” https://example.com/login]') + expect(out).toContain('navigation "Main"') + expect(out).toContain('link "Home" โˆˆ "Main" โ†’ a*=Home') + expect(out).toContain('heading[1] "Sign in"') + expect(out).toContain('textbox "Email address" โˆˆ "Login" โ†’ #email') + expect(out).toContain('button "Sign in" โˆˆ "Login" โ†’ button*=Sign in') + }) +}) + +// --------------------------------------------------------------------------- +// serializeMobileSnapshot +// --------------------------------------------------------------------------- + +function mobileEl( + tagName: string, + attrs: JSONElement['attributes'], + children: JSONElement[] = [] +): JSONElement { + return { tagName, attributes: attrs, children, path: '' } +} + +describe('serializeMobileSnapshot', () => { + it('produces a platform header with device and viewport', () => { + const root = mobileEl('hierarchy', {}) + const out = serializeMobileSnapshot(root, { + platform: 'android', + deviceName: 'Pixel 7', + viewport: { width: 412, height: 915 } + }) + expect(out).toMatch('[android โ€” Pixel 7 (412ร—915)]') + }) + + it('renders interactive Android element with accessibility-id locator', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl('android.widget.Button', { + clickable: 'true', + 'content-desc': 'Skip', + text: '' + }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain('button "Skip" โ†’ ~Skip') + }) + + it('falls back to resource-id when no content-desc', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl('android.widget.EditText', { + clickable: 'true', + 'content-desc': '', + 'resource-id': 'com.example:id/search', + text: '' + }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain('textbox "search" โ†’ id:com.example:id/search') + }) + + it('renders โˆˆ ancestor context when element has no identity', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl( + 'android.widget.LinearLayout', + { 'content-desc': 'Search section' }, + [ + mobileEl('android.widget.EditText', { + clickable: 'true', + 'content-desc': '', + 'resource-id': 'com.example:id/search', + text: '' + }) + ] + ) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain( + 'textbox "search" โˆˆ "Search section" โ†’ id:com.example:id/search' + ) + }) + + it('renders iOS element with accessibility-id', () => { + const root = mobileEl('XCUIElementTypeApplication', {}, [ + mobileEl('XCUIElementTypeButton', { + accessible: 'true', + name: 'Accept All Cookies', + label: 'Accept All Cookies' + }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'ios' }) + expect(out).toContain('button "Accept All Cookies" โ†’ ~Accept All Cookies') + }) + + it('simplifies iOS XCUIElementType prefix', () => { + const root = mobileEl('XCUIElementTypeApplication', {}, [ + mobileEl('XCUIElementTypeScrollView', {}) + ]) + const out = serializeMobileSnapshot(root, { platform: 'ios' }) + expect(out).toContain('ScrollView') + expect(out).not.toContain('XCUIElementType') + }) + + it('shows container without selector', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl('android.widget.FrameLayout', { 'content-desc': '' }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain('FrameLayout') + expect(out).not.toContain('โ†’') + }) +}) diff --git a/packages/elements/tsconfig.json b/packages/elements/tsconfig.json new file mode 100644 index 00000000..3bcc9638 --- /dev/null +++ b/packages/elements/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "ignoreDeprecations": "6.0", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "noEmit": false, + "allowImportingTsExtensions": false, + "declaration": true, + "skipLibCheck": true, + "types": ["node", "@wdio/globals/types"] + }, + "include": ["src/**/*"] +} diff --git a/packages/nightwatch-devtools/README.md b/packages/nightwatch-devtools/README.md index 00da385d..5f404f5d 100644 --- a/packages/nightwatch-devtools/README.md +++ b/packages/nightwatch-devtools/README.md @@ -82,8 +82,9 @@ module.exports = { |--------|------|---------|-------------| | `port` | `number` | `3000` | Port for the DevTools backend server. Auto-incremented if already in use. | | `hostname` | `string` | `'localhost'` | Hostname the backend server binds to. | -| `screencast` | `ScreencastOptions` | `{ enabled: false }` | Session video recording (see [Screencast](#screencast)). | +| `screencast` | `ScreencastOptions` | `{ enabled: false }` | Session video recording โ€” live mode only (see [Screencast](#screencast)). | | `bidi` | `boolean` | `false` | Opt into WebDriver BiDi capture for browser console + JS exceptions + network. Requires `webSocketUrl: true` in your capabilities and a BiDi-capable chromedriver. When attached, the per-command Chrome perf-log network path is gated off so requests don't duplicate. | +| `mode` | `'live' \| 'trace'` | `'live'` | `'live'` opens the DevTools UI window; `'trace'` skips the UI and writes a `trace-.zip` next to your `nightwatch.conf.cjs` at run end. See [Trace mode](../../README.md#-trace-mode-tracezip). | ```javascript globals: nightwatchDevtools({ diff --git a/packages/nightwatch-devtools/package.json b/packages/nightwatch-devtools/package.json index 0a92afb0..c8ad4f92 100644 --- a/packages/nightwatch-devtools/package.json +++ b/packages/nightwatch-devtools/package.json @@ -48,7 +48,8 @@ "import-meta-resolve": "^4.2.0", "stacktrace-parser": "^0.1.11", "webdriverio": "^9.27.2", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" }, "devDependencies": { "@types/node": "25.9.1", diff --git a/packages/nightwatch-devtools/src/action-snapshot.ts b/packages/nightwatch-devtools/src/action-snapshot.ts new file mode 100644 index 00000000..6d1fc8c1 --- /dev/null +++ b/packages/nightwatch-devtools/src/action-snapshot.ts @@ -0,0 +1,31 @@ +// Nightwatch adapter: wires NightwatchBrowser into core's captureActionSnapshot. + +import { captureActionSnapshot as coreCapture } from '@wdio/devtools-core' +import type { ActionSnapshot } from '@wdio/devtools-shared' +import type { NightwatchBrowser } from './types.js' + +interface BrowserWithUrl extends NightwatchBrowser { + getCurrentUrl?: () => Promise + getTitle?: () => Promise +} + +export function captureActionSnapshot( + browser: NightwatchBrowser, + command: string, + takeScreenshot?: () => Promise +): Promise { + const b = browser as BrowserWithUrl + return coreCapture({ + command, + runScript: (src) => browser.execute(`return (${src})`) as Promise, + takeScreenshot, + getUrl: () => + b.getCurrentUrl + ? b.getCurrentUrl().catch(() => undefined) + : Promise.resolve(undefined), + getTitle: () => + b.getTitle + ? b.getTitle().catch(() => undefined) + : Promise.resolve(undefined) + }) +} diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index 395e4a77..4c260678 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -6,7 +6,12 @@ */ import { fileURLToPath } from 'node:url' -import { errorMessage } from '@wdio/devtools-core' +import { + errorMessage, + resolveAdapterOutputDir, + writeTraceZip +} from '@wdio/devtools-core' +import { stop as stopBackend } from '@wdio/devtools-backend' import { REUSE_ENV, SCREENCAST_DEFAULTS } from '@wdio/devtools-shared' import logger from '@wdio/logger' import { @@ -98,16 +103,21 @@ class NightwatchDevToolsPlugin { #bidiAttachAttempted = false constructor(options: DevToolsOptions = {}) { + const mode = options.mode ?? 'live' + const ignore = mode === 'trace' && options.screencast?.enabled === true + if (ignore) { + log.warn('trace mode: ignoring screencast option (live-mode feature)') + } + const screencast = ignore ? {} : (options.screencast ?? {}) this.options = { port: options.port ?? 3000, hostname: options.hostname ?? 'localhost', - screencast: options.screencast ?? {}, - bidi: options.bidi ?? false - } - this.#screencastOptions = { - ...SCREENCAST_DEFAULTS, - ...(options.screencast ?? {}) + screencast, + bidi: options.bidi ?? false, + mode, + traceFormat: options.traceFormat ?? 'zip' } + this.#screencastOptions = { ...SCREENCAST_DEFAULTS, ...screencast } this.#bidiEnabled = options.bidi === true } @@ -132,6 +142,9 @@ class NightwatchDevToolsPlugin { get port() { return self.options.port }, + get mode() { + return self.options.mode + }, get screencastOptions() { return self.#screencastOptions }, @@ -441,6 +454,12 @@ class NightwatchDevToolsPlugin { try { await this.#finalizeAllSuites(browser) this.#logRunSummary() + if (this.options.mode === 'trace') { + await this.#writeTraceZipIfNeeded() + await this.sessionCapturer?.closeWebSocket() + await stopBackend() + return + } if (!this.#devtoolsBrowser) { // Reuse mode: force one final suites broadcast so the UI reflects the // actual outcome before the process exits. @@ -464,6 +483,33 @@ class NightwatchDevToolsPlugin { logRunSummary(this.#getInternals()) } + async #writeTraceZipIfNeeded(): Promise { + if (this.options.mode !== 'trace' || !this.sessionCapturer) { + return + } + const sessionId = this.sessionCapturer.metadata?.sessionId + if (!sessionId) { + return + } + try { + if (this.sessionCapturer.snapshotCaptures.length) { + await Promise.allSettled(this.sessionCapturer.snapshotCaptures) + } + const snapshots = this.sessionCapturer.actionSnapshots + const tracePath = await writeTraceZip(this.sessionCapturer, { + outputDir: resolveAdapterOutputDir({ + configPath: this.#configPath + }), + sessionId, + actionSnapshots: snapshots.length ? snapshots : undefined, + format: this.options.traceFormat + }) + log.info(`Trace saved to ${tracePath}`) + } catch (err) { + log.warn(`trace write failed: ${errorMessage(err)}`) + } + } + async #waitForDevtoolsBrowserClose(): Promise { await waitForDevtoolsBrowserClose(this.#getInternals()) } diff --git a/packages/nightwatch-devtools/src/plugin-internals.ts b/packages/nightwatch-devtools/src/plugin-internals.ts index 06c75f22..afce58b4 100644 --- a/packages/nightwatch-devtools/src/plugin-internals.ts +++ b/packages/nightwatch-devtools/src/plugin-internals.ts @@ -14,6 +14,7 @@ import type { TestManager } from './helpers/testManager.js' import type { SuiteManager } from './helpers/suiteManager.js' import type { BrowserProxy } from './helpers/browserProxy.js' import type { + DevToolsMode, NightwatchBrowser, ScreencastOptions, SuiteStats, @@ -22,9 +23,10 @@ import type { export interface PluginInternals { // Config + options - options: { hostname: string; port: number } + options: { hostname: string; port: number; mode?: DevToolsMode } readonly hostname: string readonly port: number + readonly mode: DevToolsMode readonly screencastOptions: ScreencastOptions readonly bidiEnabled: boolean diff --git a/packages/nightwatch-devtools/src/run-lifecycle.ts b/packages/nightwatch-devtools/src/run-lifecycle.ts index a6d7e246..4769a9b5 100644 --- a/packages/nightwatch-devtools/src/run-lifecycle.ts +++ b/packages/nightwatch-devtools/src/run-lifecycle.ts @@ -19,14 +19,18 @@ import type { SessionCapturer } from './session.js' import type { TestReporter } from './reporter.js' import type { SuiteManager } from './helpers/suiteManager.js' import type { TestManager } from './helpers/testManager.js' -import type { NightwatchBrowser, NightwatchCurrentTest } from './types.js' +import type { + DevToolsMode, + NightwatchBrowser, + NightwatchCurrentTest +} from './types.js' import { TIMING, PLUGIN_GLOBAL_KEY } from './constants.js' import { findFreePort, resolveNightwatchConfig } from './helpers/utils.js' const log = logger('@wdio/nightwatch-devtools:run-lifecycle') export interface RunLifecycleCtx { - options: { hostname: string; port: number } + options: { hostname: string; port: number; mode?: DevToolsMode } readonly testReporter: TestReporter | undefined readonly suiteManager: SuiteManager | undefined readonly testManager: TestManager @@ -97,11 +101,15 @@ export async function runPluginBefore(ctx: PluginBeforeCtx): Promise { ctx.options.port = port const url = `http://${ctx.options.hostname}:${ctx.options.port}` log.info(`โœ“ Backend started on port ${ctx.options.port}`) - log.info(` DevTools UI: ${url}`) - await ctx.openDevtoolsBrowserAt(url) - await new Promise((resolve) => - setTimeout(resolve, TIMING.UI_CONNECTION_WAIT) - ) + if (ctx.options.mode === 'trace') { + log.info('trace mode: backend started, skipping UI window launch') + } else { + log.info(` DevTools UI: ${url}`) + await ctx.openDevtoolsBrowserAt(url) + await new Promise((resolve) => + setTimeout(resolve, TIMING.UI_CONNECTION_WAIT) + ) + } ;(globalThis as Record)[PLUGIN_GLOBAL_KEY] = ctx.plugin } catch (err) { log.error(`Failed to start backend: ${errorMessage(err)}`) diff --git a/packages/nightwatch-devtools/src/session-init.ts b/packages/nightwatch-devtools/src/session-init.ts index 5b7b29c9..438132c8 100644 --- a/packages/nightwatch-devtools/src/session-init.ts +++ b/packages/nightwatch-devtools/src/session-init.ts @@ -25,6 +25,7 @@ import { SuiteManager } from './helpers/suiteManager.js' import { BrowserProxy } from './helpers/browserProxy.js' import { ScreencastRecorder } from './screencast.js' import type { + DevToolsMode, NightwatchBrowser, ScreencastOptions, SuiteStats @@ -37,6 +38,7 @@ export interface SessionInitCtx { readonly port: number readonly screencastOptions: ScreencastOptions readonly bidiEnabled: boolean + readonly mode: DevToolsMode sessionCapturer: SessionCapturer testReporter: TestReporter @@ -124,7 +126,7 @@ function broadcastSessionMetadata( ctx.srcFolders = Array.isArray(sf) ? sf : sf ? [sf] : [] } - ctx.sessionCapturer.sendUpstream('metadata', { + const metadata = { type: TraceType.Testrunner, capabilities, desiredCapabilities, @@ -133,7 +135,9 @@ function broadcastSessionMetadata( host: opts.webdriver?.host, options: ctx.buildMetadataOptions(), url: '' - }) + } + ctx.sessionCapturer.metadata = metadata + ctx.sessionCapturer.sendUpstream('metadata', metadata) const browserName = capabilities.browserName || desiredCapabilities.browserName || 'unknown' @@ -230,6 +234,7 @@ export async function ensureSessionInitialized( { port: ctx.port, hostname: ctx.hostname }, browser ) + ctx.sessionCapturer.traceMode = ctx.mode const connected = await ctx.sessionCapturer.waitForConnection(3000) if (!connected) { log.error('โŒ Worker WebSocket failed to connect!') diff --git a/packages/nightwatch-devtools/src/session.ts b/packages/nightwatch-devtools/src/session.ts index d316f95e..7fc70e42 100644 --- a/packages/nightwatch-devtools/src/session.ts +++ b/packages/nightwatch-devtools/src/session.ts @@ -10,6 +10,8 @@ import { serializeError, type LogSource } from '@wdio/devtools-core' +import { mapCommandToAction } from '@wdio/devtools-core' +import { captureActionSnapshot } from './action-snapshot.js' import { NAVIGATION_COMMANDS } from './constants.js' import { parseNetworkFromPerfLogs, @@ -22,7 +24,13 @@ import { type CapturedPerformancePayload, applyPerformanceData } from '@wdio/devtools-core' -import type { CommandLog, LogLevel, NightwatchBrowser } from './types.js' +import type { + ActionSnapshot, + CommandLog, + DevToolsMode, + LogLevel, + NightwatchBrowser +} from './types.js' const log = logger('@wdio/nightwatch-devtools:SessionCapturer') @@ -105,6 +113,11 @@ export class SessionCapturer extends SessionCapturerBase { // capture path skips when set, so we don't double-emit network requests. bidiActive = false + // Populated by captureCommand when mode === 'trace' (set by the plugin). + traceMode: DevToolsMode = 'live' + readonly actionSnapshots: ActionSnapshot[] = [] + readonly snapshotCaptures: Promise[] = [] + constructor( devtoolsOptions: { hostname?: string; port?: number } = {}, browser?: NightwatchBrowser @@ -176,6 +189,24 @@ export class SessionCapturer extends SessionCapturerBase { }) } + if ( + this.traceMode === 'trace' && + !error && + this.#browser && + mapCommandToAction(command) + ) { + const browser = this.#browser + this.snapshotCaptures.push( + captureActionSnapshot(browser, command, () => + this.takeScreenshotViaHttp(browser) + ).then((snap) => { + if (snap) { + this.actionSnapshots.push(snap) + } + }) + ) + } + return true } @@ -425,17 +456,11 @@ export class SessionCapturer extends SessionCapturerBase { // Capture network requests from Chrome performance logs await this.captureNetworkFromPerformanceLogs(browser) - // Also try the injected wdioTraceCollector script for XHR/fetch and mutations + // Also try the injected wdioTraceCollector script for XHR/fetch and mutations. + // Atomic check+read โ€” the inline `typeof === 'undefined' โ†’ null` guard is + // the only safe form; a separate existence check would race page navigation + // (the collector can disappear between the two round-trips). try { - const checkResult = await browser.execute( - 'return typeof window.wdioTraceCollector !== "undefined"' - ) - const collectorExists = unwrapDriverValue(checkResult) === true - - if (!collectorExists) { - return - } - const result = await browser.execute(` if (typeof window.wdioTraceCollector === 'undefined') { return null; diff --git a/packages/nightwatch-devtools/src/types.ts b/packages/nightwatch-devtools/src/types.ts index a2e2b8c0..c498b40d 100644 --- a/packages/nightwatch-devtools/src/types.ts +++ b/packages/nightwatch-devtools/src/types.ts @@ -2,8 +2,10 @@ export { TraceType, + type ActionSnapshot, type CommandLog, type ConsoleLog, + type DevToolsMode, type DocumentInfo, type LogLevel, type Metadata, @@ -14,10 +16,15 @@ export { type SuiteStats, type TestStats, type TestStatus, + type TraceFormat, type TraceLog } from '@wdio/devtools-shared' -import type { ScreencastOptions } from '@wdio/devtools-shared' +import type { + DevToolsMode, + ScreencastOptions, + TraceFormat +} from '@wdio/devtools-shared' export interface CommandStackFrame { command: string @@ -93,6 +100,11 @@ export interface DevToolsOptions { * entries. Defaults to `false` โ€” opt-in. */ bidi?: boolean + /** `live` (default) launches the DevTools UI; `trace` skips it. */ + mode?: DevToolsMode + /** Trace output layout โ€” `zip` (default) writes a single archive, + * `ndjson-directory` unpacks into `trace-/`. Only applies in trace mode. */ + traceFormat?: TraceFormat } export interface NightwatchBrowser { diff --git a/packages/nightwatch-devtools/tests/session.test.ts b/packages/nightwatch-devtools/tests/session.test.ts index 19822c97..1347ee57 100644 --- a/packages/nightwatch-devtools/tests/session.test.ts +++ b/packages/nightwatch-devtools/tests/session.test.ts @@ -1,6 +1,8 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { SessionCapturer } from '../src/session.js' -import type { NightwatchBrowser } from '../src/types.js' +import type { CommandLog, NightwatchBrowser } from '../src/types.js' + +type CommandLogWithId = CommandLog & { _id: number } function makeMockBrowser( overrides: Partial> = {} @@ -31,8 +33,8 @@ describe('SessionCapturer.captureCommand', () => { args: ['#btn'], result: { ok: true } }) - expect((cap.commandsLog[0] as { _id: number })._id).not.toBe( - (cap.commandsLog[1] as { _id: number })._id + expect((cap.commandsLog[0] as CommandLogWithId)._id).not.toBe( + (cap.commandsLog[1] as CommandLogWithId)._id ) }) @@ -83,7 +85,7 @@ describe('SessionCapturer.replaceCommand', () => { it('splices the old entry and reissues with a new _id', async () => { const cap = makeCapturer() await cap.captureCommand('click', ['#a'], undefined, undefined) - const oldId = (cap.commandsLog[0] as { _id: number })._id + const oldId = (cap.commandsLog[0] as CommandLogWithId)._id const oldTs = cap.commandsLog[0].timestamp const { entry, oldTimestamp } = cap.replaceCommand( oldId, @@ -94,7 +96,7 @@ describe('SessionCapturer.replaceCommand', () => { ) expect(oldTimestamp).toBe(oldTs) expect(cap.commandsLog).toHaveLength(1) - expect((cap.commandsLog[0] as { _id: number })._id).not.toBe(oldId) + expect((cap.commandsLog[0] as CommandLogWithId)._id).not.toBe(oldId) expect(entry.result).toEqual({ ok: true }) }) @@ -312,11 +314,8 @@ describe('SessionCapturer.captureTrace', () => { getLog: vi.fn(async () => []), execute: vi.fn(async () => { call++ - if (call === 1) { - // collector check - return { value: true } - } - // getTraceData + // Single atomic check+read โ€” the inline `typeof === 'undefined' โ†’ null` + // guard in the script body avoids the navigation TOCTOU race. return { value: { mutations: [ @@ -330,6 +329,6 @@ describe('SessionCapturer.captureTrace', () => { }) const cap = makeCapturer(browser) await cap.captureTrace(browser) - expect(call).toBe(2) + expect(call).toBe(1) }) }) diff --git a/packages/selenium-devtools/README.md b/packages/selenium-devtools/README.md index 7722139c..89d099a9 100644 --- a/packages/selenium-devtools/README.md +++ b/packages/selenium-devtools/README.md @@ -311,6 +311,15 @@ DevTools.configure({ captureScreenshots: false }) DevTools.configure({ rerunCommand: 'npm test -- --grep "{{testName}}"' }) ``` +#### `mode` โ€” live UI vs. headless trace.zip +**Default:** `'live'` (launches the dashboard window). Set to `'trace'` to skip the dashboard entirely and write a `trace-.zip` next to your test file at session end โ€” meant for CI / offline replay / agentic diffing. `screencast` is ignored in trace mode (live-mode-only feature). + +```javascript +DevTools.configure({ mode: 'trace' }) +``` + +See [Trace mode](../../README.md#-trace-mode-tracezip) in the root README for the trace.zip layout and consumer notes. + --- ## Common recipes diff --git a/packages/selenium-devtools/package.json b/packages/selenium-devtools/package.json index 0b118b7f..869c25d8 100644 --- a/packages/selenium-devtools/package.json +++ b/packages/selenium-devtools/package.json @@ -47,7 +47,8 @@ "@wdio/logger": "^9.18.0", "stacktrace-parser": "^0.1.11", "webdriverio": "^9.27.2", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" }, "optionalDependencies": { "fluent-ffmpeg": "^2.1.3" diff --git a/packages/selenium-devtools/src/action-snapshot.ts b/packages/selenium-devtools/src/action-snapshot.ts new file mode 100644 index 00000000..7644ae1d --- /dev/null +++ b/packages/selenium-devtools/src/action-snapshot.ts @@ -0,0 +1,33 @@ +// Selenium adapter: wires SeleniumDriverLike into core's captureActionSnapshot. + +import { captureActionSnapshot as coreCapture } from '@wdio/devtools-core' +import type { ActionSnapshot } from '@wdio/devtools-shared' +import type { SeleniumDriverLike } from './types.js' + +interface DriverWithUrl extends SeleniumDriverLike { + getCurrentUrl?: () => Promise + getTitle?: () => Promise +} + +export function captureActionSnapshot( + driver: SeleniumDriverLike, + command: string +): Promise { + const d = driver as DriverWithUrl + return coreCapture({ + command, + runScript: (src) => driver.executeScript(`return (${src})`), + takeScreenshot: () => + d.takeScreenshot + ? d.takeScreenshot().catch(() => undefined) + : Promise.resolve(undefined), + getUrl: () => + d.getCurrentUrl + ? d.getCurrentUrl().catch(() => undefined) + : Promise.resolve(undefined), + getTitle: () => + d.getTitle + ? d.getTitle().catch(() => undefined) + : Promise.resolve(undefined) + }) +} diff --git a/packages/selenium-devtools/src/helpers/commandPostActions.ts b/packages/selenium-devtools/src/helpers/commandPostActions.ts index d90b9880..ad90666c 100644 --- a/packages/selenium-devtools/src/helpers/commandPostActions.ts +++ b/packages/selenium-devtools/src/helpers/commandPostActions.ts @@ -3,17 +3,21 @@ import { CAPTURE_PERFORMANCE_SCRIPT, applyPerformanceData, errorMessage, + mapCommandToAction, toError, type CapturedPerformancePayload, type RetryTracker } from '@wdio/devtools-core' import { getDriverOriginals, getElementOriginals } from '../driverPatcher.js' import { captureOrReplaceCommand } from './captureOrReplaceCommand.js' +import { captureActionSnapshot } from '../action-snapshot.js' import type { SessionCapturer } from '../session.js' import type { TestManager } from './testManager.js' import type { + ActionSnapshot, CapturedCommand, CommandLog, + DevToolsMode, SeleniumDriverLike } from '../types.js' @@ -144,10 +148,12 @@ export interface OnCommandCtx { readonly sessionCapturer: SessionCapturer | undefined readonly testManager: TestManager | undefined readonly retryTracker: RetryTracker - readonly options: { captureScreenshots: boolean } + readonly options: { captureScreenshots: boolean; mode?: DevToolsMode } readonly scriptInjected: boolean readonly finalized: boolean readonly driver: SeleniumDriverLike | undefined + readonly actionSnapshots: ActionSnapshot[] + readonly snapshotCaptures: Promise[] setScriptInjected(v: boolean): void } @@ -214,4 +220,19 @@ export async function handleOnCommand( ctx.driver ) } + if ( + ctx.options.mode === 'trace' && + !error && + ctx.driver && + mapCommandToAction(cmd.command) + ) { + const driver = ctx.driver + ctx.snapshotCaptures.push( + captureActionSnapshot(driver, cmd.command).then((snap) => { + if (snap) { + ctx.actionSnapshots.push(snap) + } + }) + ) + } } diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index 0e95b7de..127d69b4 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -1,7 +1,6 @@ // @wdio/selenium-devtools โ€” runner-agnostic Selenium WebDriver adapter. // Side-effect import that patches selenium-webdriver and starts the backend. -// MUST be the first import โ€” see setupConsole.ts. import './setupConsole.js' import logger from '@wdio/logger' import { startDetachedBackend } from './helpers/detachedBackend.js' @@ -49,7 +48,9 @@ import { NAVIGATION_COMMANDS } from './constants.js' import { + type ActionSnapshot, type CapturedCommand, + type DevToolsMode, type DevToolsOptions, type ScreencastOptions, type SeleniumDriverLike, @@ -86,6 +87,8 @@ class SeleniumDevToolsPlugin { #retryTracker = new RetryTracker() #screencast?: ScreencastRecorder #screencastOptions: ScreencastOptions + #actionSnapshots: ActionSnapshot[] = [] + #snapshotCaptures: Promise[] = [] #sessionId?: string #uiUrlOpened = false #testFilePath?: string @@ -116,11 +119,12 @@ class SeleniumDevToolsPlugin { this.#options = { port: options.port ?? 3000, hostname: options.hostname ?? 'localhost', - // Default true to match @wdio/devtools-service and @wdio/nightwatch-devtools. openUi: options.openUi ?? true, captureScreenshots: options.captureScreenshots ?? true, rerunCommand: options.rerunCommand, - headless: options.headless ?? false + headless: options.headless ?? false, + mode: options.mode ?? 'live', + traceFormat: options.traceFormat ?? 'zip' } this.#rerunManager = new RerunManager(RUNNER) if (options.rerunCommand) { @@ -187,8 +191,8 @@ class SeleniumDevToolsPlugin { ) } this.#backendStarted = true - // Skip when in REUSE mode โ€” the rerun child reuses the parent's window. - if (this.#options.openUi && !this.#isReuse) { + const { mode, openUi } = this.#options + if (mode !== 'trace' && openUi && !this.#isReuse) { this.#openUiWindow() } } catch (err) { @@ -229,24 +233,32 @@ class SeleniumDevToolsPlugin { screencast?: ScreencastOptions headless?: boolean openUi?: boolean + mode?: DevToolsMode } = {} ) { if ('rerunCommand' in opts) { this.#rerunManager.configure(opts.rerunCommand) this.#options.rerunCommand = opts.rerunCommand } - if (opts.screencast) { - this.#screencastOptions = { - ...this.#screencastOptions, - ...opts.screencast - } - } if (typeof opts.headless === 'boolean') { this.#options.headless = opts.headless } if (typeof opts.openUi === 'boolean') { this.#options.openUi = opts.openUi } + if (opts.mode) { + this.#options.mode = opts.mode + } + if (opts.screencast) { + if (this.#options.mode === 'trace' && opts.screencast.enabled) { + log.warn('trace mode: ignoring screencast option (live-mode feature)') + } else { + this.#screencastOptions = { + ...this.#screencastOptions, + ...opts.screencast + } + } + } } get options() { @@ -367,6 +379,12 @@ class SeleniumDevToolsPlugin { setScriptInjected: (v) => { self.#scriptInjected = v }, + get actionSnapshots() { + return self.#actionSnapshots + }, + get snapshotCaptures() { + return self.#snapshotCaptures + }, ensureBackendStarted: () => self.ensureBackendStarted(), flushPendingTestActions: () => self.#flushPendingTestActions(), resetRetryTracker: () => self.#retryTracker.reset(), @@ -565,7 +583,13 @@ if (!registerHooks()) { registerProcessHooks(plugin) export const DevTools = { - configure: (opts: { rerunCommand?: string }) => plugin.configure(opts), + configure: (opts: { + rerunCommand?: string + screencast?: ScreencastOptions + headless?: boolean + openUi?: boolean + mode?: DevToolsMode + }) => plugin.configure(opts), startTest: (name: string, meta?: { file?: string }) => plugin.startTest(name, meta), endTest: (state: TestStats['state'] = 'passed') => plugin.endTest(state) diff --git a/packages/selenium-devtools/src/plugin-internals.ts b/packages/selenium-devtools/src/plugin-internals.ts index 9484d07b..3c742fdf 100644 --- a/packages/selenium-devtools/src/plugin-internals.ts +++ b/packages/selenium-devtools/src/plugin-internals.ts @@ -12,7 +12,13 @@ import type { TestReporter } from './reporter.js' import type { SuiteManager } from './helpers/suiteManager.js' import type { TestManager } from './helpers/testManager.js' import type { ScreencastRecorder } from './screencast.js' -import type { ScreencastOptions, SeleniumDriverLike } from './types.js' +import type { + ActionSnapshot, + DevToolsMode, + ScreencastOptions, + SeleniumDriverLike, + TraceFormat +} from './types.js' import type { RetryTracker } from '@wdio/devtools-core' import type { PendingTestAction, PendingScenario } from './test-management.js' @@ -24,6 +30,8 @@ export interface PluginInternals { openUi: boolean captureScreenshots: boolean rerunCommand?: string + mode?: DevToolsMode + traceFormat?: TraceFormat } readonly screencastOptions: ScreencastOptions readonly runner: string @@ -49,6 +57,10 @@ export interface PluginInternals { pendingTestActions: PendingTestAction[] pendingScenario: PendingScenario | null + // trace-mode snapshot accumulators (mode === 'trace' only) + readonly actionSnapshots: ActionSnapshot[] + readonly snapshotCaptures: Promise[] + // Plugin-side delegates setFinalized(v: boolean): void setScriptInjected(v: boolean): void diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index 59c823e5..87059f54 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -12,7 +12,8 @@ import logger from '@wdio/logger' import { errorMessage, finalizeScreencast, - resolveAdapterOutputDir + resolveAdapterOutputDir, + writeTraceZip } from '@wdio/devtools-core' import { TIMING } from './constants.js' import { SessionCapturer } from './session.js' @@ -22,7 +23,14 @@ import { ScreencastRecorder } from './screencast.js' import { buildDriverMetadata } from './helpers/driverMetadata.js' import { attachBidiHandlers, buildBidiSinks } from './bidi.js' import { gracefulShutdown } from './helpers/processHooks.js' -import type { ScreencastOptions, SeleniumDriverLike } from './types.js' +import type { + ActionSnapshot, + DevToolsMode, + Metadata, + ScreencastOptions, + SeleniumDriverLike, + TraceFormat +} from './types.js' import type { TestManager } from './helpers/testManager.js' const log = logger('@wdio/selenium-devtools:session-lifecycle') @@ -34,6 +42,8 @@ export interface SessionLifecycleCtx { openUi: boolean captureScreenshots: boolean rerunCommand?: string + mode?: DevToolsMode + traceFormat?: TraceFormat } readonly screencastOptions: ScreencastOptions readonly runner: string @@ -53,6 +63,10 @@ export interface SessionLifecycleCtx { testFilePath: string | undefined keepAliveTimer: ReturnType | undefined + // Populated by handleOnCommand when mode === 'trace'. + readonly actionSnapshots: ActionSnapshot[] + readonly snapshotCaptures: Promise[] + setFinalized(v: boolean): void ensureBackendStarted(): Promise flushPendingTestActions(): void @@ -136,6 +150,10 @@ async function initPerDriverCapture( }) ctx.sessionId = sessionId if (metadata) { + // buildDriverMetadata returns a Record-shaped payload; the relevant + // Metadata fields (sessionId, capabilities, viewport, ...) are present + // at runtime but TS can't prove the discriminant `type`. + ctx.sessionCapturer.metadata = metadata as unknown as Metadata ctx.sessionCapturer.sendUpstream('metadata', metadata) } @@ -198,6 +216,9 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { } ctx.setFinalized(true) const shutdownStart = Date.now() + // Capture for the trace.zip write before onDriverEnd clears ctx state. + const capturerAtStart = ctx.sessionCapturer + const testFilePathAtStart = ctx.testFilePath try { await onDriverEnd(ctx).catch(() => {}) @@ -211,14 +232,29 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { ctx.testManager?.finalizeSession() ctx.testReporter?.updateSuites() + await maybeWriteTrace(ctx, capturerAtStart, testFilePathAtStart) + logSessionSummary(ctx) ctx.sessionCapturer?.cleanup() - if (ctx.options.openUi && !ctx.isReuse) { + if (ctx.options.openUi && ctx.options.mode !== 'trace' && !ctx.isReuse) { handleInteractivePath(ctx, shutdownStart) return } + // trace mode: no UI to wait for; close the WS so the backend can wind + // down naturally. process.exit is avoided โ€” Jest/runners may treat + // forced exits as failures. + if (ctx.options.mode === 'trace' && !ctx.isReuse) { + try { + await ctx.sessionCapturer?.closeWebSocket() + } catch { + /* best-effort */ + } + log.info(`๐Ÿ›‘ Shutdown complete (${Date.now() - shutdownStart}ms)`) + return + } + // Non-interactive path (no dashboard or rerun child). Don't close the // WS yet: this `onSessionEnd` is reached via the patched `driver.quit()` // (cucumber's per-scenario `After` hook), but the runner's @@ -231,6 +267,33 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { } } +async function maybeWriteTrace( + ctx: SessionLifecycleCtx, + capturer: SessionCapturer | undefined, + testFilePath: string | undefined +): Promise { + const sessionId = capturer?.metadata?.sessionId + if (ctx.options.mode !== 'trace' || !capturer || !sessionId) { + return + } + try { + if (ctx.snapshotCaptures.length) { + await Promise.allSettled(ctx.snapshotCaptures) + } + const tracePath = await writeTraceZip(capturer, { + outputDir: resolveAdapterOutputDir({ testFilePath }), + sessionId, + actionSnapshots: ctx.actionSnapshots.length + ? ctx.actionSnapshots + : undefined, + format: ctx.options.traceFormat + }) + log.info(`Trace saved to ${tracePath}`) + } catch (err) { + log.warn(`trace write failed: ${errorMessage(err)}`) + } +} + function logSessionSummary(ctx: SessionLifecycleCtx): void { const cmdCount = ctx.sessionCapturer?.commandsLog.length ?? 0 const consoleCount = ctx.sessionCapturer?.consoleLogs.length ?? 0 diff --git a/packages/selenium-devtools/src/session.ts b/packages/selenium-devtools/src/session.ts index 2312770e..70cbe98c 100644 --- a/packages/selenium-devtools/src/session.ts +++ b/packages/selenium-devtools/src/session.ts @@ -256,16 +256,13 @@ export class SessionCapturer extends SessionCapturerBase { return } try { - const ready = await exec( - driver, - 'return typeof window.wdioTraceCollector !== "undefined";' - ) - if (ready !== true) { - return - } + // Atomic check+read in one script eval so the collector can't disappear + // (page navigation) between the existence check and the getTraceData + // call. Two round-trips left a TOCTOU race that logged spurious + // "Cannot read properties of undefined (reading 'getTraceData')" errors. const traceData = await exec( driver, - 'return window.wdioTraceCollector.getTraceData();' + 'return typeof window.wdioTraceCollector !== "undefined" ? window.wdioTraceCollector.getTraceData() : null;' ) if (!traceData) { return diff --git a/packages/selenium-devtools/src/types.ts b/packages/selenium-devtools/src/types.ts index d239a308..dfcd4a0d 100644 --- a/packages/selenium-devtools/src/types.ts +++ b/packages/selenium-devtools/src/types.ts @@ -2,8 +2,10 @@ export { TraceType, + type ActionSnapshot, type CommandLog, type ConsoleLog, + type DevToolsMode, type DocumentInfo, type LogLevel, type Metadata, @@ -11,7 +13,8 @@ export { type PerformanceData, type SuiteStats, type TestStats, - type TestStatus + type TestStatus, + type TraceFormat } from '@wdio/devtools-shared' export interface DevToolsOptions { @@ -19,6 +22,11 @@ export interface DevToolsOptions { hostname?: string /** Open a Chrome window pointing at the UI. Default true. */ openUi?: boolean + /** `live` (default) launches the DevTools UI; `trace` skips it. Overrides `openUi`. */ + mode?: DevToolsMode + /** Trace output layout โ€” `zip` (default) writes a single archive, + * `ndjson-directory` unpacks into `trace-/`. Only applies in trace mode. */ + traceFormat?: TraceFormat /** Capture screenshots after each command. Default true. */ captureScreenshots?: boolean /** Command template for per-test rerun. {{testName}} is substituted. */ @@ -35,7 +43,11 @@ export interface DevToolsOptions { // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing selenium-internal imports. -import type { ScreencastOptions } from '@wdio/devtools-shared' +import type { + DevToolsMode, + ScreencastOptions, + TraceFormat +} from '@wdio/devtools-shared' export type { ScreencastFrame, ScreencastOptions } from '@wdio/devtools-shared' /** diff --git a/packages/selenium-devtools/tests/session.test.ts b/packages/selenium-devtools/tests/session.test.ts index a780adf4..785b5d3b 100644 --- a/packages/selenium-devtools/tests/session.test.ts +++ b/packages/selenium-devtools/tests/session.test.ts @@ -221,20 +221,18 @@ describe('selenium SessionCapturer (with stashed executeScript)', () => { let call = 0 stubExec(async () => { call++ - return call === 1 ? true : null + return null }) const cap = makeCapturer({ id: 'd' }) await expect(cap.captureTrace()).resolves.toBeUndefined() - expect(call).toBe(2) + // Single atomic check+read in one executeScript โ€” see session.ts comment. + expect(call).toBe(1) }) it('captureTrace processes payload when collector returns data', async () => { let call = 0 stubExec(async () => { call++ - if (call === 1) { - return true - } return { mutations: [], networkRequests: [], @@ -243,7 +241,7 @@ describe('selenium SessionCapturer (with stashed executeScript)', () => { }) const cap = makeCapturer({ id: 'd' }) await cap.captureTrace() - expect(call).toBe(2) + expect(call).toBe(1) }) it('captureTrace swallows ECONNREFUSED / no-such-session errors silently', async () => { diff --git a/packages/service/README.md b/packages/service/README.md index 887f885d..d5532798 100644 --- a/packages/service/README.md +++ b/packages/service/README.md @@ -46,7 +46,8 @@ services: [['devtools', options]] | `port` | `number` | random | Port the DevTools UI server listens on | | `hostname` | `string` | `'localhost'` | Hostname the DevTools UI server binds to | | `devtoolsCapabilities` | `Capabilities` | Chrome 1600ร—1200 | Capabilities used to open the DevTools UI window | -| `screencast` | `ScreencastOptions` | โ€” | Session video recording (see below) | +| `screencast` | `ScreencastOptions` | โ€” | Session video recording (live mode only โ€” see below) | +| `mode` | `'live' \| 'trace'` | `'live'` | `'live'` opens the DevTools UI window; `'trace'` skips the UI and writes a `trace-.zip` at session end. See [Trace mode](../../README.md#-trace-mode-tracezip) | ## Screencast Recording diff --git a/packages/service/package.json b/packages/service/package.json index 66461d70..b00eb468 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -40,6 +40,7 @@ "@babel/types": "^7.29.7", "@wdio/devtools-backend": "workspace:^", "@wdio/devtools-script": "workspace:^", + "@wdio/elements": "workspace:^", "@wdio/logger": "9.18.0", "@wdio/reporter": "9.27.2", "@wdio/types": "9.27.2", @@ -47,7 +48,8 @@ "import-meta-resolve": "^4.2.0", "stack-trace": "^1.0.0", "stacktrace-parser": "^0.1.11", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" }, "license": "MIT", "devDependencies": { diff --git a/packages/service/src/action-snapshot.ts b/packages/service/src/action-snapshot.ts new file mode 100644 index 00000000..6512f33a --- /dev/null +++ b/packages/service/src/action-snapshot.ts @@ -0,0 +1,32 @@ +// WDIO adapter: wires WebdriverIO.Browser into core's captureActionSnapshot. +// `browser.execute` is passed a Function reconstructed from the script body +// string (the same trick @wdio/elements uses); a raw string would route +// through a different WDIO path that doesn't preserve the script closure. +// +// `src` is NOT user-controlled: it's one of two compile-time constants +// produced by `@wdio/devtools-core/element-scripts` and shipped with the +// library. No external input reaches new Function() โ€” the lint flag here is +// a false positive given the closed input set. + +import { captureActionSnapshot as coreCapture } from '@wdio/devtools-core' +import type { ActionSnapshot } from '@wdio/devtools-shared' + +function reviveScript(src: string): () => unknown { + // `src` from core/element-scripts is already a self-invoking IIFE + // (`(function () { ... })()`); we just wrap it in a return so it's + // a function browser.execute() can call. + return new Function(`return (${src})`) as () => unknown +} + +export function captureActionSnapshot( + browser: WebdriverIO.Browser, + command: string +): Promise { + return coreCapture({ + command, + runScript: (src) => browser.execute(reviveScript(src)), + takeScreenshot: () => browser.takeScreenshot().catch(() => undefined), + getUrl: () => browser.getUrl().catch(() => undefined), + getTitle: () => browser.getTitle().catch(() => undefined) + }) +} diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 1f54eba1..2984500d 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -3,7 +3,13 @@ import fs from 'node:fs/promises' import path from 'node:path' import logger from '@wdio/logger' -import { errorMessage } from '@wdio/devtools-core' +import { + errorMessage, + mapCommandToAction, + writeTraceZip +} from '@wdio/devtools-core' +import { captureActionSnapshot } from './action-snapshot.js' +import type { ActionSnapshot } from '@wdio/devtools-shared' import { SevereServiceError } from 'webdriverio' import type { Services, Reporters, Capabilities, Options } from '@wdio/types' import type { WebDriverCommands } from '@wdio/protocols' @@ -47,9 +53,18 @@ export default class DevToolsHookService implements Services.ServiceInstance { #bidiListenersSetup = false #screencastRecorder?: ScreencastRecorder #screencastOptions?: ScreencastOptions + #options: ServiceOptions + #actionSnapshots: ActionSnapshot[] = [] + #snapshotCaptures: Promise[] = [] constructor(serviceOptions: ServiceOptions = {}) { - this.#screencastOptions = serviceOptions.screencast + this.#options = serviceOptions + if (serviceOptions.mode === 'trace' && serviceOptions.screencast?.enabled) { + log.warn('trace mode: ignoring screencast option (live-mode feature)') + this.#screencastOptions = undefined + } else { + this.#screencastOptions = serviceOptions.screencast + } } /** @@ -275,7 +290,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { if (frame?.command === command) { this.#commandStack.pop() if (this.#browser) { - return this.#sessionCapturer.afterCommand( + const captured = this.#sessionCapturer.afterCommand( this.#browser, command, args, @@ -283,6 +298,21 @@ export default class DevToolsHookService implements Services.ServiceInstance { error, frame.callSource ) + if ( + this.#options.mode === 'trace' && + !error && + mapCommandToAction(command) + ) { + const browser = this.#browser + this.#snapshotCaptures.push( + captureActionSnapshot(browser, command).then((snap) => { + if (snap) { + this.#actionSnapshots.push(snap) + } + }) + ) + } + return captured } } @@ -304,6 +334,11 @@ export default class DevToolsHookService implements Services.ServiceInstance { // Stop and encode the screencast for the current session. await this.#finalizeScreencast(this.#browser.sessionId) + // Drain in-flight per-action snapshots before writing the trace. + if (this.#snapshotCaptures.length) { + await Promise.allSettled(this.#snapshotCaptures) + } + const outputDir = this.#outputDir const { ...options } = this.#browser.options const traceLog: TraceLog = { @@ -319,7 +354,10 @@ export default class DevToolsHookService implements Services.ServiceInstance { }, commands: this.#sessionCapturer.commandsLog, sources: Object.fromEntries(this.#sessionCapturer.sources), - suites: this.#testReporters.map((reporter) => reporter.report) + suites: this.#testReporters.map((reporter) => reporter.report), + ...(this.#actionSnapshots.length + ? { actionSnapshots: this.#actionSnapshots } + : {}) } const traceFilePath = path.join( @@ -329,6 +367,19 @@ export default class DevToolsHookService implements Services.ServiceInstance { await fs.writeFile(traceFilePath, JSON.stringify(traceLog)) log.info(`DevTools trace saved to ${traceFilePath}`) + if (this.#options.mode === 'trace') { + const tracePath = await writeTraceZip(this.#sessionCapturer, { + outputDir, + sessionId: this.#browser.sessionId, + capabilities: this.#browser.capabilities, + actionSnapshots: this.#actionSnapshots.length + ? this.#actionSnapshots + : undefined, + format: this.#options.traceFormat + }) + log.info(`Trace saved to ${tracePath}`) + } + // Clean up console patching this.#sessionCapturer.cleanup() } diff --git a/packages/service/src/launcher.ts b/packages/service/src/launcher.ts index b8fd252d..d874513a 100644 --- a/packages/service/src/launcher.ts +++ b/packages/service/src/launcher.ts @@ -136,6 +136,10 @@ export class DevToolsAppLauncher { port, hostname: this.#options.hostname || 'localhost' }) + if (this.#options.mode === 'trace') { + log.info('trace mode: backend started, skipping UI window launch') + return + } this.#browser = await remote({ automationProtocol: 'devtools', capabilities: { diff --git a/packages/service/src/session.ts b/packages/service/src/session.ts index fcd6b508..97026437 100644 --- a/packages/service/src/session.ts +++ b/packages/service/src/session.ts @@ -207,20 +207,21 @@ export class SessionCapturer extends SessionCapturerBase { } try { - const collectorExists = await browser.execute( - () => typeof window.wdioTraceCollector !== 'undefined' + // Atomic check+read in a single browser.execute so the collector can't + // disappear (page navigation) between the existence check and the + // getTraceData call. Two round-trips left a TOCTOU race that surfaced + // spurious "Cannot read properties of undefined" errors. + const payload = await browser.execute(() => + typeof window.wdioTraceCollector !== 'undefined' + ? window.wdioTraceCollector.getTraceData() + : null ) - - if (!collectorExists) { + if (!payload) { log.warn( 'wdioTraceCollector not loaded yet - page loaded before preload script took effect' ) return } - - const payload = await browser.execute(() => - window.wdioTraceCollector.getTraceData() - ) this.processTracePayload(payload as Record) } catch (err) { log.error(`Failed to capture trace: ${errorMessage(err)}`) diff --git a/packages/service/src/types.ts b/packages/service/src/types.ts index a4dce985..f10068da 100644 --- a/packages/service/src/types.ts +++ b/packages/service/src/types.ts @@ -23,8 +23,17 @@ export { // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing service-internal imports. -import type { ScreencastOptions } from '@wdio/devtools-shared' -export type { ScreencastFrame, ScreencastOptions } from '@wdio/devtools-shared' +import type { + DevToolsMode, + ScreencastOptions, + TraceFormat +} from '@wdio/devtools-shared' +export type { + DevToolsMode, + ScreencastFrame, + ScreencastOptions, + TraceFormat +} from '@wdio/devtools-shared' export interface ExtendedCapabilities extends WebdriverIO.Capabilities { 'wdio:devtoolsOptions'?: ServiceOptions @@ -58,6 +67,11 @@ export interface ServiceOptions { * uses CDP push mode; all other browsers fall back to screenshot polling. */ screencast?: ScreencastOptions + /** `live` (default) launches the DevTools UI; `trace` skips it. */ + mode?: DevToolsMode + /** Trace output layout โ€” `zip` (default) writes a single archive, + * `ndjson-directory` unpacks into `trace-/`. Only applies in trace mode. */ + traceFormat?: TraceFormat } declare namespace WebdriverIO { diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index f7b7fdfd..0a4bbe62 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -16,6 +16,15 @@ export enum TraceType { export type TestStatus = 'passed' | 'failed' | 'skipped' | 'pending' | 'running' +/** `live` opens the DevTools UI window; `trace` skips it and lets a downstream exporter consume captured state. */ +export type DevToolsMode = 'live' | 'trace' + +/** `zip` (default) writes a single `trace-.zip`; `ndjson-directory` writes + * the same `trace.trace` + `trace.network` + `resources/` layout unpacked + * into `trace-/`. Both are consumable by `playwright show-trace` โ€” the + * unpacked form skips the unzip step for agentic / scripted consumers. */ +export type TraceFormat = 'zip' | 'ndjson-directory' + /** * Enum-style accessor for the canonical TestStatus values. Adapter code uses * this for readable comparisons (`state === TEST_STATE.PASSED`). The app's @@ -244,6 +253,20 @@ export interface TraceMutation { url?: string } +/** + * Captured at each user-facing action boundary in `trace` mode. Feeds the + * downstream trace.zip exporter (Phase 4). `screenshot` is base64-encoded JPEG. + */ +export interface ActionSnapshot { + timestamp: number + command: string + url?: string + title?: string + screenshot?: string + elements?: unknown[] + snapshotText?: string +} + export interface TraceLog { mutations: TraceMutation[] logs: string[] @@ -255,6 +278,8 @@ export interface TraceLog { suites?: Record[] screencast?: ScreencastInfo config?: { configFile?: string } + /** Per-action snapshots captured in `mode: 'trace'` for the trace.zip exporter. */ + actionSnapshots?: ActionSnapshot[] } // โ”€โ”€โ”€ Preserve-and-rerun โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 55b17f19..9d5b6ce8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -299,18 +299,52 @@ importers: '@types/ws': specifier: ^8.18.1 version: 8.18.1 + '@types/yazl': + specifier: ^2.4.6 + version: 2.4.6 '@wdio/devtools-script': specifier: workspace:* version: link:../script '@wdio/devtools-shared': specifier: workspace:^ version: link:../shared + '@xmldom/xmldom': + specifier: ^0.9.8 + version: 0.9.10 stacktrace-parser: specifier: ^0.1.11 version: 0.1.11 ws: specifier: ^8.21.0 version: 8.21.0 + xpath: + specifier: ^0.0.34 + version: 0.0.34 + yazl: + specifier: ^2.5.1 + version: 2.5.1 + + packages/elements: + dependencies: + webdriverio: + specifier: ^9.0.0 + version: 9.27.2(puppeteer-core@21.11.0) + devDependencies: + '@types/node': + specifier: 25.9.1 + version: 25.9.1 + '@wdio/devtools-core': + specifier: workspace:^ + version: link:../core + '@wdio/globals': + specifier: 9.27.0 + version: 9.27.0(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0)) + typescript: + specifier: 6.0.2 + version: 6.0.2 + vitest: + specifier: ^4.0.16 + version: 4.1.8(@types/node@25.9.1)(@vitest/coverage-v8@4.1.8)(happy-dom@20.9.0)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) packages/nightwatch-devtools: dependencies: @@ -341,6 +375,9 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 devDependencies: '@types/node': specifier: 25.9.1 @@ -406,6 +443,9 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 devDependencies: '@cucumber/cucumber': specifier: ^13.0.0 @@ -465,6 +505,9 @@ importers: '@wdio/devtools-script': specifier: workspace:^ version: link:../script + '@wdio/elements': + specifier: workspace:^ + version: link:../elements '@wdio/logger': specifier: 9.18.0 version: 9.18.0 @@ -495,6 +538,9 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 devDependencies: '@types/babel__core': specifier: ^7.20.5 @@ -2319,6 +2365,9 @@ packages: '@types/yauzl@2.10.3': resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} + '@types/yazl@2.4.6': + resolution: {integrity: sha512-/ifFjQtcKaoZOjl5NNCQRR0fAKafB3Foxd7J/WvFPTMea46zekapcR30uzkwIkKAAuq5T6d0dkwz754RFH27hg==} + '@typescript-eslint/eslint-plugin@8.60.1': resolution: {integrity: sha512-JQ4S5GB0tfjO8BuJ4fcX+HodkzJjYBV+7OJ+wLygaX7OGQ7FudyHL4NSCA6ob+w3Yn+5MkKIozOwQhXeM7opVg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -2570,6 +2619,13 @@ packages: resolution: {integrity: sha512-xoBgmACafV4L7e7e3DUN8UM1N+I225oms38JtxtfgrMfvHm8QtcmZWXfycxEGM28Gm2M3NmeV3oso7hZeBk6Ww==} engines: {node: '>=18.20.0'} + '@wdio/globals@9.27.0': + resolution: {integrity: sha512-yT6EAyvEqm+wFD11fg89BMxvFkYLgnIVCihfJx+k73Gm3utL/DfZQpSheQdwrlQzu5p7jHi/JwOD76740F5Peg==} + engines: {node: '>=18.20.0'} + peerDependencies: + expect-webdriverio: ^5.6.5 + webdriverio: ^9.0.0 + '@wdio/globals@9.27.2': resolution: {integrity: sha512-Rx9bqD4/8iR3CNPMWYxywQSCqsR/WGwIYT2Q0uUmrvPxOdYFridDEhVRGO32kQ55UM5+JXzXppxgwGLRQ60fJg==} engines: {node: '>=18.20.0'} @@ -2634,6 +2690,10 @@ packages: resolution: {integrity: sha512-Rj8AP/VYVd5clZFKy+P7zzoXCKshjrog6lcV65nnUzATbUYT/PpUCy6OhEWHTSmLQY2Oc5ztY/IetLSg4nmB3w==} engines: {node: '>=18'} + '@xmldom/xmldom@0.9.10': + resolution: {integrity: sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw==} + engines: {node: '>=14.6'} + '@zip.js/zip.js@2.8.26': resolution: {integrity: sha512-RQ4h9F6DOiHxpdocUDrOl6xBM+yOtz+LkUol47AVWcfebGBDpZ7w7Xvz9PS24JgXvLGiXXzSAfdCdVy1tPlaFA==} engines: {bun: '>=0.7.0', deno: '>=1.0.0', node: '>=18.0.0'} @@ -6739,6 +6799,11 @@ packages: engines: {node: '>=14.17'} hasBin: true + typescript@6.0.2: + resolution: {integrity: sha512-bGdAIrZ0wiGDo5l8c++HWtbaNCWTS4UTv7RaTH/ThVIgjkveJt83m74bBHMJkuCbslY8ixgLBVZJIOiQlQTjfQ==} + engines: {node: '>=14.17'} + hasBin: true + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -7197,6 +7262,10 @@ packages: xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + xpath@0.0.34: + resolution: {integrity: sha512-FxF6+rkr1rNSQrhUNYrAFJpRXNzlDoMxeXN5qI84939ylEv3qqPFKa85Oxr6tDaJKqwW6KKyo2v26TSv3k6LeA==} + engines: {node: '>=0.6.0'} + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -7235,6 +7304,9 @@ packages: yauzl@2.10.0: resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + yazl@2.5.1: + resolution: {integrity: sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw==} + yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} @@ -9070,6 +9142,10 @@ snapshots: '@types/node': 25.9.1 optional: true + '@types/yazl@2.4.6': + dependencies: + '@types/node': 25.9.1 + '@typescript-eslint/eslint-plugin@8.60.1(@typescript-eslint/parser@8.60.1(eslint@10.4.1(jiti@2.7.0))(typescript@6.0.3))(eslint@10.4.1(jiti@2.7.0))(typescript@6.0.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -9416,6 +9492,11 @@ snapshots: '@wdio/types': 9.27.2 chalk: 5.6.2 + '@wdio/globals@9.27.0(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0))': + dependencies: + expect-webdriverio: 5.6.7(@wdio/globals@9.27.0)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)) + webdriverio: 9.27.2(puppeteer-core@21.11.0) + '@wdio/globals@9.27.2(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0))': dependencies: expect-webdriverio: 5.6.7(@wdio/globals@9.27.2)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)) @@ -9558,6 +9639,8 @@ snapshots: dependencies: '@wdio/logger': 9.18.0 + '@xmldom/xmldom@0.9.10': {} + '@zip.js/zip.js@2.8.26': {} abort-controller@3.0.0: @@ -11082,6 +11165,16 @@ snapshots: expect-type@1.3.0: {} + expect-webdriverio@5.6.7(@wdio/globals@9.27.0)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)): + dependencies: + '@vitest/snapshot': 4.1.8 + '@wdio/globals': 9.27.0(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0)) + '@wdio/logger': 9.18.0 + deep-eql: 5.0.2 + expect: 30.4.1 + jest-matcher-utils: 30.4.1 + webdriverio: 9.27.2(puppeteer-core@21.11.0) + expect-webdriverio@5.6.7(@wdio/globals@9.27.2)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)): dependencies: '@vitest/snapshot': 4.1.8 @@ -14384,6 +14477,8 @@ snapshots: typescript@5.9.3: optional: true + typescript@6.0.2: {} + typescript@6.0.3: {} ua-parser-js@1.0.41: {} @@ -14833,6 +14928,8 @@ snapshots: xmlchars@2.2.0: {} + xpath@0.0.34: {} + y18n@5.0.8: {} yallist@3.1.1: {} @@ -14877,6 +14974,10 @@ snapshots: buffer-crc32: 0.2.13 fd-slicer: 1.1.0 + yazl@2.5.1: + dependencies: + buffer-crc32: 0.2.13 + yn@3.1.1: {} yocto-queue@0.1.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c577713a..038df802 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -2,6 +2,7 @@ packages: - 'packages/shared' - 'packages/core' + - 'packages/elements' - 'packages/backend' - 'packages/script' - 'packages/service' diff --git a/tsconfig.json b/tsconfig.json index 97c599d0..6cd2e8c9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,6 +29,8 @@ "@wdio/devtools-shared/*": ["packages/shared/src/*"], "@wdio/devtools-core": ["packages/core/src/index.ts"], "@wdio/devtools-core/*": ["packages/core/src/*"], + "@wdio/elements": ["packages/elements/src/index.ts"], + "@wdio/elements/*": ["packages/elements/src/*"], "@wdio/devtools-backend": ["packages/backend/src/index.ts"], "@wdio/devtools-backend/*": ["packages/backend/src/*"], "@wdio/devtools-script": ["packages/script/src/index.ts"],