From e9a4b665bb93757ea4f8e4473899d67433748950 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 4 Jun 2026 15:54:53 +0530 Subject: [PATCH 01/13] Phase 1: Introduce live and trace mode; Gate UI from launching for trace mode --- packages/nightwatch-devtools/src/index.ts | 3 ++- .../src/plugin-internals.ts | 3 ++- .../nightwatch-devtools/src/run-lifecycle.ts | 22 +++++++++++++------ packages/nightwatch-devtools/src/types.ts | 5 ++++- packages/selenium-devtools/src/index.ts | 9 ++++---- packages/selenium-devtools/src/types.ts | 5 ++++- packages/service/src/launcher.ts | 4 ++++ packages/service/src/types.ts | 10 +++++++-- packages/shared/src/types.ts | 3 +++ 9 files changed, 46 insertions(+), 18 deletions(-) diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index 395e4a77..dcb49dd9 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -102,7 +102,8 @@ class NightwatchDevToolsPlugin { port: options.port ?? 3000, hostname: options.hostname ?? 'localhost', screencast: options.screencast ?? {}, - bidi: options.bidi ?? false + bidi: options.bidi ?? false, + mode: options.mode ?? 'live' } this.#screencastOptions = { ...SCREENCAST_DEFAULTS, diff --git a/packages/nightwatch-devtools/src/plugin-internals.ts b/packages/nightwatch-devtools/src/plugin-internals.ts index 06c75f22..ce0165bf 100644 --- a/packages/nightwatch-devtools/src/plugin-internals.ts +++ b/packages/nightwatch-devtools/src/plugin-internals.ts @@ -14,6 +14,7 @@ import type { TestManager } from './helpers/testManager.js' import type { SuiteManager } from './helpers/suiteManager.js' import type { BrowserProxy } from './helpers/browserProxy.js' import type { + DevToolsMode, NightwatchBrowser, ScreencastOptions, SuiteStats, @@ -22,7 +23,7 @@ import type { export interface PluginInternals { // Config + options - options: { hostname: string; port: number } + options: { hostname: string; port: number; mode?: DevToolsMode } readonly hostname: string readonly port: number readonly screencastOptions: ScreencastOptions diff --git a/packages/nightwatch-devtools/src/run-lifecycle.ts b/packages/nightwatch-devtools/src/run-lifecycle.ts index a6d7e246..4769a9b5 100644 --- a/packages/nightwatch-devtools/src/run-lifecycle.ts +++ b/packages/nightwatch-devtools/src/run-lifecycle.ts @@ -19,14 +19,18 @@ import type { SessionCapturer } from './session.js' import type { TestReporter } from './reporter.js' import type { SuiteManager } from './helpers/suiteManager.js' import type { TestManager } from './helpers/testManager.js' -import type { NightwatchBrowser, NightwatchCurrentTest } from './types.js' +import type { + DevToolsMode, + NightwatchBrowser, + NightwatchCurrentTest +} from './types.js' import { TIMING, PLUGIN_GLOBAL_KEY } from './constants.js' import { findFreePort, resolveNightwatchConfig } from './helpers/utils.js' const log = logger('@wdio/nightwatch-devtools:run-lifecycle') export interface RunLifecycleCtx { - options: { hostname: string; port: number } + options: { hostname: string; port: number; mode?: DevToolsMode } readonly testReporter: TestReporter | undefined readonly suiteManager: SuiteManager | undefined readonly testManager: TestManager @@ -97,11 +101,15 @@ export async function runPluginBefore(ctx: PluginBeforeCtx): Promise { ctx.options.port = port const url = `http://${ctx.options.hostname}:${ctx.options.port}` log.info(`✓ Backend started on port ${ctx.options.port}`) - log.info(` DevTools UI: ${url}`) - await ctx.openDevtoolsBrowserAt(url) - await new Promise((resolve) => - setTimeout(resolve, TIMING.UI_CONNECTION_WAIT) - ) + if (ctx.options.mode === 'trace') { + log.info('trace mode: backend started, skipping UI window launch') + } else { + log.info(` DevTools UI: ${url}`) + await ctx.openDevtoolsBrowserAt(url) + await new Promise((resolve) => + setTimeout(resolve, TIMING.UI_CONNECTION_WAIT) + ) + } ;(globalThis as Record)[PLUGIN_GLOBAL_KEY] = ctx.plugin } catch (err) { log.error(`Failed to start backend: ${errorMessage(err)}`) diff --git a/packages/nightwatch-devtools/src/types.ts b/packages/nightwatch-devtools/src/types.ts index a2e2b8c0..9a81c03c 100644 --- a/packages/nightwatch-devtools/src/types.ts +++ b/packages/nightwatch-devtools/src/types.ts @@ -4,6 +4,7 @@ export { TraceType, type CommandLog, type ConsoleLog, + type DevToolsMode, type DocumentInfo, type LogLevel, type Metadata, @@ -17,7 +18,7 @@ export { type TraceLog } from '@wdio/devtools-shared' -import type { ScreencastOptions } from '@wdio/devtools-shared' +import type { DevToolsMode, ScreencastOptions } from '@wdio/devtools-shared' export interface CommandStackFrame { command: string @@ -93,6 +94,8 @@ export interface DevToolsOptions { * entries. Defaults to `false` — opt-in. */ bidi?: boolean + /** `live` (default) launches the DevTools UI; `trace` skips it. */ + mode?: DevToolsMode } export interface NightwatchBrowser { diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index 0e95b7de..63a091d1 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -1,7 +1,6 @@ // @wdio/selenium-devtools — runner-agnostic Selenium WebDriver adapter. // Side-effect import that patches selenium-webdriver and starts the backend. -// MUST be the first import — see setupConsole.ts. import './setupConsole.js' import logger from '@wdio/logger' import { startDetachedBackend } from './helpers/detachedBackend.js' @@ -116,11 +115,11 @@ class SeleniumDevToolsPlugin { this.#options = { port: options.port ?? 3000, hostname: options.hostname ?? 'localhost', - // Default true to match @wdio/devtools-service and @wdio/nightwatch-devtools. openUi: options.openUi ?? true, captureScreenshots: options.captureScreenshots ?? true, rerunCommand: options.rerunCommand, - headless: options.headless ?? false + headless: options.headless ?? false, + mode: options.mode ?? 'live' } this.#rerunManager = new RerunManager(RUNNER) if (options.rerunCommand) { @@ -187,8 +186,8 @@ class SeleniumDevToolsPlugin { ) } this.#backendStarted = true - // Skip when in REUSE mode — the rerun child reuses the parent's window. - if (this.#options.openUi && !this.#isReuse) { + const { mode, openUi } = this.#options + if (mode !== 'trace' && openUi && !this.#isReuse) { this.#openUiWindow() } } catch (err) { diff --git a/packages/selenium-devtools/src/types.ts b/packages/selenium-devtools/src/types.ts index d239a308..2e60f850 100644 --- a/packages/selenium-devtools/src/types.ts +++ b/packages/selenium-devtools/src/types.ts @@ -4,6 +4,7 @@ export { TraceType, type CommandLog, type ConsoleLog, + type DevToolsMode, type DocumentInfo, type LogLevel, type Metadata, @@ -19,6 +20,8 @@ export interface DevToolsOptions { hostname?: string /** Open a Chrome window pointing at the UI. Default true. */ openUi?: boolean + /** `live` (default) launches the DevTools UI; `trace` skips it. Overrides `openUi`. */ + mode?: DevToolsMode /** Capture screenshots after each command. Default true. */ captureScreenshots?: boolean /** Command template for per-test rerun. {{testName}} is substituted. */ @@ -35,7 +38,7 @@ export interface DevToolsOptions { // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing selenium-internal imports. -import type { ScreencastOptions } from '@wdio/devtools-shared' +import type { DevToolsMode, ScreencastOptions } from '@wdio/devtools-shared' export type { ScreencastFrame, ScreencastOptions } from '@wdio/devtools-shared' /** diff --git a/packages/service/src/launcher.ts b/packages/service/src/launcher.ts index b8fd252d..d874513a 100644 --- a/packages/service/src/launcher.ts +++ b/packages/service/src/launcher.ts @@ -136,6 +136,10 @@ export class DevToolsAppLauncher { port, hostname: this.#options.hostname || 'localhost' }) + if (this.#options.mode === 'trace') { + log.info('trace mode: backend started, skipping UI window launch') + return + } this.#browser = await remote({ automationProtocol: 'devtools', capabilities: { diff --git a/packages/service/src/types.ts b/packages/service/src/types.ts index a4dce985..b20de413 100644 --- a/packages/service/src/types.ts +++ b/packages/service/src/types.ts @@ -23,8 +23,12 @@ export { // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing service-internal imports. -import type { ScreencastOptions } from '@wdio/devtools-shared' -export type { ScreencastFrame, ScreencastOptions } from '@wdio/devtools-shared' +import type { DevToolsMode, ScreencastOptions } from '@wdio/devtools-shared' +export type { + DevToolsMode, + ScreencastFrame, + ScreencastOptions +} from '@wdio/devtools-shared' export interface ExtendedCapabilities extends WebdriverIO.Capabilities { 'wdio:devtoolsOptions'?: ServiceOptions @@ -58,6 +62,8 @@ export interface ServiceOptions { * uses CDP push mode; all other browsers fall back to screenshot polling. */ screencast?: ScreencastOptions + /** `live` (default) launches the DevTools UI; `trace` skips it. */ + mode?: DevToolsMode } declare namespace WebdriverIO { diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index f7b7fdfd..bd23a501 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -16,6 +16,9 @@ export enum TraceType { export type TestStatus = 'passed' | 'failed' | 'skipped' | 'pending' | 'running' +/** `live` opens the DevTools UI window; `trace` skips it and lets a downstream exporter consume captured state. */ +export type DevToolsMode = 'live' | 'trace' + /** * Enum-style accessor for the canonical TestStatus values. Adapter code uses * this for readable comparisons (`state === TEST_STATE.PASSED`). The app's From e9bf75d25aa0c038ebef0a183dabffd76504355c Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 4 Jun 2026 16:04:23 +0530 Subject: [PATCH 02/13] Phase 2: Incorporating @wdio/elements into devtools for the trace structure --- packages/elements/.npmignore | 7 + packages/elements/ROADMAP.md | 81 ++ packages/elements/package.json | 42 + packages/elements/src/accessibility-tree.ts | 480 +++++++++++ packages/elements/src/browser-elements.ts | 303 +++++++ packages/elements/src/get-elements.ts | 67 ++ packages/elements/src/index.ts | 21 + packages/elements/src/locators/constants.ts | 169 ++++ .../elements/src/locators/element-filter.ts | 234 ++++++ packages/elements/src/locators/index.ts | 279 +++++++ .../src/locators/locator-generation.ts | 644 +++++++++++++++ packages/elements/src/locators/types.ts | 110 +++ packages/elements/src/locators/xml-parsing.ts | 329 ++++++++ packages/elements/src/mobile-elements.ts | 199 +++++ packages/elements/src/snapshot.ts | 758 ++++++++++++++++++ .../elements/tests/accessibility-tree.test.ts | 27 + .../elements/tests/browser-elements.test.ts | 22 + .../tests/locators/locator-generation.test.ts | 23 + .../elements/tests/mobile-elements.test.ts | 13 + packages/elements/tests/snapshot.test.ts | 250 ++++++ packages/elements/tsconfig.json | 16 + .../nightwatch-devtools/tests/session.test.ts | 12 +- pnpm-lock.yaml | 66 ++ pnpm-workspace.yaml | 1 + tsconfig.json | 2 + 25 files changed, 4150 insertions(+), 5 deletions(-) create mode 100644 packages/elements/.npmignore create mode 100644 packages/elements/ROADMAP.md create mode 100644 packages/elements/package.json create mode 100644 packages/elements/src/accessibility-tree.ts create mode 100644 packages/elements/src/browser-elements.ts create mode 100644 packages/elements/src/get-elements.ts create mode 100644 packages/elements/src/index.ts create mode 100644 packages/elements/src/locators/constants.ts create mode 100644 packages/elements/src/locators/element-filter.ts create mode 100644 packages/elements/src/locators/index.ts create mode 100644 packages/elements/src/locators/locator-generation.ts create mode 100644 packages/elements/src/locators/types.ts create mode 100644 packages/elements/src/locators/xml-parsing.ts create mode 100644 packages/elements/src/mobile-elements.ts create mode 100644 packages/elements/src/snapshot.ts create mode 100644 packages/elements/tests/accessibility-tree.test.ts create mode 100644 packages/elements/tests/browser-elements.test.ts create mode 100644 packages/elements/tests/locators/locator-generation.test.ts create mode 100644 packages/elements/tests/mobile-elements.test.ts create mode 100644 packages/elements/tests/snapshot.test.ts create mode 100644 packages/elements/tsconfig.json diff --git a/packages/elements/.npmignore b/packages/elements/.npmignore new file mode 100644 index 00000000..344b429f --- /dev/null +++ b/packages/elements/.npmignore @@ -0,0 +1,7 @@ +src +node_modules +tests +index.html +tsconfig.json +vite.config.ts +*.tgz diff --git a/packages/elements/ROADMAP.md b/packages/elements/ROADMAP.md new file mode 100644 index 00000000..f387fb4b --- /dev/null +++ b/packages/elements/ROADMAP.md @@ -0,0 +1,81 @@ +# @wdio/elements Roadmap + +## Current state (May 2026) + +The package delivers LLM-readable element snapshots for both web and mobile: + +| Capability | Web | Mobile | +|---|---|---| +| Interactable element list | `getInteractableBrowserElements()` | `getMobileVisibleElements()` | +| Semantic tree | `getBrowserAccessibilityTree()` | *(raw `JSONElement` only)* | +| Snapshot serialization | `serializeWebSnapshot()` | `serializeMobileSnapshot()` | +| Unified API | `getElements()` returns both | `getElements()` returns both | +| Viewport filtering | `inViewportOnly` (default true) | `inViewportOnly` (default true) | +| Role classification | Computed in-browser from tag/ARIA | `ANDROID_ROLE_MAP` / `IOS_ROLE_MAP` in snapshot.ts | +| Locator generation | CSS selectors in browser script | `getSuggestedLocators()` from locator-generation.ts | +| Context disambiguation | `∈` via `inferPurpose()` | `∈` via `mobileInferPurpose()` | +| Duplicate selector indexing | N/A (selectors are unique) | `.instance(N)` suffix | + +## Architectural concerns + +### 1. Two independent mobile pipelines + +`serializeMobileSnapshot` in `snapshot.ts` has its own copies of: + +- **Role classification** — `ANDROID_ROLE_MAP` / `IOS_ROLE_MAP` duplicate logic from `locators/constants.ts` and `locators/element-filter.ts`. +- **Interactivity detection** — `isMobileInteractive()` shadows `isInteractableElement()` from `element-filter.ts`. They use different criteria (tag-based vs attribute-based) and can disagree. +- **Locator generation** — `getBestAndroidLocator()` / `getBestIOSLocator()` are simplified fallbacks. The full pipeline (`getSuggestedLocators()`) is now wired in when source XML is available, but the fallback still exists and the two paths can produce different selectors for the same element. + +These should be collapsed: `serializeMobileSnapshot` should consume pre-computed roles, interactivity flags, and selectors from the locator pipeline, not recompute them. + +### 2. No mobile equivalent of `getBrowserAccessibilityTree()` + +The web path returns a flat `AccessibilityNode[]` with roles, names, selectors, depths, and state. The mobile path returns a raw `JSONElement` tree — the snapshot does all enrichment internally via `collectMobileNodes()` → `MobileFlatNode[]` (a private interface). There is no public function to get an enriched flat node list for mobile. + +**Proposal:** Extract `collectMobileNodes()` into a public `getMobileAccessibilityTree()` that returns `MobileFlatNode[]` (or a shared type). `serializeMobileSnapshot()` becomes a pure formatting pass — like `serializeWebSnapshot()` already is. + +### 3. Layout noise in mobile snapshots + +The Android view hierarchy includes every layout container (`FrameLayout`, `LinearLayout`, `ViewGroup`, etc.). The current noise filter (`NOISY_ROLES`) collapses anonymous containers at depth ≥ 2, but named containers and depth 0-1 scaffolding still appear. The web a11y tree doesn't have this problem because the browser's accessibility computation already skips layout-only `
`s. + +**Proposal:** A `collapseContainers` option on the snapshot (default `true`) that skips any container without an interactive descendant. Alternatively, the tree collection pass could flag "informative" vs "structural" containers and let the renderer decide. + +### 4. Selector format for mobile + +Mobile selectors are Appium/WDIO-specific strings (`~Accessibility`, `android=new UiSelector()...`, `id:com.example:id/foo`). The web path outputs CSS selectors (`a*=Highlights`, `#cart-icon-bubble`). An LLM/agent needs different selector parsing logic per platform. There's no common selector abstraction. + +**Proposal:** A `SelectorString` type with platform-aware parsing, or at minimum consistent prefix conventions documented for LLM consumption. + +### 5. The raw tree doesn't carry locators unless processed + +`getMobileVisibleElementsWithTree()` returns `{ elements, tree }` where `tree` is the raw `xmlToJSON()` output. Locators are only on `elements` (from `generateAllElementLocators()`). The snapshot reads locators by running `getSuggestedLocators()` again (or falling back). If a consumer wants to annotate the tree themselves, they must re-run the locator pipeline. + +**Proposal:** Enrich the tree in-place during `generateAllElementLocators()` — attach `_selector`, `_role`, and `_interactive` attributes to each `JSONElement` node that passes the filter. The raw tree becomes self-describing. + +## Improvement backlog + +| Priority | What | Effort | +|---|---|---| +| P0 | Merge `isMobileInteractive` + role classification into `generateAllElementLocators` — one source of truth | Medium | +| P1 | Extract `getMobileAccessibilityTree()` as a public API returning enriched flat nodes | Medium | +| P1 | Enrich `JSONElement` tree nodes with locators during `generateAllElementLocators()` | Small | +| P2 | `collapseContainers` option on `serializeMobileSnapshot` | Small | +| P2 | Unify web + mobile serialization into a single `serializeSnapshot()` function | Large | +| P3 | Document selector format conventions for LLM consumption | Small | +| P3 | Add `checked`/`selected`/`expanded` state rendering to mobile snapshot (parity with web) | Small | + +## Verified capabilities + +- [x] Web: viewport-only snapshot with semantic roles and unique CSS selectors +- [x] Web: `∈` disambiguation for duplicate selectors (6 "Add to Wishlist" buttons → each with book title context) +- [x] Web: `statictext` role capturing visible text (book titles, promo copy, cookie text) +- [x] Web: deduplication of echoed text (child text already in parent name → skipped) +- [x] Mobile: semantic role mapping (TextView→statictext, ImageView→img, Button→button, etc.) +- [x] Mobile: full-pipeline selectors via `getSuggestedLocators()` wired into snapshot +- [x] Mobile: `~` prefix for accessibility-id, `id:` for resource-id, `android=new UiSelector()...` for compound +- [x] Mobile: `.instance(N)` indexing for duplicate selectors +- [x] Mobile: explicit tap-target promotion (clickable parent carries `→`, label children provide `∈` context) +- [x] Mobile: layout noise collapse for anonymous containers +- [x] Mobile: `∈` context from actual parent, not previous list-item sibling +- [x] Unified `getElements()` API returning `{ elements, tree }` for both platforms +- [x] `inViewportOnly` default `true` across all entry points with per-function toggles diff --git a/packages/elements/package.json b/packages/elements/package.json new file mode 100644 index 00000000..3c208c09 --- /dev/null +++ b/packages/elements/package.json @@ -0,0 +1,42 @@ +{ + "name": "@wdio/elements", + "version": "1.0.0", + "description": "Element detection scripts for WebdriverIO", + "author": "Vince Graics", + "license": "MIT", + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./locators": { + "types": "./dist/locators/index.d.ts", + "import": "./dist/locators/index.js" + } + }, + "types": "./dist/index.d.ts", + "repository": { + "type": "git", + "url": "git+https://github.com/webdriverio/devtools.git", + "directory": "packages/elements" + }, + "scripts": { + "build": "tsc -p ./tsconfig.json", + "lint": "eslint . --fix", + "test": "vitest run" + }, + "dependencies": { + "@xmldom/xmldom": "^0.9.8", + "xpath": "^0.0.34" + }, + "devDependencies": { + "@types/node": "25.5.2", + "@wdio/globals": "9.27.0", + "typescript": "6.0.2", + "vitest": "^4.0.16" + }, + "peerDependencies": { + "webdriverio": "^9.0.0" + } +} diff --git a/packages/elements/src/accessibility-tree.ts b/packages/elements/src/accessibility-tree.ts new file mode 100644 index 00000000..a5e593bb --- /dev/null +++ b/packages/elements/src/accessibility-tree.ts @@ -0,0 +1,480 @@ +/** + * Browser accessibility tree + * Single browser.execute() call: DOM walk → flat accessibility node list + * + * NOTE: This script runs in browser context via browser.execute() + * It must be self-contained with no external dependencies + */ + +export interface AccessibilityNode { + role: string + name: string + selector: string + depth: number + level: number | string + disabled: string + checked: string + expanded: string + selected: string + pressed: string + required: string + readonly: string + /** Whether the element's bounding rect intersects the viewport. */ + isInViewport?: boolean +} + +const accessibilityTreeScript = (inViewportOnly: boolean) => + (function () { + const INPUT_TYPE_ROLES: Record = { + text: 'textbox', + search: 'searchbox', + email: 'textbox', + url: 'textbox', + tel: 'textbox', + password: 'textbox', + number: 'spinbutton', + checkbox: 'checkbox', + radio: 'radio', + range: 'slider', + submit: 'button', + reset: 'button', + image: 'button', + file: 'button', + color: 'button' + } + + // Container roles: named only via aria-label/aria-labelledby, not textContent + const CONTAINER_ROLES = new Set([ + 'navigation', + 'banner', + 'contentinfo', + 'complementary', + 'main', + 'form', + 'region', + 'group', + 'list', + 'listitem', + 'table', + 'row', + 'rowgroup', + 'generic' + ]) + + function getRole(el: HTMLElement): string | null { + const explicit = el.getAttribute('role') + if (explicit) { + return explicit.split(' ')[0] + } + + const tag = el.tagName.toLowerCase() + + switch (tag) { + case 'button': + return 'button' + case 'a': + return el.hasAttribute('href') ? 'link' : null + case 'input': { + const type = (el.getAttribute('type') || 'text').toLowerCase() + if (type === 'hidden') { + return null + } + return INPUT_TYPE_ROLES[type] || 'textbox' + } + case 'select': + return 'combobox' + case 'textarea': + return 'textbox' + case 'h1': + case 'h2': + case 'h3': + case 'h4': + case 'h5': + case 'h6': + return 'heading' + case 'img': + return 'img' + case 'nav': + return 'navigation' + case 'main': + return 'main' + case 'header': + return !el.closest('article,aside,main,nav,section') ? 'banner' : null + case 'footer': + return !el.closest('article,aside,main,nav,section') + ? 'contentinfo' + : null + case 'aside': + return 'complementary' + case 'dialog': + return 'dialog' + case 'form': + return 'form' + case 'section': + return el.hasAttribute('aria-label') || + el.hasAttribute('aria-labelledby') + ? 'region' + : null + case 'summary': + return 'button' + case 'details': + return 'group' + case 'progress': + return 'progressbar' + case 'meter': + return 'meter' + case 'ul': + case 'ol': + return 'list' + case 'li': + return 'listitem' + case 'table': + return 'table' + } + + if ( + (el as HTMLElement & { contentEditable: string }).contentEditable === + 'true' + ) { + return 'textbox' + } + if ( + el.hasAttribute('tabindex') && + parseInt(el.getAttribute('tabindex') || '-1', 10) >= 0 + ) { + return 'generic' + } + + // Capture elements with visible direct text that don't match + // any semantic role — book titles, prices, labels, etc. + if (getDirectText(el)) { + return 'statictext' + } + + return null + } + + function getAccessibleName(el: HTMLElement, role: string | null): string { + const ariaLabel = el.getAttribute('aria-label') + if (ariaLabel) { + return ariaLabel.trim() + } + + const labelledBy = el.getAttribute('aria-labelledby') + if (labelledBy) { + const texts = labelledBy + .split(/\s+/) + .map((id) => document.getElementById(id)?.textContent?.trim() || '') + .filter(Boolean) + if (texts.length > 0) { + return texts.join(' ').slice(0, 200) + } + } + + const tag = el.tagName.toLowerCase() + + if ( + tag === 'img' || + (tag === 'input' && el.getAttribute('type') === 'image') + ) { + const alt = el.getAttribute('alt') + if (alt !== null) { + return alt.trim() + } + } + + if (['input', 'select', 'textarea'].includes(tag)) { + const id = el.getAttribute('id') + if (id) { + const label = document.querySelector(`label[for="${CSS.escape(id)}"]`) + if (label) { + return label.textContent?.trim() || '' + } + } + const parentLabel = el.closest('label') + if (parentLabel) { + const clone = parentLabel.cloneNode(true) as HTMLElement + clone + .querySelectorAll('input,select,textarea') + .forEach((n) => n.remove()) + const lt = clone.textContent?.trim() + if (lt) { + return lt + } + } + } + + const ph = el.getAttribute('placeholder') + if (ph) { + return ph.trim() + } + + const title = el.getAttribute('title') + if (title) { + return title.trim() + } + + // 9. Child — common pattern for image links and buttons + const childImg = el.querySelector('img') + if (childImg) { + const alt = childImg.getAttribute('alt') + if (alt) { + return alt.trim() + } + } + + if (role && CONTAINER_ROLES.has(role)) { + return '' + } + return (el.textContent?.trim().replace(/\s+/g, ' ') || '').slice(0, 200) + } + + function getSelector(element: HTMLElement): string { + const tag = element.tagName.toLowerCase() + + const text = element.textContent?.trim().replace(/\s+/g, ' ') + if (text && text.length > 0 && text.length <= 120) { + const sameTagElements = document.querySelectorAll(tag) + let matchCount = 0 + sameTagElements.forEach((el) => { + if (el.textContent?.includes(text)) { + matchCount++ + } + }) + if (matchCount === 1) { + return `${tag}*=${text}` + } + } + + const ariaLabel = element.getAttribute('aria-label') + if (ariaLabel && ariaLabel.length <= 200) { + const sel = `[aria-label="${CSS.escape(ariaLabel)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + const testId = element.getAttribute('data-testid') + if (testId) { + const sel = `[data-testid="${CSS.escape(testId)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + if (element.id) { + return `#${CSS.escape(element.id)}` + } + + const nameAttr = element.getAttribute('name') + if (nameAttr) { + const sel = `${tag}[name="${CSS.escape(nameAttr)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + if (element.className && typeof element.className === 'string') { + const classes = element.className.trim().split(/\s+/).filter(Boolean) + for (const cls of classes) { + const sel = `${tag}.${CSS.escape(cls)}` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + if (classes.length >= 2) { + const sel = `${tag}${classes + .slice(0, 2) + .map((c) => `.${CSS.escape(c)}`) + .join('')}` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + } + + let current: HTMLElement | null = element + const path: string[] = [] + while (current && current !== document.documentElement) { + let seg = current.tagName.toLowerCase() + if (current.id) { + path.unshift(`#${CSS.escape(current.id)}`) + break + } + const parent = current.parentElement + if (parent) { + const siblings = Array.from(parent.children).filter( + (c) => c.tagName === current!.tagName + ) + if (siblings.length > 1) { + seg += `:nth-of-type(${siblings.indexOf(current) + 1})` + } + } + path.unshift(seg) + current = current.parentElement + if (path.length >= 4) { + break + } + } + return path.join(' > ') + } + + /** Extract text from immediate text-node children only (not nested elements). */ + function getDirectText(el: HTMLElement): string { + let text = '' + for (const child of Array.from(el.childNodes)) { + if (child.nodeType === 3 /* TEXT_NODE */) { + text += child.textContent + } + } + return text.trim().replace(/\s+/g, ' ') + } + + function isVisible(el: HTMLElement): boolean { + if (typeof el.checkVisibility === 'function') { + return el.checkVisibility({ + opacityProperty: true, + visibilityProperty: true, + contentVisibilityAuto: true + }) + } + const style = window.getComputedStyle(el) + return ( + style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0' && + el.offsetWidth > 0 && + el.offsetHeight > 0 + ) + } + + function isInViewport(el: HTMLElement): boolean { + const rect = el.getBoundingClientRect() + return ( + rect.top >= 0 && + rect.left >= 0 && + rect.bottom <= + (window.innerHeight || document.documentElement.clientHeight) && + rect.right <= + (window.innerWidth || document.documentElement.clientWidth) + ) + } + + function getLevel(el: HTMLElement): number | undefined { + const m = el.tagName.toLowerCase().match(/^h([1-6])$/) + if (m) { + return parseInt(m[1], 10) + } + const ariaLevel = el.getAttribute('aria-level') + if (ariaLevel) { + return parseInt(ariaLevel, 10) + } + return undefined + } + + function getState(el: HTMLElement): Record { + const inputEl = el as HTMLInputElement + const isCheckable = + ['input', 'menuitemcheckbox', 'menuitemradio'].includes( + el.tagName.toLowerCase() + ) || + ['checkbox', 'radio', 'switch'].includes(el.getAttribute('role') || '') + return { + disabled: + el.getAttribute('aria-disabled') === 'true' || inputEl.disabled + ? 'true' + : '', + checked: + isCheckable && inputEl.checked + ? 'true' + : el.getAttribute('aria-checked') || '', + expanded: el.getAttribute('aria-expanded') || '', + selected: el.getAttribute('aria-selected') || '', + pressed: el.getAttribute('aria-pressed') || '', + required: + inputEl.required || el.getAttribute('aria-required') === 'true' + ? 'true' + : '', + readonly: + inputEl.readOnly || el.getAttribute('aria-readonly') === 'true' + ? 'true' + : '' + } + } + + type RawNode = Record + + const result: RawNode[] = [] + + function walk(el: HTMLElement, depth = 0): void { + if (depth > 200) { + return + } + if (!isVisible(el)) { + return + } + + const role = getRole(el) + const inViewport = isInViewport(el) + + if (!role) { + for (const child of Array.from(el.children)) { + walk(child as HTMLElement, depth + 1) + } + return + } + + // When viewport filtering is on, skip nodes outside the viewport. + // Still recurse into children — they may have different positioning + // (e.g. position:fixed elements inside an off-screen container). + if (inViewportOnly && !inViewport) { + for (const child of Array.from(el.children)) { + walk(child as HTMLElement, depth + 1) + } + return + } + + const name = getAccessibleName(el, role) + // Always generate a selector — even elements without an accessible + // name need a CSS-path fallback so the snapshot doesn't lose them. + const selector = getSelector(el) + const node: RawNode = { + role, + name, + selector, + depth, + level: getLevel(el) ?? '', + isInViewport: inViewport, + ...getState(el) + } + result.push(node) + + for (const child of Array.from(el.children)) { + walk(child as HTMLElement, depth + 1) + } + } + + for (const child of Array.from(document.body.children)) { + walk(child as HTMLElement, 0) + } + + return result + })() + +/** + * Get browser accessibility tree via a single DOM walk. + * + * @param browser WebdriverIO browser instance + * @param options {@link inViewportOnly} defaults to `true` — only nodes + * whose bounding rect intersects the viewport are included. + */ +export async function getBrowserAccessibilityTree( + browser: WebdriverIO.Browser, + options: { inViewportOnly?: boolean } = {} +): Promise { + const { inViewportOnly = true } = options + return (browser as any).execute( + accessibilityTreeScript, + inViewportOnly + ) as unknown as Promise +} diff --git a/packages/elements/src/browser-elements.ts b/packages/elements/src/browser-elements.ts new file mode 100644 index 00000000..0e38e5ac --- /dev/null +++ b/packages/elements/src/browser-elements.ts @@ -0,0 +1,303 @@ +/** + * Browser element detection + * Single browser.execute() call: querySelectorAll → flat interactable element list + * + * NOTE: This script runs in browser context via browser.execute() + * It must be self-contained with no external dependencies + */ + +export interface BrowserElementInfo { + tagName: string + name: string // computed accessible name (ARIA spec) + type: string + value: string + href: string + selector: string + isInViewport: boolean + boundingBox?: { x: number; y: number; width: number; height: number } +} + +export interface GetBrowserElementsOptions { + includeBounds?: boolean + /** Only return elements whose bounding rect intersects the viewport (default true). */ + inViewportOnly?: boolean +} + +const elementsScript = (includeBounds: boolean, inViewportOnly: boolean) => + (function () { + const interactableSelectors = [ + 'a[href]', + 'button', + 'input:not([type="hidden"])', + 'select', + 'textarea', + '[role="button"]', + '[role="link"]', + '[role="checkbox"]', + '[role="radio"]', + '[role="tab"]', + '[role="menuitem"]', + '[role="combobox"]', + '[role="option"]', + '[role="switch"]', + '[role="slider"]', + '[role="textbox"]', + '[role="searchbox"]', + '[role="spinbutton"]', + '[contenteditable="true"]', + '[tabindex]:not([tabindex="-1"])' + ].join(',') + + function isVisible(element: HTMLElement): boolean { + if (typeof element.checkVisibility === 'function') { + return element.checkVisibility({ + opacityProperty: true, + visibilityProperty: true, + contentVisibilityAuto: true + }) + } + const style = window.getComputedStyle(element) + return ( + style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0' && + element.offsetWidth > 0 && + element.offsetHeight > 0 + ) + } + + function getAccessibleName(el: HTMLElement): string { + // 1. aria-label + const ariaLabel = el.getAttribute('aria-label') + if (ariaLabel) { + return ariaLabel.trim() + } + + // 2. aria-labelledby — resolve referenced elements + const labelledBy = el.getAttribute('aria-labelledby') + if (labelledBy) { + const texts = labelledBy + .split(/\s+/) + .map((id) => document.getElementById(id)?.textContent?.trim() || '') + .filter(Boolean) + if (texts.length > 0) { + return texts.join(' ').slice(0, 200) + } + } + + const tag = el.tagName.toLowerCase() + + // 3. alt for images and input[type=image] + if ( + tag === 'img' || + (tag === 'input' && el.getAttribute('type') === 'image') + ) { + const alt = el.getAttribute('alt') + if (alt !== null) { + return alt.trim() + } + } + + // 4. label[for=id] for form elements + if (['input', 'select', 'textarea'].includes(tag)) { + const id = el.getAttribute('id') + if (id) { + const label = document.querySelector(`label[for="${CSS.escape(id)}"]`) + if (label) { + return label.textContent?.trim() || '' + } + } + // 5. Wrapping label — clone, strip inputs, read text + const parentLabel = el.closest('label') + if (parentLabel) { + const clone = parentLabel.cloneNode(true) as HTMLElement + clone + .querySelectorAll('input,select,textarea') + .forEach((n) => n.remove()) + const lt = clone.textContent?.trim() + if (lt) { + return lt + } + } + } + + // 6. placeholder + const ph = el.getAttribute('placeholder') + if (ph) { + return ph.trim() + } + + // 7. title + const title = el.getAttribute('title') + if (title) { + return title.trim() + } + + // 8. text content (truncated, whitespace normalized) + return (el.textContent?.trim().replace(/\s+/g, ' ') || '').slice(0, 200) + } + + function getSelector(element: HTMLElement): string { + const tag = element.tagName.toLowerCase() + + // 1. tag*=Text — best per WebdriverIO docs + const text = element.textContent?.trim().replace(/\s+/g, ' ') + if (text && text.length > 0 && text.length <= 120) { + const sameTagElements = document.querySelectorAll(tag) + let matchCount = 0 + sameTagElements.forEach((el) => { + if (el.textContent?.includes(text)) { + matchCount++ + } + }) + if (matchCount === 1) { + return `${tag}*=${text}` + } + } + + // 2. aria/label + const ariaLabel = element.getAttribute('aria-label') + if (ariaLabel && ariaLabel.length <= 200) { + const sel = `[aria-label="${CSS.escape(ariaLabel)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + // 3. data-testid + const testId = element.getAttribute('data-testid') + if (testId) { + const sel = `[data-testid="${CSS.escape(testId)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + // 4. #id + if (element.id) { + return `#${CSS.escape(element.id)}` + } + + // 5. [name] — form elements + const nameAttr = element.getAttribute('name') + if (nameAttr) { + const sel = `${tag}[name="${CSS.escape(nameAttr)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + // 6. tag.class — try each class individually, then first-two combination + if (element.className && typeof element.className === 'string') { + const classes = element.className.trim().split(/\s+/).filter(Boolean) + for (const cls of classes) { + const sel = `${tag}.${CSS.escape(cls)}` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + if (classes.length >= 2) { + const sel = `${tag}${classes + .slice(0, 2) + .map((c) => `.${CSS.escape(c)}`) + .join('')}` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + } + + // 7. CSS path fallback + let current: HTMLElement | null = element + const path: string[] = [] + while (current && current !== document.documentElement) { + let seg = current.tagName.toLowerCase() + if (current.id) { + path.unshift(`#${CSS.escape(current.id)}`) + break + } + const parent = current.parentElement + if (parent) { + const siblings = Array.from(parent.children).filter( + (c) => c.tagName === current!.tagName + ) + if (siblings.length > 1) { + seg += `:nth-of-type(${siblings.indexOf(current) + 1})` + } + } + path.unshift(seg) + current = current.parentElement + if (path.length >= 4) { + break + } + } + return path.join(' > ') + } + + const elements: Record[] = [] + const seen = new Set() + + document.querySelectorAll(interactableSelectors).forEach((el) => { + if (seen.has(el)) { + return + } + seen.add(el) + + const htmlEl = el as HTMLElement + if (!isVisible(htmlEl)) { + return + } + + const inputEl = htmlEl as HTMLInputElement + const rect = htmlEl.getBoundingClientRect() + const isInViewport = + rect.top >= 0 && + rect.left >= 0 && + rect.bottom <= + (window.innerHeight || document.documentElement.clientHeight) && + rect.right <= + (window.innerWidth || document.documentElement.clientWidth) + + if (inViewportOnly && !isInViewport) { + return + } + + const entry: Record = { + tagName: htmlEl.tagName.toLowerCase(), + name: getAccessibleName(htmlEl), + type: htmlEl.getAttribute('type') || '', + value: inputEl.value || '', + href: htmlEl.getAttribute('href') || '', + selector: getSelector(htmlEl), + isInViewport + } + + if (includeBounds) { + entry.boundingBox = { + x: rect.x + window.scrollX, + y: rect.y + window.scrollY, + width: rect.width, + height: rect.height + } + } + + elements.push(entry) + }) + + return elements + })() + +/** + * Get interactable browser elements via querySelectorAll. + */ +export async function getInteractableBrowserElements( + browser: WebdriverIO.Browser, + options: GetBrowserElementsOptions = {} +): Promise { + const { includeBounds = false, inViewportOnly = true } = options + return (browser as any).execute( + elementsScript, + includeBounds, + inViewportOnly + ) as unknown as Promise +} diff --git a/packages/elements/src/get-elements.ts b/packages/elements/src/get-elements.ts new file mode 100644 index 00000000..e763a1ff --- /dev/null +++ b/packages/elements/src/get-elements.ts @@ -0,0 +1,67 @@ +import { getInteractableBrowserElements } from './browser-elements.js' +import { getMobileVisibleElementsWithTree } from './mobile-elements.js' +import type { JSONElement } from './locators/types.js' + +export type VisibleElementsResult = { + total: number + showing: number + hasMore: boolean + elements: unknown[] + /** Raw JSON element tree — only present for mobile (android/ios) sessions */ + tree?: JSONElement +} + +export async function getElements( + browser: WebdriverIO.Browser, + params: { + inViewportOnly?: boolean + includeContainers?: boolean + includeBounds?: boolean + limit?: number + offset?: number + } +): Promise { + const { + inViewportOnly = true, + includeContainers = false, + includeBounds = false, + limit = 0, + offset = 0 + } = params + + let elements: { isInViewport?: boolean }[] + let tree: JSONElement | undefined + + if (browser.isAndroid || browser.isIOS) { + const platform = browser.isAndroid ? 'android' : 'ios' + const result = await getMobileVisibleElementsWithTree(browser, platform, { + includeContainers, + includeBounds, + inViewportOnly + }) + elements = result.elements + tree = result.tree ?? undefined + } else { + elements = await getInteractableBrowserElements(browser, { + includeBounds, + inViewportOnly + }) + } + + const total = elements.length + + if (offset > 0) { + elements = elements.slice(offset) + } + if (limit > 0) { + elements = elements.slice(0, limit) + } + + return { + total, + showing: elements.length, + hasMore: offset + elements.length < total, + elements, + ...(tree !== undefined ? { tree } : {}) + } +} diff --git a/packages/elements/src/index.ts b/packages/elements/src/index.ts new file mode 100644 index 00000000..7aeabf78 --- /dev/null +++ b/packages/elements/src/index.ts @@ -0,0 +1,21 @@ +export { getInteractableBrowserElements } from './browser-elements.js' +export type { + BrowserElementInfo, + GetBrowserElementsOptions +} from './browser-elements.js' + +export { getBrowserAccessibilityTree } from './accessibility-tree.js' +export type { AccessibilityNode } from './accessibility-tree.js' + +export { getMobileVisibleElements } from './mobile-elements.js' +export type { + MobileElementInfo, + GetMobileElementsOptions +} from './mobile-elements.js' + +export { getElements } from './get-elements.js' +export type { VisibleElementsResult } from './get-elements.js' + +export { serializeWebSnapshot, serializeMobileSnapshot } from './snapshot.js' +export type { WebSnapshotOptions, MobileSnapshotOptions } from './snapshot.js' +export type { JSONElement } from './locators/types.js' diff --git a/packages/elements/src/locators/constants.ts b/packages/elements/src/locators/constants.ts new file mode 100644 index 00000000..540784b3 --- /dev/null +++ b/packages/elements/src/locators/constants.ts @@ -0,0 +1,169 @@ +/** + * Platform-specific element tag constants for mobile automation + */ + +export const ANDROID_INTERACTABLE_TAGS = [ + // Input elements + 'android.widget.EditText', + 'android.widget.AutoCompleteTextView', + 'android.widget.MultiAutoCompleteTextView', + 'android.widget.SearchView', + + // Button-like elements + 'android.widget.Button', + 'android.widget.ImageButton', + 'android.widget.ToggleButton', + 'android.widget.CompoundButton', + 'android.widget.RadioButton', + 'android.widget.CheckBox', + 'android.widget.Switch', + 'android.widget.FloatingActionButton', + 'com.google.android.material.button.MaterialButton', + 'com.google.android.material.floatingactionbutton.FloatingActionButton', + + // Text elements (often tappable) + 'android.widget.TextView', + 'android.widget.CheckedTextView', + + // Image elements (often tappable) + 'android.widget.ImageView', + 'android.widget.QuickContactBadge', + + // Selection elements + 'android.widget.Spinner', + 'android.widget.SeekBar', + 'android.widget.RatingBar', + 'android.widget.ProgressBar', + 'android.widget.DatePicker', + 'android.widget.TimePicker', + 'android.widget.NumberPicker', + + // List/grid items + 'android.widget.AdapterView' +] + +export const ANDROID_LAYOUT_CONTAINERS = [ + // Core ViewGroup classes + 'android.view.ViewGroup', + 'android.view.View', + 'android.widget.FrameLayout', + 'android.widget.LinearLayout', + 'android.widget.RelativeLayout', + 'android.widget.GridLayout', + 'android.widget.TableLayout', + 'android.widget.TableRow', + 'android.widget.AbsoluteLayout', + + // AndroidX layout classes + 'androidx.constraintlayout.widget.ConstraintLayout', + 'androidx.coordinatorlayout.widget.CoordinatorLayout', + 'androidx.appcompat.widget.LinearLayoutCompat', + 'androidx.cardview.widget.CardView', + 'androidx.appcompat.widget.ContentFrameLayout', + 'androidx.appcompat.widget.FitWindowsFrameLayout', + + // Scrolling containers + 'android.widget.ScrollView', + 'android.widget.HorizontalScrollView', + 'android.widget.NestedScrollView', + 'androidx.core.widget.NestedScrollView', + 'androidx.recyclerview.widget.RecyclerView', + 'android.widget.ListView', + 'android.widget.GridView', + 'android.widget.AbsListView', + + // App chrome / system elements + 'android.widget.ActionBarContainer', + 'android.widget.ActionBarOverlayLayout', + 'android.view.ViewStub', + 'androidx.appcompat.widget.ActionBarContainer', + 'androidx.appcompat.widget.ActionBarContextView', + 'androidx.appcompat.widget.ActionBarOverlayLayout', + + // Decor views + 'com.android.internal.policy.DecorView', + 'android.widget.DecorView' +] + +export const IOS_INTERACTABLE_TAGS = [ + // Input elements + 'XCUIElementTypeTextField', + 'XCUIElementTypeSecureTextField', + 'XCUIElementTypeTextView', + 'XCUIElementTypeSearchField', + + // Button-like elements + 'XCUIElementTypeButton', + 'XCUIElementTypeLink', + + // Text elements (often tappable) + 'XCUIElementTypeStaticText', + + // Image elements + 'XCUIElementTypeImage', + 'XCUIElementTypeIcon', + + // Selection elements + 'XCUIElementTypeSwitch', + 'XCUIElementTypeSlider', + 'XCUIElementTypeStepper', + 'XCUIElementTypeSegmentedControl', + 'XCUIElementTypePicker', + 'XCUIElementTypePickerWheel', + 'XCUIElementTypeDatePicker', + 'XCUIElementTypePageIndicator', + + // Table/list items + 'XCUIElementTypeCell', + 'XCUIElementTypeMenuItem', + 'XCUIElementTypeMenuBarItem', + + // Toggle elements + 'XCUIElementTypeCheckBox', + 'XCUIElementTypeRadioButton', + 'XCUIElementTypeToggle', + + // Other interactive + 'XCUIElementTypeKey', + 'XCUIElementTypeKeyboard', + 'XCUIElementTypeAlert', + 'XCUIElementTypeSheet' +] + +export const IOS_LAYOUT_CONTAINERS = [ + // Generic containers + 'XCUIElementTypeOther', + 'XCUIElementTypeGroup', + 'XCUIElementTypeLayoutItem', + + // Scroll containers + 'XCUIElementTypeScrollView', + 'XCUIElementTypeTable', + 'XCUIElementTypeCollectionView', + 'XCUIElementTypeScrollBar', + + // Navigation chrome + 'XCUIElementTypeNavigationBar', + 'XCUIElementTypeTabBar', + 'XCUIElementTypeToolbar', + 'XCUIElementTypeStatusBar', + 'XCUIElementTypeMenuBar', + + // Windows and views + 'XCUIElementTypeWindow', + 'XCUIElementTypeSheet', + 'XCUIElementTypeDrawer', + 'XCUIElementTypeDialog', + 'XCUIElementTypePopover', + 'XCUIElementTypePopUpButton', + + // Outline elements + 'XCUIElementTypeOutline', + 'XCUIElementTypeOutlineRow', + 'XCUIElementTypeBrowser', + 'XCUIElementTypeSplitGroup', + 'XCUIElementTypeSplitter', + + // Application root + 'XCUIElementTypeApplication' +] diff --git a/packages/elements/src/locators/element-filter.ts b/packages/elements/src/locators/element-filter.ts new file mode 100644 index 00000000..d249f3a2 --- /dev/null +++ b/packages/elements/src/locators/element-filter.ts @@ -0,0 +1,234 @@ +/** + * Element filtering logic for mobile automation + */ + +import type { JSONElement, FilterOptions } from './types.js' +import { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS, + ANDROID_LAYOUT_CONTAINERS, + IOS_LAYOUT_CONTAINERS +} from './constants.js' + +/** + * Check if element tag matches any in the list (handles partial matches) + */ +function matchesTagList(tagName: string, tagList: string[]): boolean { + if (tagList.includes(tagName)) { + return true + } + + for (const tag of tagList) { + if (tagName.endsWith(tag) || tagName.includes(tag)) { + return true + } + } + + return false +} + +/** + * Check if element matches tag name filters + */ +function matchesTagFilters( + element: JSONElement, + includeTagNames: string[], + excludeTagNames: string[] +): boolean { + if ( + includeTagNames.length > 0 && + !matchesTagList(element.tagName, includeTagNames) + ) { + return false + } + + if (matchesTagList(element.tagName, excludeTagNames)) { + return false + } + + return true +} + +/** + * Check if element matches attribute-based filters + */ +function matchesAttributeFilters( + element: JSONElement, + requireAttributes: string[], + minAttributeCount: number +): boolean { + if (requireAttributes.length > 0) { + const hasRequiredAttr = requireAttributes.some( + (attr) => element.attributes?.[attr] + ) + if (!hasRequiredAttr) { + return false + } + } + + if (element.attributes && minAttributeCount > 0) { + const attrCount = Object.values(element.attributes).filter( + (v) => v !== undefined && v !== null && v !== '' + ).length + if (attrCount < minAttributeCount) { + return false + } + } + + return true +} + +/** + * Check if element is interactable based on platform + */ +export function isInteractableElement( + element: JSONElement, + _isNative: boolean, + automationName: string +): boolean { + const isAndroid = automationName.toLowerCase().includes('uiautomator') + const interactableTags = isAndroid + ? ANDROID_INTERACTABLE_TAGS + : IOS_INTERACTABLE_TAGS + + if (matchesTagList(element.tagName, interactableTags)) { + return true + } + + if (isAndroid) { + if ( + element.attributes?.clickable === 'true' || + element.attributes?.focusable === 'true' || + element.attributes?.checkable === 'true' || + element.attributes?.['long-clickable'] === 'true' + ) { + return true + } + } + + if (!isAndroid) { + if (element.attributes?.accessible === 'true') { + return true + } + } + + return false +} + +/** + * Check if element is a layout container + */ +export function isLayoutContainer( + element: JSONElement, + platform: 'android' | 'ios' +): boolean { + const containerList = + platform === 'android' ? ANDROID_LAYOUT_CONTAINERS : IOS_LAYOUT_CONTAINERS + return matchesTagList(element.tagName, containerList) +} + +/** + * Check if element has meaningful content (text, accessibility info) + */ +export function hasMeaningfulContent( + element: JSONElement, + platform: 'android' | 'ios' +): boolean { + const attrs = element.attributes + + if (attrs.text && attrs.text.trim() !== '' && attrs.text !== 'null') { + return true + } + + if (platform === 'android') { + if ( + attrs['content-desc'] && + attrs['content-desc'].trim() !== '' && + attrs['content-desc'] !== 'null' + ) { + return true + } + } else { + if (attrs.label && attrs.label.trim() !== '' && attrs.label !== 'null') { + return true + } + if (attrs.name && attrs.name.trim() !== '' && attrs.name !== 'null') { + return true + } + } + + return false +} + +/** + * Determine if an element should be included based on all filter criteria + */ +export function shouldIncludeElement( + element: JSONElement, + filters: FilterOptions, + isNative: boolean, + automationName: string +): boolean { + const { + includeTagNames = [], + excludeTagNames = ['hierarchy'], + requireAttributes = [], + minAttributeCount = 0, + fetchableOnly = false, + clickableOnly = false, + visibleOnly = true + } = filters + + if (!matchesTagFilters(element, includeTagNames, excludeTagNames)) { + if (element.attributes?.clickable !== 'true') { + return false + } + } + + if (!matchesAttributeFilters(element, requireAttributes, minAttributeCount)) { + return false + } + + if (clickableOnly && element.attributes?.clickable !== 'true') { + return false + } + + if (visibleOnly) { + const isAndroid = automationName.toLowerCase().includes('uiautomator') + if (isAndroid && element.attributes?.displayed === 'false') { + return false + } + if (!isAndroid && element.attributes?.visible === 'false') { + return false + } + } + + if ( + fetchableOnly && + !isInteractableElement(element, isNative, automationName) + ) { + return false + } + + return true +} + +/** + * Get default filter options for a platform + */ +export function getDefaultFilters( + platform: 'android' | 'ios', + includeContainers: boolean = false +): FilterOptions { + const layoutContainers = + platform === 'android' ? ANDROID_LAYOUT_CONTAINERS : IOS_LAYOUT_CONTAINERS + + return { + excludeTagNames: includeContainers + ? ['hierarchy'] + : ['hierarchy', ...layoutContainers], + fetchableOnly: !includeContainers, + visibleOnly: true, + clickableOnly: false + } +} diff --git a/packages/elements/src/locators/index.ts b/packages/elements/src/locators/index.ts new file mode 100644 index 00000000..20e2330c --- /dev/null +++ b/packages/elements/src/locators/index.ts @@ -0,0 +1,279 @@ +/** + * Mobile element locator generation + * + * Main orchestrator module that coordinates XML parsing, element filtering, + * and locator generation for mobile automation. + * + * Based on: https://github.com/appium/appium-mcp + */ + +// Types +export type { + ElementAttributes, + JSONElement, + Bounds, + FilterOptions, + UniquenessResult, + LocatorStrategy, + LocatorContext, + ElementWithLocators, + GenerateLocatorsOptions +} from './types.js' + +// Constants +export { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS, + ANDROID_LAYOUT_CONTAINERS, + IOS_LAYOUT_CONTAINERS +} from './constants.js' + +// XML Parsing +export { + xmlToJSON, + xmlToDOM, + evaluateXPath, + checkXPathUniqueness, + findDOMNodeByPath, + parseAndroidBounds, + parseIOSBounds, + flattenElementTree, + countAttributeOccurrences, + isAttributeUnique +} from './xml-parsing.js' + +// Element Filtering +export { + isInteractableElement, + isLayoutContainer, + hasMeaningfulContent, + shouldIncludeElement, + getDefaultFilters +} from './element-filter.js' + +// Locator Generation +export { + getSuggestedLocators, + getBestLocator, + locatorsToObject +} from './locator-generation.js' + +import type { + JSONElement, + FilterOptions, + LocatorStrategy, + ElementWithLocators, + GenerateLocatorsOptions, + XMLDocument +} from './types.js' + +import { + xmlToJSON, + xmlToDOM, + parseAndroidBounds, + parseIOSBounds, + findDOMNodeByPath +} from './xml-parsing.js' +import { + shouldIncludeElement, + isLayoutContainer, + hasMeaningfulContent +} from './element-filter.js' +import { getSuggestedLocators, locatorsToObject } from './locator-generation.js' + +interface ProcessingContext { + sourceXML: string + platform: 'android' | 'ios' + automationName: string + isNative: boolean + viewportSize: { width: number; height: number } + filters: FilterOptions + inViewportOnly: boolean + results: ElementWithLocators[] + parsedDOM: XMLDocument | null +} + +/** + * Parse element bounds based on platform + */ +function parseBounds( + element: JSONElement, + platform: 'android' | 'ios' +): { x: number; y: number; width: number; height: number } { + return platform === 'android' + ? parseAndroidBounds(element.attributes.bounds || '') + : parseIOSBounds(element.attributes) +} + +/** + * Check if bounds are within viewport + */ +function isWithinViewport( + bounds: { x: number; y: number; width: number; height: number }, + viewport: { width: number; height: number } +): boolean { + return ( + bounds.x >= 0 && + bounds.y >= 0 && + bounds.width > 0 && + bounds.height > 0 && + bounds.x + bounds.width <= viewport.width && + bounds.y + bounds.height <= viewport.height + ) +} + +/** + * Transform JSONElement to ElementWithLocators + */ +function transformElement( + element: JSONElement, + locators: [LocatorStrategy, string][], + ctx: ProcessingContext +): ElementWithLocators { + const attrs = element.attributes + const bounds = parseBounds(element, ctx.platform) + + return { + tagName: element.tagName, + locators: locatorsToObject(locators), + text: attrs.text || attrs.label || '', + contentDesc: attrs['content-desc'] || '', + resourceId: attrs['resource-id'] || '', + accessibilityId: attrs.name || attrs['content-desc'] || '', + label: attrs.label || '', + value: attrs.value || '', + className: attrs.class || element.tagName, + clickable: + attrs.clickable === 'true' || + attrs.accessible === 'true' || + attrs['long-clickable'] === 'true', + enabled: attrs.enabled !== 'false', + displayed: + ctx.platform === 'android' + ? attrs.displayed !== 'false' + : attrs.visible !== 'false', + bounds, + isInViewport: isWithinViewport(bounds, ctx.viewportSize) + } +} + +/** + * Check if element should be processed + */ +function shouldProcess(element: JSONElement, ctx: ProcessingContext): boolean { + if ( + shouldIncludeElement(element, ctx.filters, ctx.isNative, ctx.automationName) + ) { + return true + } + return ( + isLayoutContainer(element, ctx.platform) && + hasMeaningfulContent(element, ctx.platform) + ) +} + +/** + * Process a single element and add to results if valid + */ +function processElement(element: JSONElement, ctx: ProcessingContext): void { + if (!shouldProcess(element, ctx)) { + return + } + + // Skip off-screen elements early when viewport filtering is on — + // avoids expensive locator generation for elements the caller doesn't want. + if (ctx.inViewportOnly) { + const b = parseBounds(element, ctx.platform) + if (!isWithinViewport(b, ctx.viewportSize)) { + return + } + } + + try { + const targetNode = ctx.parsedDOM + ? findDOMNodeByPath(ctx.parsedDOM, element.path) + : undefined + + const locators = getSuggestedLocators( + element, + ctx.sourceXML, + ctx.automationName, + { + sourceXML: ctx.sourceXML, + parsedDOM: ctx.parsedDOM, + isAndroid: ctx.platform === 'android' + }, + targetNode || undefined + ) + if (locators.length === 0) { + return + } + + // Stash the best locator on the tree node so serializeMobileSnapshot + // can reuse the full locator pipeline instead of recomputing. + element.attributes._selector = locators[0][1] + + const transformed = transformElement(element, locators, ctx) + if (Object.keys(transformed.locators).length === 0) { + return + } + + ctx.results.push(transformed) + } catch (error) { + console.error(`[processElement] Error at path ${element.path}:`, error) + } +} + +/** + * Recursively traverse and process element tree + */ +function traverseTree( + element: JSONElement | null, + ctx: ProcessingContext +): void { + if (!element) { + return + } + + processElement(element, ctx) + + for (const child of element.children || []) { + traverseTree(child, ctx) + } +} + +/** + * Generate locators for all elements from page source XML + */ +export function generateAllElementLocators( + sourceXML: string, + options: GenerateLocatorsOptions +): ElementWithLocators[] { + const sourceJSON = xmlToJSON(sourceXML) + + if (!sourceJSON) { + console.error( + '[generateAllElementLocators] Failed to parse page source XML' + ) + return [] + } + + const parsedDOM = xmlToDOM(sourceXML) + + const ctx: ProcessingContext = { + sourceXML, + platform: options.platform, + automationName: + options.platform === 'android' ? 'uiautomator2' : 'xcuitest', + isNative: options.isNative ?? true, + viewportSize: options.viewportSize ?? { width: 9999, height: 9999 }, + filters: options.filters ?? {}, + inViewportOnly: options.inViewportOnly ?? true, + results: [], + parsedDOM + } + + traverseTree(sourceJSON, ctx) + + return ctx.results +} diff --git a/packages/elements/src/locators/locator-generation.ts b/packages/elements/src/locators/locator-generation.ts new file mode 100644 index 00000000..cba05cc8 --- /dev/null +++ b/packages/elements/src/locators/locator-generation.ts @@ -0,0 +1,644 @@ +/** + * Locator strategy generation for mobile elements + */ + +import type { + JSONElement, + LocatorStrategy, + LocatorContext, + UniquenessResult, + XMLNode, + XMLDocument +} from './types.js' +import type { Element as XMLElement } from '@xmldom/xmldom' +import { + checkXPathUniqueness, + evaluateXPath, + isAttributeUnique +} from './xml-parsing.js' + +/** + * Check if a string value is valid for use in a locator + */ +function isValidValue(value: string | undefined): value is string { + return ( + value !== undefined && + value !== null && + value !== 'null' && + value.trim() !== '' + ) +} + +/** + * Escape special characters in text for use in selectors + */ +function escapeText(text: string): string { + return text.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n') +} + +/** + * Escape value for use in XPath expressions + */ +function escapeXPathValue(value: string): string { + if (!value.includes("'")) { + return `'${value}'` + } + if (!value.includes('"')) { + return `"${value}"` + } + const parts: string[] = [] + let current = '' + for (const char of value) { + if (char === "'") { + if (current) { + parts.push(`'${current}'`) + } + parts.push('"\'"') + current = '' + } else { + current += char + } + } + if (current) { + parts.push(`'${current}'`) + } + return `concat(${parts.join(',')})` +} + +/** + * Wrap non-unique XPath with index + */ +function generateIndexedXPath(baseXPath: string, index: number): string { + return `(${baseXPath})[${index}]` +} + +/** + * Add .instance(n) for UiAutomator (0-based) + */ +function generateIndexedUiAutomator( + baseSelector: string, + index: number +): string { + return `${baseSelector}.instance(${index - 1})` +} + +/** + * Check uniqueness, falling back to regex if no DOM available + */ +function checkUniqueness( + ctx: LocatorContext, + xpath: string, + targetNode?: XMLNode +): UniquenessResult { + if (ctx.parsedDOM) { + return checkXPathUniqueness(ctx.parsedDOM, xpath, targetNode) + } + + const match = xpath.match(/\/\/\*\[@([^=]+)="([^"]+)"\]/) + if (match) { + const [, attr, value] = match + return { isUnique: isAttributeUnique(ctx.sourceXML, attr, value) } + } + return { isUnique: false } +} + +/** + * Get sibling index (1-based) among same-tag siblings + */ +function getSiblingIndex(element: XMLElement): number { + const parent = element.parentNode + if (!parent) { + return 1 + } + + const tagName = element.nodeName + let index = 0 + + for (let i = 0; i < parent.childNodes.length; i++) { + const child = parent.childNodes.item(i) + if (child?.nodeType === 1 && child.nodeName === tagName) { + index++ + if (child === element) { + return index + } + } + } + + return 1 +} + +/** + * Count siblings with same tag name + */ +function countSiblings(element: XMLElement): number { + const parent = element.parentNode + if (!parent) { + return 1 + } + + const tagName = element.nodeName + let count = 0 + + for (let i = 0; i < parent.childNodes.length; i++) { + const child = parent.childNodes.item(i) + if (child?.nodeType === 1 && child.nodeName === tagName) { + count++ + } + } + + return count +} + +/** + * Find unique attribute for element in XPath format + */ +function findUniqueAttribute( + element: XMLElement, + ctx: LocatorContext +): string | null { + const attrs = ctx.isAndroid + ? ['resource-id', 'content-desc', 'text'] + : ['name', 'label', 'value'] + + for (const attr of attrs) { + const value = element.getAttribute(attr) + if (value && value.trim()) { + const xpath = `//*[@${attr}=${escapeXPathValue(value)}]` + const result = ctx.parsedDOM + ? checkXPathUniqueness(ctx.parsedDOM, xpath) + : { isUnique: isAttributeUnique(ctx.sourceXML, attr, value) } + + if (result.isUnique) { + return `@${attr}=${escapeXPathValue(value)}` + } + } + } + + return null +} + +/** + * Build hierarchical XPath by traversing up the DOM tree + */ +function buildHierarchicalXPath( + ctx: LocatorContext, + element: XMLElement, + maxDepth: number = 3 +): string | null { + if (!ctx.parsedDOM) { + return null + } + + const pathParts: string[] = [] + let current: XMLElement | null = element + let depth = 0 + + while (current && depth < maxDepth) { + const tagName = current.nodeName + const uniqueAttr = findUniqueAttribute(current, ctx) + + if (uniqueAttr) { + pathParts.unshift(`//${tagName}[${uniqueAttr}]`) + break + } else { + const siblingIndex = getSiblingIndex(current) + const siblingCount = countSiblings(current) + + if (siblingCount > 1) { + pathParts.unshift(`${tagName}[${siblingIndex}]`) + } else { + pathParts.unshift(tagName) + } + } + + const parent = current.parentNode as XMLElement | null + current = parent && parent.nodeType === 1 ? parent : null + depth++ + } + + if (pathParts.length === 0) { + return null + } + + let result = pathParts[0] + for (let i = 1; i < pathParts.length; i++) { + result += '/' + pathParts[i] + } + + if (!result.startsWith('//')) { + result = '//' + result + } + + return result +} + +/** + * Add XPath locator with uniqueness checking and fallbacks + */ +function addXPathLocator( + results: [LocatorStrategy, string][], + xpath: string, + ctx: LocatorContext, + targetNode?: XMLNode +): void { + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + if (uniqueness.isUnique) { + results.push(['xpath', xpath]) + } else if (uniqueness.index) { + results.push(['xpath', generateIndexedXPath(xpath, uniqueness.index)]) + } else { + if (targetNode && ctx.parsedDOM) { + // @xmldom/xmldom 0.9+ XMLNode doesn't satisfy global Node; safe at runtime + const hierarchical = buildHierarchicalXPath( + ctx, + targetNode as unknown as XMLElement + ) + if (hierarchical) { + results.push(['xpath', hierarchical]) + } + } + results.push(['xpath', xpath]) + } +} + +/** + * Check if element is within UiAutomator scope + */ +function isInUiAutomatorScope( + element: JSONElement, + doc: XMLDocument | null +): boolean { + if (!doc) { + return true + } + + const hierarchyNodes = evaluateXPath(doc, '/hierarchy/*') + if (hierarchyNodes.length === 0) { + return true + } + + const lastIndex = hierarchyNodes.length + const pathParts = element.path.split('.') + if (pathParts.length === 0 || pathParts[0] === '') { + return true + } + + const firstIndex = parseInt(pathParts[0], 10) + return firstIndex === lastIndex - 1 +} + +/** + * Build Android UiAutomator selector with multiple attributes + */ +function buildUiAutomatorSelector(element: JSONElement): string | null { + const attrs = element.attributes + const parts: string[] = [] + + if (isValidValue(attrs['resource-id'])) { + parts.push(`resourceId("${attrs['resource-id']}")`) + } + if (isValidValue(attrs.text) && attrs.text!.length < 100) { + parts.push(`text("${escapeText(attrs.text!)}")`) + } + if (isValidValue(attrs['content-desc'])) { + parts.push(`description("${attrs['content-desc']}")`) + } + if (isValidValue(attrs.class)) { + parts.push(`className("${attrs.class}")`) + } + + if (parts.length === 0) { + return null + } + + return `android=new UiSelector().${parts.join('.')}` +} + +/** + * Build iOS predicate string with multiple conditions + */ +function buildPredicateString(element: JSONElement): string | null { + const attrs = element.attributes + const conditions: string[] = [] + + if (isValidValue(attrs.name)) { + conditions.push(`name == "${escapeText(attrs.name!)}"`) + } + if (isValidValue(attrs.label)) { + conditions.push(`label == "${escapeText(attrs.label!)}"`) + } + if (isValidValue(attrs.value)) { + conditions.push(`value == "${escapeText(attrs.value!)}"`) + } + if (attrs.visible === 'true') { + conditions.push('visible == 1') + } + if (attrs.enabled === 'true') { + conditions.push('enabled == 1') + } + + if (conditions.length === 0) { + return null + } + + return `-ios predicate string:${conditions.join(' AND ')}` +} + +/** + * Build iOS class chain selector + */ +function buildClassChain(element: JSONElement): string | null { + const attrs = element.attributes + const tagName = element.tagName + + if (!tagName.startsWith('XCUI')) { + return null + } + + let selector = `**/${tagName}` + + if (isValidValue(attrs.label)) { + selector += `[\`label == "${escapeText(attrs.label!)}"\`]` + } else if (isValidValue(attrs.name)) { + selector += `[\`name == "${escapeText(attrs.name!)}"\`]` + } + + return `-ios class chain:${selector}` +} + +/** + * Build XPath for element with unique identification + */ +function buildXPath( + element: JSONElement, + _sourceXML: string, + isAndroid: boolean +): string | null { + const attrs = element.attributes + const tagName = element.tagName + const conditions: string[] = [] + + if (isAndroid) { + if (isValidValue(attrs['resource-id'])) { + conditions.push(`@resource-id="${attrs['resource-id']}"`) + } + if (isValidValue(attrs['content-desc'])) { + conditions.push(`@content-desc="${attrs['content-desc']}"`) + } + if (isValidValue(attrs.text) && attrs.text!.length < 100) { + conditions.push(`@text="${escapeText(attrs.text!)}"`) + } + } else { + if (isValidValue(attrs.name)) { + conditions.push(`@name="${attrs.name}"`) + } + if (isValidValue(attrs.label)) { + conditions.push(`@label="${attrs.label}"`) + } + if (isValidValue(attrs.value)) { + conditions.push(`@value="${attrs.value}"`) + } + } + + if (conditions.length === 0) { + return `//${tagName}` + } + + return `//${tagName}[${conditions.join(' and ')}]` +} + +/** + * Get simple locators based on single attributes + */ +function getSimpleSuggestedLocators( + element: JSONElement, + ctx: LocatorContext, + automationName: string, + targetNode?: XMLNode +): [LocatorStrategy, string][] { + const results: [LocatorStrategy, string][] = [] + const isAndroid = automationName.toLowerCase().includes('uiautomator') + const attrs = element.attributes + const inUiAutomatorScope = isAndroid + ? isInUiAutomatorScope(element, ctx.parsedDOM) + : true + + if (isAndroid) { + // Resource ID + const resourceId = attrs['resource-id'] + if (isValidValue(resourceId)) { + const xpath = `//*[@resource-id="${resourceId}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique && inUiAutomatorScope) { + results.push([ + 'id', + `android=new UiSelector().resourceId("${resourceId}")` + ]) + } else if (uniqueness.index && inUiAutomatorScope) { + const base = `android=new UiSelector().resourceId("${resourceId}")` + results.push(['id', generateIndexedUiAutomator(base, uniqueness.index)]) + } + } + + // Content Description + const contentDesc = attrs['content-desc'] + if (isValidValue(contentDesc)) { + const xpath = `//*[@content-desc="${contentDesc}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push(['accessibility-id', `~${contentDesc}`]) + } + } + + // Text + const text = attrs.text + if (isValidValue(text) && text.length < 100) { + const xpath = `//*[@text="${escapeText(text)}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique && inUiAutomatorScope) { + results.push([ + 'text', + `android=new UiSelector().text("${escapeText(text)}")` + ]) + } else if (uniqueness.index && inUiAutomatorScope) { + const base = `android=new UiSelector().text("${escapeText(text)}")` + results.push([ + 'text', + generateIndexedUiAutomator(base, uniqueness.index) + ]) + } + } + } else { + // iOS: Accessibility ID (name) + const name = attrs.name + if (isValidValue(name)) { + const xpath = `//*[@name="${name}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push(['accessibility-id', `~${name}`]) + } + } + + // iOS: Label + const label = attrs.label + if (isValidValue(label) && label !== attrs.name) { + const xpath = `//*[@label="${escapeText(label)}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push([ + 'predicate-string', + `-ios predicate string:label == "${escapeText(label)}"` + ]) + } + } + + // iOS: Value + const value = attrs.value + if (isValidValue(value)) { + const xpath = `//*[@value="${escapeText(value)}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push([ + 'predicate-string', + `-ios predicate string:value == "${escapeText(value)}"` + ]) + } + } + } + + return results +} + +/** + * Get complex locators (combinations, XPath, etc.) + */ +function getComplexSuggestedLocators( + element: JSONElement, + ctx: LocatorContext, + automationName: string, + targetNode?: XMLNode +): [LocatorStrategy, string][] { + const results: [LocatorStrategy, string][] = [] + const isAndroid = automationName.toLowerCase().includes('uiautomator') + const inUiAutomatorScope = isAndroid + ? isInUiAutomatorScope(element, ctx.parsedDOM) + : true + + if (isAndroid) { + if (inUiAutomatorScope) { + const uiAutomator = buildUiAutomatorSelector(element) + if (uiAutomator) { + results.push(['uiautomator', uiAutomator]) + } + } + + const xpath = buildXPath(element, ctx.sourceXML, true) + if (xpath) { + addXPathLocator(results, xpath, ctx, targetNode) + } + + if (inUiAutomatorScope && isValidValue(element.attributes.class)) { + results.push([ + 'class-name', + `android=new UiSelector().className("${element.attributes.class}")` + ]) + } + } else { + const predicate = buildPredicateString(element) + if (predicate) { + results.push(['predicate-string', predicate]) + } + + const classChain = buildClassChain(element) + if (classChain) { + results.push(['class-chain', classChain]) + } + + const xpath = buildXPath(element, ctx.sourceXML, false) + if (xpath) { + addXPathLocator(results, xpath, ctx, targetNode) + } + + const type = element.tagName + if (type.startsWith('XCUIElementType')) { + results.push(['class-name', `-ios class chain:**/${type}`]) + } + } + + return results +} + +/** + * Get all suggested locators for an element + */ +export function getSuggestedLocators( + element: JSONElement, + sourceXML: string, + automationName: string, + ctx?: LocatorContext, + targetNode?: XMLNode +): [LocatorStrategy, string][] { + const locatorCtx = ctx ?? { + sourceXML, + parsedDOM: null, + isAndroid: automationName.toLowerCase().includes('uiautomator') + } + + const simpleLocators = getSimpleSuggestedLocators( + element, + locatorCtx, + automationName, + targetNode + ) + const complexLocators = getComplexSuggestedLocators( + element, + locatorCtx, + automationName, + targetNode + ) + + const seen = new Set() + const results: [LocatorStrategy, string][] = [] + + for (const locator of [...simpleLocators, ...complexLocators]) { + if (!seen.has(locator[1])) { + seen.add(locator[1]) + results.push(locator) + } + } + + return results +} + +/** + * Get the best (first priority) locator for an element + */ +export function getBestLocator( + element: JSONElement, + sourceXML: string, + automationName: string +): string | null { + const locators = getSuggestedLocators(element, sourceXML, automationName) + return locators.length > 0 ? locators[0][1] : null +} + +/** + * Convert locator array to object format + */ +export function locatorsToObject( + locators: [LocatorStrategy, string][] +): Record { + const result: Record = {} + for (const [strategy, value] of locators) { + if (!result[strategy]) { + result[strategy] = value + } + } + return result +} diff --git a/packages/elements/src/locators/types.ts b/packages/elements/src/locators/types.ts new file mode 100644 index 00000000..28f5a497 --- /dev/null +++ b/packages/elements/src/locators/types.ts @@ -0,0 +1,110 @@ +/** + * Type definitions for mobile element locator generation + */ + +import type { Document as XMLDocument, Node as XMLNode } from '@xmldom/xmldom' +export type { XMLDocument, XMLNode } + +export interface ElementAttributes { + // Android attributes + 'resource-id'?: string + 'content-desc'?: string + text?: string + class?: string + package?: string + clickable?: string + 'long-clickable'?: string + focusable?: string + checkable?: string + scrollable?: string + enabled?: string + displayed?: string + bounds?: string // Format: "[x1,y1][x2,y2]" + + // iOS attributes + type?: string + name?: string + label?: string + value?: string + accessible?: string + visible?: string + x?: string + y?: string + width?: string + height?: string + + // Generic + [key: string]: string | undefined +} + +export interface JSONElement { + children: JSONElement[] + tagName: string + attributes: ElementAttributes + path: string // Dot-separated index path for tree traversal +} + +export interface Bounds { + x: number + y: number + width: number + height: number +} + +export interface FilterOptions { + includeTagNames?: string[] // Only include these tags (whitelist) + excludeTagNames?: string[] // Exclude these tags (blacklist) + requireAttributes?: string[] // Must have at least one of these attributes + minAttributeCount?: number // Minimum number of non-empty attributes + fetchableOnly?: boolean // Only interactable elements + clickableOnly?: boolean // Only elements with clickable="true" + visibleOnly?: boolean // Only visible/displayed elements +} + +export interface UniquenessResult { + isUnique: boolean + index?: number // 1-based index if not unique + totalMatches?: number +} + +export type LocatorStrategy = + | 'accessibility-id' + | 'id' + | 'class-name' + | 'xpath' + | 'predicate-string' + | 'class-chain' + | 'uiautomator' + | 'text' + +export interface LocatorContext { + sourceXML: string + parsedDOM: XMLDocument | null + isAndroid: boolean +} + +export interface ElementWithLocators { + tagName: string + locators: Record + text: string + contentDesc: string + resourceId: string + accessibilityId: string + label: string + value: string + className: string + clickable: boolean + enabled: boolean + displayed: boolean + bounds: Bounds + isInViewport: boolean +} + +export interface GenerateLocatorsOptions { + platform: 'android' | 'ios' + viewportSize?: { width: number; height: number } + filters?: FilterOptions + isNative?: boolean + /** Only return elements whose bounds intersect the viewport (default true). */ + inViewportOnly?: boolean +} diff --git a/packages/elements/src/locators/xml-parsing.ts b/packages/elements/src/locators/xml-parsing.ts new file mode 100644 index 00000000..a100a042 --- /dev/null +++ b/packages/elements/src/locators/xml-parsing.ts @@ -0,0 +1,329 @@ +/** + * XML parsing utilities for mobile element source + */ + +import { DOMParser } from '@xmldom/xmldom' +import type { + Document as XMLDocument, + Element as XMLElement, + Node as XMLNode +} from '@xmldom/xmldom' +import xpath from 'xpath' +import type { + ElementAttributes, + JSONElement, + Bounds, + UniquenessResult +} from './types.js' + +/** + * Get child nodes that are elements (not text nodes, comments, etc.) + */ +function childNodesOf(node: XMLNode): XMLNode[] { + const children: XMLNode[] = [] + if (node.childNodes) { + for (let i = 0; i < node.childNodes.length; i++) { + const child = node.childNodes.item(i) + if (child?.nodeType === 1) { + children.push(child) + } + } + } + return children +} + +/** + * Recursively translate DOM node to JSONElement + */ +function translateRecursively( + domNode: XMLNode, + parentPath: string = '', + index: number | null = null +): JSONElement { + const attributes: ElementAttributes = {} + + const element = domNode as XMLElement + if (element.attributes) { + for (let attrIdx = 0; attrIdx < element.attributes.length; attrIdx++) { + const attr = element.attributes.item(attrIdx) + if (attr) { + attributes[attr.name] = attr.value.replace(/(\n)/gm, '\\n') + } + } + } + + const path = + index === null ? '' : `${parentPath ? parentPath + '.' : ''}${index}` + + return { + children: childNodesOf(domNode).map((childNode, childIndex) => + translateRecursively(childNode as XMLNode, path, childIndex) + ), + tagName: domNode.nodeName, + attributes, + path + } +} + +/** + * Compare two nodes for equality by platform-specific attributes + * (reference equality via === may fail when nodes come from different traversals) + */ +function isSameElement(node1: XMLNode, node2: XMLNode): boolean { + if (node1.nodeType !== 1 || node2.nodeType !== 1) { + return false + } + const el1 = node1 as XMLElement + const el2 = node2 as XMLElement + + if (el1.nodeName !== el2.nodeName) { + return false + } + + // For Android, compare by bounds (unique per element) + const bounds1 = el1.getAttribute('bounds') + const bounds2 = el2.getAttribute('bounds') + if (bounds1 && bounds2) { + return bounds1 === bounds2 + } + + // For iOS, compare by x, y, width, height + const x1 = el1.getAttribute('x') + const y1 = el1.getAttribute('y') + const x2 = el2.getAttribute('x') + const y2 = el2.getAttribute('y') + if (x1 && y1 && x2 && y2) { + return ( + x1 === x2 && + y1 === y2 && + el1.getAttribute('width') === el2.getAttribute('width') && + el1.getAttribute('height') === el2.getAttribute('height') + ) + } + + return false +} + +/** + * Convert XML page source to JSON tree structure + */ +export function xmlToJSON(sourceXML: string): JSONElement | null { + try { + const parser = new DOMParser() + const sourceDoc = parser.parseFromString(sourceXML, 'text/xml') + + // xmldom 0.9+ throws ParseError for fatal errors (caught below); this catches non-fatal cases + const parseErrors = sourceDoc.getElementsByTagName('parsererror') + if (parseErrors.length > 0) { + console.error( + '[xmlToJSON] XML parsing error:', + parseErrors[0].textContent + ) + return null + } + + const children = childNodesOf(sourceDoc) + const firstChild = + children[0] || + (sourceDoc.documentElement + ? childNodesOf(sourceDoc.documentElement)[0] + : null) + + return firstChild + ? translateRecursively(firstChild) + : { children: [], tagName: '', attributes: {}, path: '' } + } catch (e) { + console.error('[xmlToJSON] Failed to parse XML:', e) + return null + } +} + +/** + * Parse XML source to DOM Document for XPath evaluation + */ +export function xmlToDOM(sourceXML: string): XMLDocument | null { + try { + const parser = new DOMParser() + const doc = parser.parseFromString(sourceXML, 'text/xml') + + // xmldom 0.9+ throws ParseError for fatal errors (caught below); this catches non-fatal cases + const parseErrors = doc.getElementsByTagName('parsererror') + if (parseErrors.length > 0) { + console.error('[xmlToDOM] XML parsing error:', parseErrors[0].textContent) + return null + } + + return doc + } catch (e) { + console.error('[xmlToDOM] Failed to parse XML:', e) + return null + } +} + +/** + * Execute XPath query on DOM document + */ +export function evaluateXPath(doc: XMLDocument, xpathExpr: string): XMLNode[] { + try { + // @xmldom/xmldom 0.9+ types don't satisfy global Node; xpath still works at runtime + const nodes = xpath.select(xpathExpr, doc as unknown as Node) + if (Array.isArray(nodes)) { + return nodes as unknown as XMLNode[] + } + return [] + } catch (e) { + console.error(`[evaluateXPath] Failed to evaluate "${xpathExpr}":`, e) + return [] + } +} + +/** + * Check if an XPath selector is unique and get index if not + */ +export function checkXPathUniqueness( + doc: XMLDocument, + xpathExpr: string, + targetNode?: XMLNode +): UniquenessResult { + try { + const nodes = evaluateXPath(doc, xpathExpr) + const totalMatches = nodes.length + + if (totalMatches === 0) { + return { isUnique: false } + } + + if (totalMatches === 1) { + return { isUnique: true } + } + + // Not unique - find index of target node if provided + if (targetNode) { + for (let i = 0; i < nodes.length; i++) { + if (nodes[i] === targetNode || isSameElement(nodes[i], targetNode)) { + return { + isUnique: false, + index: i + 1, // 1-based index for XPath + totalMatches + } + } + } + } + + return { isUnique: false, totalMatches } + } catch (e) { + console.error(`[checkXPathUniqueness] Error checking "${xpathExpr}":`, e) + return { isUnique: false } + } +} + +/** + * Find DOM node by JSONElement path (e.g., "0.2.1") + */ +export function findDOMNodeByPath( + doc: XMLDocument, + path: string +): XMLNode | null { + if (!path) { + return doc.documentElement + } + + const indices = path.split('.').map(Number) + let current: XMLNode | null = doc.documentElement + + for (const index of indices) { + if (!current) { + return null + } + + const children: XMLNode[] = [] + if (current.childNodes) { + for (let i = 0; i < current.childNodes.length; i++) { + const child = current.childNodes.item(i) + if (child?.nodeType === 1) { + children.push(child) + } + } + } + + current = children[index] || null + } + + return current +} + +/** + * Parse Android bounds string "[x1,y1][x2,y2]" to coordinates + */ +export function parseAndroidBounds(bounds: string): Bounds { + const match = bounds.match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/) + if (!match) { + return { x: 0, y: 0, width: 0, height: 0 } + } + + const x1 = parseInt(match[1], 10) + const y1 = parseInt(match[2], 10) + const x2 = parseInt(match[3], 10) + const y2 = parseInt(match[4], 10) + + return { + x: x1, + y: y1, + width: x2 - x1, + height: y2 - y1 + } +} + +/** + * Parse iOS element bounds from individual x, y, width, height attributes + */ +export function parseIOSBounds(attributes: ElementAttributes): Bounds { + return { + x: parseInt(attributes.x || '0', 10), + y: parseInt(attributes.y || '0', 10), + width: parseInt(attributes.width || '0', 10), + height: parseInt(attributes.height || '0', 10) + } +} + +/** + * Flatten JSON element tree to array (depth-first) + */ +export function flattenElementTree(root: JSONElement): JSONElement[] { + const result: JSONElement[] = [] + + function traverse(element: JSONElement) { + result.push(element) + for (const child of element.children) { + traverse(child) + } + } + + traverse(root) + return result +} + +/** + * Count occurrences of an attribute value in the source XML + */ +export function countAttributeOccurrences( + sourceXML: string, + attribute: string, + value: string +): number { + const escapedValue = value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const pattern = new RegExp(`${attribute}=["']${escapedValue}["']`, 'g') + const matches = sourceXML.match(pattern) + return matches ? matches.length : 0 +} + +/** + * Check if an attribute value is unique in the source (fast regex-based check) + */ +export function isAttributeUnique( + sourceXML: string, + attribute: string, + value: string +): boolean { + return countAttributeOccurrences(sourceXML, attribute, value) === 1 +} diff --git a/packages/elements/src/mobile-elements.ts b/packages/elements/src/mobile-elements.ts new file mode 100644 index 00000000..1eabbc53 --- /dev/null +++ b/packages/elements/src/mobile-elements.ts @@ -0,0 +1,199 @@ +/** + * Mobile element detection utilities for iOS and Android + * + * Uses page source parsing for optimal performance (2 HTTP calls vs 600+ for 50 elements) + */ + +import type { + ElementWithLocators, + FilterOptions, + JSONElement, + LocatorStrategy +} from './locators/index.js' +import { + generateAllElementLocators, + getDefaultFilters, + xmlToJSON +} from './locators/index.js' + +/** + * Element info returned by getMobileVisibleElements + * Uses uniform fields (all elements have same keys) to enable TOON tabular format + */ +export interface MobileElementInfo { + selector: string + tagName: string + isInViewport: boolean + text: string + resourceId: string + accessibilityId: string + isEnabled: boolean + altSelector: string // Single alternative selector (flattened for tabular format) + // Only present when includeBounds=true + bounds?: { x: number; y: number; width: number; height: number } +} + +/** + * Options for getMobileVisibleElements + */ +export interface GetMobileElementsOptions { + includeContainers?: boolean + includeBounds?: boolean + /** Only return elements whose bounds intersect the viewport (default true). */ + inViewportOnly?: boolean + filterOptions?: FilterOptions +} + +/** + * Locator strategy priority order for selecting best selector + * Earlier = higher priority + */ +const LOCATOR_PRIORITY: LocatorStrategy[] = [ + 'accessibility-id', // Most stable, cross-platform + 'id', // Android resource-id + 'text', // Text-based (can be fragile but readable) + 'predicate-string', // iOS predicate + 'class-chain', // iOS class chain + 'uiautomator', // Android UiAutomator compound + 'xpath' // XPath (last resort, brittle) + // 'class-name' intentionally excluded - too generic +] + +/** + * Select best locators from available strategies + * Returns [primarySelector, ...alternativeSelectors] + */ +function selectBestLocators(locators: Record): string[] { + const selected: string[] = [] + + // Find primary selector based on priority + for (const strategy of LOCATOR_PRIORITY) { + if (locators[strategy]) { + selected.push(locators[strategy]) + break + } + } + + // Add one alternative if available (different strategy) + for (const strategy of LOCATOR_PRIORITY) { + if (locators[strategy] && !selected.includes(locators[strategy])) { + selected.push(locators[strategy]) + break + } + } + + return selected +} + +/** + * Convert ElementWithLocators to MobileElementInfo + * Uses uniform fields (all elements have same keys) to enable CSV tabular format + */ +function toMobileElementInfo( + element: ElementWithLocators, + includeBounds: boolean +): MobileElementInfo { + const selectedLocators = selectBestLocators(element.locators) + + // Use contentDesc for accessibilityId on Android, or name on iOS + const accessId = element.accessibilityId || element.contentDesc + + // Build object with ALL fields for uniform schema (enables CSV tabular format) + // Empty string '' used for missing values to keep schema consistent + const info: MobileElementInfo = { + selector: selectedLocators[0] || '', + tagName: element.tagName, + isInViewport: element.isInViewport, + text: element.text || '', + resourceId: element.resourceId || '', + accessibilityId: accessId || '', + isEnabled: element.enabled !== false, + altSelector: selectedLocators[1] || '' // Single alternative (flattened for tabular) + } + + // Only include bounds if explicitly requested (adds 4 extra columns) + if (includeBounds) { + info.bounds = element.bounds + } + + return info +} + +/** + * Get viewport size from browser + */ +async function getViewportSize( + browser: WebdriverIO.Browser +): Promise<{ width: number; height: number }> { + try { + const size = await browser.getWindowSize() + return { width: size.width, height: size.height } + } catch { + return { width: 9999, height: 9999 } + } +} + +/** + * Get all visible elements from a mobile app, also returning the raw JSON element tree. + * Single parse of page source: tree and flat list share one xmlToJSON call. + * + * Performance: 2 HTTP calls (getWindowSize + getPageSource) vs 12+ per element with legacy approach + */ +export async function getMobileVisibleElementsWithTree( + browser: WebdriverIO.Browser, + platform: 'ios' | 'android', + options: GetMobileElementsOptions = {} +): Promise<{ elements: MobileElementInfo[]; tree: JSONElement | null }> { + const { + includeContainers = false, + includeBounds = false, + inViewportOnly = true, + filterOptions + } = options + + const viewportSize = await getViewportSize(browser) + const pageSource = await browser.getPageSource() + + const filters: FilterOptions = { + ...getDefaultFilters(platform, includeContainers), + ...filterOptions + } + + const tree = xmlToJSON(pageSource) + + // Stash the source XML on the root element so serializeMobileSnapshot + // can use the full locator pipeline without requiring it as a separate arg. + if (tree) { + tree.attributes._sourceXML = pageSource + } + + const elementLocators = generateAllElementLocators(pageSource, { + platform, + viewportSize, + filters, + inViewportOnly + }) + + const elements = elementLocators.map((el) => + toMobileElementInfo(el, includeBounds) + ) + return { elements, tree } +} + +/** + * Get all visible elements from a mobile app + * + * Performance: 2 HTTP calls (getWindowSize + getPageSource) vs 12+ per element with legacy approach + */ +export async function getMobileVisibleElements( + browser: WebdriverIO.Browser, + platform: 'ios' | 'android', + options: GetMobileElementsOptions = {} +): Promise { + const { elements } = await getMobileVisibleElementsWithTree( + browser, + platform, + options + ) + return elements +} diff --git a/packages/elements/src/snapshot.ts b/packages/elements/src/snapshot.ts new file mode 100644 index 00000000..ad45c1c4 --- /dev/null +++ b/packages/elements/src/snapshot.ts @@ -0,0 +1,758 @@ +/** + * AI-readable snapshot serializers + * + * Converts accessibility trees and mobile element trees into depth-indented + * text files that LLMs can consume without any parsing. + */ + +import type { AccessibilityNode } from './accessibility-tree.js' +import type { JSONElement } from './locators/types.js' +import { parseAndroidBounds, parseIOSBounds } from './locators/xml-parsing.js' +import { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS +} from './locators/constants.js' +import { getSuggestedLocators } from './locators/locator-generation.js' + +/** + * Roles that can be interacted with — rendered with `→ selector`. + * Structural roles (heading, img, form, nav, …) are intentionally excluded. + */ +const INTERACTIVE_ROLES = new Set([ + 'button', + 'link', + 'textbox', + 'checkbox', + 'radio', + 'combobox', + 'slider', + 'searchbox', + 'spinbutton', + 'switch', + 'tab', + 'menuitem', + 'option' +]) + +/** + * Walk backwards from `index` to find the nearest ancestor or preceding + * structural sibling with a non-empty name. Same-depth nodes are only + * used when they are structural (img, heading, statictext, …) — never + * another interactive element. + */ +function inferPurpose( + nodes: AccessibilityNode[], + index: number +): string | undefined { + const myDepth = nodes[index].depth + for (let i = index - 1; i >= 0; i--) { + if (nodes[i].depth <= myDepth && nodes[i].name) { + // Same-depth sibling: only structural elements count + if (nodes[i].depth === myDepth && INTERACTIVE_ROLES.has(nodes[i].role)) { + continue + } + return nodes[i].name + } + } + return undefined +} + +export interface WebSnapshotOptions { + /** Only include nodes whose bounding rect intersects the viewport (default true). */ + inViewportOnly?: boolean +} + +/** + * Serialize a web accessibility tree into a depth-indented text snapshot. + * + * @param nodes Flat ordered node list from getBrowserAccessibilityTree() + * @param context Optional page context for the header line + * @param options {@link WebSnapshotOptions} + */ +export function serializeWebSnapshot( + nodes: AccessibilityNode[], + context?: { url?: string; title?: string }, + options: WebSnapshotOptions = {} +): string { + const { inViewportOnly = true } = options + + let header = '[Page' + if (context?.title) { + header += `: ${context.title}` + } + if (context?.url) { + header += ` — ${context.url}` + } + header += ']' + + const lines: string[] = [header] + + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + + // When viewport filtering is on, skip nodes that are known to be off-screen. + // Nodes from a tree captured with inViewportOnly=false will have + // isInViewport populated; nodes from a pre-filtered tree all have + // isInViewport=true (or undefined for pre-existing data). + if (inViewportOnly && node.isInViewport === false) { + continue + } + + const indent = ' '.repeat(node.depth + 1) // +1 indents everything under the header + const isInteractive = INTERACTIVE_ROLES.has(node.role) + + // Skip statictext that merely echoes the parent link/button name. + // Example: link "Highlights" → a*=Highlights doesn't need + // statictext "Highlights" as a child because it adds no information. + if (node.role === 'statictext' && node.name) { + let echoedByParent = false + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + const parentRole = nodes[j].role + const parentName = nodes[j].name + if ( + INTERACTIVE_ROLES.has(parentRole) && + parentName && + parentName.includes(node.name) + ) { + echoedByParent = true + } + break // only check the immediate structural parent + } + } + if (echoedByParent) { + continue + } + } + + // Heading gets level suffix: heading[2] + const roleLabel = + node.role === 'heading' && node.level + ? `heading[${node.level}]` + : node.role + + if (isInteractive) { + // No selector → agent can't act on this node; skip entirely + if (!node.selector) { + continue + } + const purpose = inferPurpose(nodes, i) + if (node.name) { + // Show parent context when available — disambiguates + // duplicate selectors like six "Add to Wishlist" buttons. + lines.push( + purpose + ? `${indent}${roleLabel} "${node.name}" ∈ "${purpose}" → ${node.selector}` + : `${indent}${roleLabel} "${node.name}" → ${node.selector}` + ) + } else if (purpose) { + lines.push(`${indent}${roleLabel} ∈ "${purpose}" → ${node.selector}`) + } else { + lines.push(`${indent}${roleLabel} → ${node.selector}`) + } + } else { + // Container / structural: show role + name when present, no selector + lines.push( + node.name + ? `${indent}${roleLabel} "${node.name}"` + : `${indent}${roleLabel}` + ) + } + } + + return lines.join('\n') +} + +// --------------------------------------------------------------------------- +// Mobile snapshot helpers +// --------------------------------------------------------------------------- + +/** Shorten fully-qualified Android/iOS class names to the last segment. */ +function simplifyTag(tagName: string): string { + const dot = tagName.lastIndexOf('.') + if (dot !== -1) { + return tagName.slice(dot + 1) + } + return tagName.replace(/^XCUIElementType/, '') +} + +// --------------------------------------------------------------------------- +// Mobile role classification — maps raw Android/iOS class names to semantic +// roles so the snapshot reads like the web version (button, textbox, img, …). +// --------------------------------------------------------------------------- + +const ANDROID_ROLE_MAP: Record = { + 'android.widget.Button': 'button', + 'android.widget.ImageButton': 'button', + 'android.widget.ToggleButton': 'button', + 'android.widget.FloatingActionButton': 'button', + 'com.google.android.material.button.MaterialButton': 'button', + 'com.google.android.material.floatingactionbutton.FloatingActionButton': + 'button', + 'android.widget.EditText': 'textbox', + 'android.widget.AutoCompleteTextView': 'textbox', + 'android.widget.MultiAutoCompleteTextView': 'textbox', + 'android.widget.SearchView': 'searchbox', + 'android.widget.ImageView': 'img', + 'android.widget.QuickContactBadge': 'img', + 'android.widget.CheckBox': 'checkbox', + 'android.widget.RadioButton': 'radio', + 'android.widget.Switch': 'switch', + 'android.widget.Spinner': 'combobox', + 'android.widget.SeekBar': 'slider', + 'android.widget.RatingBar': 'slider', + 'android.widget.ProgressBar': 'progressbar', + 'android.widget.TextView': 'statictext', + 'android.widget.CheckedTextView': 'statictext', + 'android.widget.RecyclerView': 'list', + 'android.widget.ListView': 'list', + 'android.widget.GridView': 'list', + 'android.webkit.WebView': 'webview' +} + +const IOS_ROLE_MAP: Record = { + XCUIElementTypeButton: 'button', + XCUIElementTypeLink: 'link', + XCUIElementTypeTextField: 'textbox', + XCUIElementTypeSecureTextField: 'textbox', + XCUIElementTypeTextView: 'textbox', + XCUIElementTypeSearchField: 'searchbox', + XCUIElementTypeImage: 'img', + XCUIElementTypeIcon: 'img', + XCUIElementTypeSwitch: 'switch', + XCUIElementTypeSlider: 'slider', + XCUIElementTypeStepper: 'slider', + XCUIElementTypeCheckBox: 'checkbox', + XCUIElementTypeRadioButton: 'radio', + XCUIElementTypePicker: 'combobox', + XCUIElementTypePickerWheel: 'combobox', + XCUIElementTypeDatePicker: 'combobox', + XCUIElementTypeSegmentedControl: 'combobox', + XCUIElementTypeStaticText: 'statictext', + XCUIElementTypeCell: 'listitem', + XCUIElementTypeTable: 'list', + XCUIElementTypeCollectionView: 'list' +} + +function classifyMobileRole( + tagName: string, + platform: 'android' | 'ios' +): string { + if (platform === 'android') { + return ANDROID_ROLE_MAP[tagName] || simplifyTag(tagName) + } + return IOS_ROLE_MAP[tagName] || simplifyTag(tagName) +} + +// --------------------------------------------------------------------------- +// Locator generation +// --------------------------------------------------------------------------- + +function getBestAndroidLocator( + attrs: JSONElement['attributes'] +): string | undefined { + // Pre-computed by the full locator pipeline (generateAllElementLocators). + // Takes priority over the simplified fallback logic below. + if (attrs._selector) { + return attrs._selector + } + // ~ prefix = accessibility-id shorthand in WebdriverIO ($('~foo')) + if (attrs['content-desc']) { + return `~${attrs['content-desc']}` + } + if (attrs['resource-id']) { + return `id:${attrs['resource-id']}` + } + if (attrs.text) { + return `~${attrs.text}` + } + // Fallback: class-based locator (only useful with :nth-of-type or index) + if (attrs.class) { + return `class:${simplifyTag(attrs.class)}` + } + return undefined +} + +function getBestIOSLocator( + attrs: JSONElement['attributes'] +): string | undefined { + // Pre-computed by the full locator pipeline. + if (attrs._selector) { + return attrs._selector + } + // ~ prefix = accessibility-id shorthand (maps to `name` on iOS) + if (attrs.name) { + return `~${attrs.name}` + } + if (attrs.label) { + return `~${attrs.label}` + } + if (attrs.value) { + return `~${attrs.value}` + } + // Fallback: class-based locator + if (attrs.type) { + return `class:${simplifyTag(attrs.type)}` + } + return undefined +} + +// --------------------------------------------------------------------------- +// Identity +// --------------------------------------------------------------------------- + +function getMobileNodeIdentity( + attrs: JSONElement['attributes'], + platform: 'android' | 'ios' +): string { + if (platform === 'android') { + const contentDesc = attrs['content-desc'] + if (contentDesc) { + return contentDesc + } + if (attrs.text) { + return attrs.text + } + // Fall back to the last segment of the resource-id (e.g. "search_action_bar") + const rid = attrs['resource-id'] + if (rid) { + const slash = rid.lastIndexOf('/') + return slash !== -1 ? rid.slice(slash + 1) : rid + } + return '' + } + return attrs.name || attrs.label || attrs.value || attrs.text || '' +} + +// --------------------------------------------------------------------------- +// Interactivity +// --------------------------------------------------------------------------- + +const ANDROID_INTERACTABLE_SET = new Set(ANDROID_INTERACTABLE_TAGS) +const IOS_INTERACTABLE_SET = new Set(IOS_INTERACTABLE_TAGS) + +/** An element is *explicitly* interactive when it carries a click/focus/check + * attribute — as opposed to being interactive only because its tag is in the + * interactable-tag list. Explicit parents should carry the → selector, not + * their tag-interactive children. */ +function isExplicitlyInteractive( + attrs: JSONElement['attributes'], + platform: 'android' | 'ios' +): boolean { + if (platform === 'android') { + return ( + attrs.clickable === 'true' || + attrs.focusable === 'true' || + attrs.checkable === 'true' || + attrs['long-clickable'] === 'true' + ) + } + return attrs.accessible === 'true' +} + +function isMobileInteractive( + element: JSONElement, + platform: 'android' | 'ios' +): boolean { + const attrs = element.attributes + if (platform === 'android') { + if (ANDROID_INTERACTABLE_SET.has(element.tagName)) { + return true + } + return ( + attrs.clickable === 'true' || + attrs['long-clickable'] === 'true' || + attrs.focusable === 'true' || + attrs.checkable === 'true' + ) + } + if (IOS_INTERACTABLE_SET.has(element.tagName)) { + return true + } + return attrs.accessible === 'true' +} + +// --------------------------------------------------------------------------- +// Viewport +// --------------------------------------------------------------------------- + +interface WalkMobileOptions { + inViewportOnly: boolean + viewport: { width: number; height: number } + /** Raw page-source XML. When provided, the full locator pipeline is used. */ + sourceXML?: string + /** 'uiautomator2' or 'xcuitest'. Required when sourceXML is set. */ + automationName?: string +} + +function isMobileInViewport( + element: JSONElement, + platform: 'android' | 'ios', + viewport: { width: number; height: number } +): boolean { + const bounds = + platform === 'android' + ? parseAndroidBounds(element.attributes.bounds || '') + : parseIOSBounds(element.attributes) + + if (bounds.width === 0 && bounds.height === 0) { + return true + } + + return ( + bounds.x >= 0 && + bounds.y >= 0 && + bounds.width > 0 && + bounds.height > 0 && + bounds.x + bounds.width <= viewport.width && + bounds.y + bounds.height <= viewport.height + ) +} + +// --------------------------------------------------------------------------- +// Flat-node representation (mirrors AccessibilityNode so both pipelines share +// inferPurpose, dedup, and rendering logic). +// --------------------------------------------------------------------------- + +interface MobileFlatNode { + role: string + name: string + selector: string + depth: number + isInteractive: boolean + /** True when the element has clickable/focusable/checkable — the intended tap target. */ + isExplicitInteractive: boolean + isInViewport: boolean +} + +/** + * First pass: walk the JSONElement tree, apply viewport filtering and + * collect every node into a flat array with semantic roles and selectors. + */ +function collectMobileNodes( + element: JSONElement, + platform: 'android' | 'ios', + depth: number, + nodes: MobileFlatNode[], + walkOpts: WalkMobileOptions +): void { + const attrs = element.attributes + const role = classifyMobileRole(element.tagName, platform) + const name = getMobileNodeIdentity(attrs, platform) + const explicit = isExplicitlyInteractive(attrs, platform) + const interactive = isMobileInteractive(element, platform) + const inViewport = isMobileInViewport(element, platform, walkOpts.viewport) + + // Viewport filtering + if (walkOpts.inViewportOnly) { + if (interactive && !inViewport) { + // Skip this node but still recurse (scroll children may be in view). + for (const child of element.children || []) { + collectMobileNodes(child, platform, depth + 1, nodes, walkOpts) + } + return + } + if (!interactive && !inViewport) { + // Collapse off-screen container to a placeholder. + nodes.push({ + role: 'generic', + name: name ? `${role} "${name}"` : role, + selector: '', + depth, + isInteractive: false, + isExplicitInteractive: false, + isInViewport: false + }) + return + } + } + + // Generate a selector for every interactive element. + // Use the full locator pipeline when source XML is available; + // otherwise fall back to the simplified attribute-based heuristics. + let locator = '' + if (interactive) { + if (walkOpts.sourceXML && walkOpts.automationName) { + // Full pipeline: accessible-id, id, text, uiautomator, xpath, class-name + const suggested = getSuggestedLocators( + element, + walkOpts.sourceXML, + walkOpts.automationName, + { + sourceXML: walkOpts.sourceXML, + parsedDOM: null, + isAndroid: platform === 'android' + } + ) + if (suggested.length > 0) { + locator = suggested[0][1] // first = best priority + } + } + if (!locator) { + // Simplified fallback + locator = + (platform === 'android' + ? getBestAndroidLocator(attrs) + : getBestIOSLocator(attrs)) ?? '' + } + } + + nodes.push({ + role, + name, + selector: locator, + depth, + isInteractive: interactive, + isExplicitInteractive: explicit, + isInViewport: inViewport + }) + + for (const child of element.children || []) { + collectMobileNodes(child, platform, depth + 1, nodes, walkOpts) + } +} + +// --------------------------------------------------------------------------- +// Context inference — shared with the web pipeline. +// Same-depth structural siblings (img, statictext, heading, …) provide +// context for following interactive nodes. +// --------------------------------------------------------------------------- + +const MOBILE_STRUCTURAL_ROLES = new Set([ + 'img', + 'heading', + 'list', + 'listitem', + 'webview', + 'progressbar', + 'slider', + 'switch', + 'generic' +]) + +function mobileInferPurpose( + nodes: MobileFlatNode[], + index: number +): string | undefined { + const myDepth = nodes[index].depth + for (let i = index - 1; i >= 0; i--) { + if (nodes[i].depth <= myDepth && nodes[i].name) { + if ( + nodes[i].depth === myDepth && + !MOBILE_STRUCTURAL_ROLES.has(nodes[i].role) + ) { + continue + } + return nodes[i].name + } + } + return undefined +} + +// --------------------------------------------------------------------------- +// When a tag-only-interactive child (e.g. a statictext TextView) sits +// directly under an explicitly-interactive parent (e.g. a clickable +// LinearLayout row), the *parent* should carry the → selector — the +// child is just a label. Suppress the child's interactivity so the +// parent renders as the actionable element. +// --------------------------------------------------------------------------- + +function suppressTagOnlyChildren(nodes: MobileFlatNode[]): void { + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + if (!node.isInteractive || node.isExplicitInteractive) { + continue + } + // Walk up through ALL ancestors looking for an explicitly-interactive + // parent. The immediate depth-1 parent may just be a layout wrapper; + // the real clickable row could be 2-3 levels up. + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + if (nodes[j].isExplicitInteractive) { + node.isInteractive = false + break // found — suppress and stop + } + // keep looking upward through the ancestor chain + } + } + } +} + +// --------------------------------------------------------------------------- +// Render pass: flat nodes into lines with ∈ context, dedup, noise filter, +// and class-instance indexing. +// --------------------------------------------------------------------------- + +/** Layout roles that carry no semantic meaning by themselves. */ +const NOISY_ROLES = new Set([ + 'FrameLayout', + 'LinearLayout', + 'ViewGroup', + 'RelativeLayout', + 'View', + 'CardView', + 'ConstraintLayout', + 'ScrollView' +]) + +/** + * Pre-count selector occurrences so we can attach .instance(N) suffixes + * to duplicate selectors. + */ +function countSelectors(nodes: MobileFlatNode[]): Map { + const counts = new Map() + for (const node of nodes) { + if (node.selector) { + counts.set(node.selector, (counts.get(node.selector) ?? 0) + 1) + } + } + return counts +} + +function renderMobileNodes(nodes: MobileFlatNode[]): string[] { + const lines: string[] = [] + const selectorCounts = countSelectors(nodes) + const selectorIndex = new Map() + + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + const indent = ' '.repeat(node.depth + 1) + + // Collapse anonymous layout containers at depth ≥ 2. + // Keep depth 0-1 structural chrome and any named container. + if ( + NOISY_ROLES.has(node.role) && + !node.name && + node.depth > 1 && + !node.isInteractive + ) { + continue + } + + // Off-screen containers rendered as collapsed placedersen + if (node.isInViewport === false && !node.isInteractive) { + lines.push(`${indent}⋯ ${node.name} (off-screen)`) + continue + } + + // Dedup: skip statictext whose text is echoed by the parent interactive element + if (node.role === 'statictext' && node.name) { + let echoedByParent = false + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + if ( + nodes[j].isInteractive && + nodes[j].name && + nodes[j].name.includes(node.name) + ) { + echoedByParent = true + } + break + } + } + if (echoedByParent) { + continue + } + } + + if (node.isInteractive && node.selector) { + // Append .instance(N) when the same selector repeats + let selector = node.selector + const total = selectorCounts.get(selector) ?? 1 + if (total > 1) { + const idx = selectorIndex.get(selector) ?? 0 + selectorIndex.set(selector, idx + 1) + selector = `${selector}.instance(${idx})` + } + + const purpose = mobileInferPurpose(nodes, i) + if (node.name) { + lines.push( + purpose + ? `${indent}${node.role} "${node.name}" ∈ "${purpose}" → ${selector}` + : `${indent}${node.role} "${node.name}" → ${selector}` + ) + } else if (purpose) { + lines.push(`${indent}${node.role} ∈ "${purpose}" → ${selector}`) + } else { + lines.push(`${indent}${node.role} → ${selector}`) + } + } else { + // Container / structural / non-locatable + lines.push( + node.name + ? `${indent}${node.role} "${node.name}"` + : `${indent}${node.role}` + ) + } + } + + return lines +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +export interface MobileSnapshotOptions { + /** Only include elements whose bounds intersect the viewport (default true). */ + inViewportOnly?: boolean + /** + * Raw XML page source string. When provided the full locator pipeline + * (getSuggestedLocators) runs on every interactive node, producing the same + * selectors that getElements() returns. Omit to use simplified heuristics. + */ + sourceXML?: string +} + +/** + * Serialize a mobile element tree into a depth-indented text snapshot. + * + * @param root Root JSONElement from the page source XML parse + * @param context Platform, optional device name, viewport, and source XML. + * Include `sourceXML` to use the full locator pipeline. + * @param options {@link MobileSnapshotOptions} + */ +export function serializeMobileSnapshot( + root: JSONElement, + context: { + platform: 'android' | 'ios' + deviceName?: string + viewport?: { width: number; height: number } + /** Raw page-source XML. When set, selectors match getElements() output. */ + sourceXML?: string + }, + options: MobileSnapshotOptions = {} +): string { + const { platform, deviceName, viewport, sourceXML } = context + const { inViewportOnly = true } = options + + // Auto-detect source XML stashed by getMobileVisibleElementsWithTree + const effectiveXML = sourceXML || root.attributes._sourceXML + + const effectiveViewport = viewport ?? { width: 9999, height: 9999 } + const automationName = platform === 'android' ? 'uiautomator2' : 'xcuitest' + + let header = `[${platform}` + if (deviceName) { + header += ` — ${deviceName}` + } + if (viewport) { + header += ` (${viewport.width}×${viewport.height})` + } + header += ']' + + const nodes: MobileFlatNode[] = [] + collectMobileNodes(root, platform, 0, nodes, { + inViewportOnly, + viewport: effectiveViewport, + sourceXML: effectiveXML, + automationName: effectiveXML ? automationName : undefined + }) + + // Let explicitly-interactive parents carry the → selector + suppressTagOnlyChildren(nodes) + + const lines = renderMobileNodes(nodes) + return [header, ...lines].join('\n') +} diff --git a/packages/elements/tests/accessibility-tree.test.ts b/packages/elements/tests/accessibility-tree.test.ts new file mode 100644 index 00000000..6fd7be13 --- /dev/null +++ b/packages/elements/tests/accessibility-tree.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect, vi } from 'vitest' +import { getBrowserAccessibilityTree } from '../src/accessibility-tree.js' + +describe('getBrowserAccessibilityTree', () => { + it('calls browser.execute and returns result', async () => { + const nodes = [ + { + role: 'button', + name: 'Submit', + selector: 'button*=Submit', + depth: 0, + level: '', + disabled: '', + checked: '', + expanded: '', + selected: '', + pressed: '', + required: '', + readonly: '' + } + ] + const mockBrowser = { execute: vi.fn().mockResolvedValue(nodes) } as any + const result = await getBrowserAccessibilityTree(mockBrowser) + expect(mockBrowser.execute).toHaveBeenCalledTimes(1) + expect(result).toEqual(nodes) + }) +}) diff --git a/packages/elements/tests/browser-elements.test.ts b/packages/elements/tests/browser-elements.test.ts new file mode 100644 index 00000000..6fba3af5 --- /dev/null +++ b/packages/elements/tests/browser-elements.test.ts @@ -0,0 +1,22 @@ +import { describe, it, expect, vi } from 'vitest' +import { getInteractableBrowserElements } from '../src/browser-elements.js' + +describe('getInteractableBrowserElements', () => { + it('calls browser.execute with includeBounds=false by default', async () => { + const mockBrowser = { execute: vi.fn().mockResolvedValue([]) } as any + const result = await getInteractableBrowserElements(mockBrowser) + expect(mockBrowser.execute).toHaveBeenCalledTimes(1) + expect(result).toEqual([]) + }) + + it('passes includeBounds option to script', async () => { + const mockBrowser = { + execute: vi.fn().mockResolvedValue([{ tagName: 'button', name: 'OK' }]) + } as any + const result = await getInteractableBrowserElements(mockBrowser, { + includeBounds: true + }) + expect(result).toHaveLength(1) + expect(result[0].tagName).toBe('button') + }) +}) diff --git a/packages/elements/tests/locators/locator-generation.test.ts b/packages/elements/tests/locators/locator-generation.test.ts new file mode 100644 index 00000000..c5d2de4b --- /dev/null +++ b/packages/elements/tests/locators/locator-generation.test.ts @@ -0,0 +1,23 @@ +import { describe, it, expect } from 'vitest' +import { locatorsToObject } from '@wdio/elements/locators' + +describe('locatorsToObject', () => { + it('converts locator array to object', () => { + const locators: [any, string][] = [ + ['accessibility-id', '~Submit'], + ['xpath', '//XCUIElementTypeButton[@name="Submit"]'] + ] + const result = locatorsToObject(locators) + expect(result['accessibility-id']).toBe('~Submit') + expect(result['xpath']).toBe('//XCUIElementTypeButton[@name="Submit"]') + }) + + it('returns first value for duplicate strategies', () => { + const locators: [any, string][] = [ + ['xpath', '//first'], + ['xpath', '//second'] + ] + const result = locatorsToObject(locators) + expect(result['xpath']).toBe('//first') + }) +}) diff --git a/packages/elements/tests/mobile-elements.test.ts b/packages/elements/tests/mobile-elements.test.ts new file mode 100644 index 00000000..64f19861 --- /dev/null +++ b/packages/elements/tests/mobile-elements.test.ts @@ -0,0 +1,13 @@ +import { describe, it, expect, vi } from 'vitest' +import { getMobileVisibleElements } from '../src/mobile-elements.js' + +describe('getMobileVisibleElements', () => { + it('returns empty array for unparseable XML', async () => { + const mockBrowser = { + getWindowSize: vi.fn().mockResolvedValue({ width: 375, height: 812 }), + getPageSource: vi.fn().mockResolvedValue(' & { role: string; depth: number } +): AccessibilityNode { + return { + name: '', + selector: '', + level: '', + disabled: '', + checked: '', + expanded: '', + selected: '', + pressed: '', + required: '', + readonly: '', + ...overrides + } +} + +describe('serializeWebSnapshot', () => { + it('produces a page header', () => { + const out = serializeWebSnapshot([]) + expect(out).toBe('[Page]') + }) + + it('includes title and url in header', () => { + const out = serializeWebSnapshot([], { + title: 'Login', + url: 'https://example.com/login' + }) + expect(out).toMatch('[Page: Login — https://example.com/login]') + }) + + it('renders interactive role with name and selector', () => { + const nodes = [ + node({ + role: 'button', + depth: 0, + name: 'Submit', + selector: 'button*=Submit' + }) + ] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('button "Submit" → button*=Submit') + }) + + it('renders interactive role with ∈ ancestor name when self has no name', () => { + const nodes = [ + node({ role: 'form', depth: 0, name: 'Login form' }), + node({ role: 'checkbox', depth: 1, name: '', selector: '#remember' }) + ] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('checkbox ∈ "Login form" → #remember') + }) + + it('omits interactive node with no selector regardless of name', () => { + const nodes = [ + node({ role: 'button', depth: 0, name: '', selector: '' }), + node({ + role: 'button', + depth: 0, + name: 'Named but unselector', + selector: '' + }) + ] + const out = serializeWebSnapshot(nodes) + // Only the header — both nodes skipped due to missing selector + expect(out.split('\n').length).toBe(1) + }) + + it('omits interactive node with ∈ context but no selector', () => { + const nodes = [ + node({ role: 'form', depth: 0, name: 'Login form' }), + node({ role: 'combobox', depth: 1, name: '', selector: '' }) + ] + const out = serializeWebSnapshot(nodes) + // combobox has ancestor context but no selector — must be dropped + expect(out).not.toContain('combobox') + expect(out).not.toContain('→') + }) + + it('renders container role without selector', () => { + const nodes = [node({ role: 'navigation', depth: 0, name: 'Main' })] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('navigation "Main"') + expect(out).not.toContain('→') + }) + + it('renders heading with level suffix', () => { + const nodes = [ + node({ role: 'heading', depth: 0, name: 'Sign in', level: 1 }) + ] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('heading[1] "Sign in"') + }) + + it('indents nodes according to depth', () => { + const nodes = [ + node({ role: 'navigation', depth: 0, name: 'Nav' }), + node({ role: 'link', depth: 1, name: 'Home', selector: 'a*=Home' }) + ] + const lines = serializeWebSnapshot(nodes).split('\n') + // depth 0 → 1 level of indent (' ' × 1), depth 1 → 2 levels (' ' × 2) + expect(lines[1]).toMatch(/^ navigation/) + expect(lines[2]).toMatch(/^ link/) + }) + + it('renders full login page example correctly', () => { + const nodes: AccessibilityNode[] = [ + node({ role: 'navigation', depth: 0, name: 'Main' }), + node({ role: 'link', depth: 1, name: 'Home', selector: 'a*=Home' }), + node({ role: 'main', depth: 0, name: '' }), + node({ role: 'heading', depth: 1, name: 'Sign in', level: 1 }), + node({ role: 'form', depth: 1, name: 'Login' }), + node({ + role: 'textbox', + depth: 2, + name: 'Email address', + selector: '#email' + }), + node({ + role: 'button', + depth: 2, + name: 'Sign in', + selector: 'button*=Sign in' + }) + ] + const out = serializeWebSnapshot(nodes, { + title: 'Login', + url: 'https://example.com/login' + }) + expect(out).toContain('[Page: Login — https://example.com/login]') + expect(out).toContain('navigation "Main"') + expect(out).toContain('link "Home" ∈ "Main" → a*=Home') + expect(out).toContain('heading[1] "Sign in"') + expect(out).toContain('textbox "Email address" ∈ "Login" → #email') + expect(out).toContain('button "Sign in" ∈ "Login" → button*=Sign in') + }) +}) + +// --------------------------------------------------------------------------- +// serializeMobileSnapshot +// --------------------------------------------------------------------------- + +function mobileEl( + tagName: string, + attrs: JSONElement['attributes'], + children: JSONElement[] = [] +): JSONElement { + return { tagName, attributes: attrs, children, path: '' } +} + +describe('serializeMobileSnapshot', () => { + it('produces a platform header with device and viewport', () => { + const root = mobileEl('hierarchy', {}) + const out = serializeMobileSnapshot(root, { + platform: 'android', + deviceName: 'Pixel 7', + viewport: { width: 412, height: 915 } + }) + expect(out).toMatch('[android — Pixel 7 (412×915)]') + }) + + it('renders interactive Android element with accessibility-id locator', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl('android.widget.Button', { + clickable: 'true', + 'content-desc': 'Skip', + text: '' + }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain('button "Skip" → ~Skip') + }) + + it('falls back to resource-id when no content-desc', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl('android.widget.EditText', { + clickable: 'true', + 'content-desc': '', + 'resource-id': 'com.example:id/search', + text: '' + }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain('textbox "search" → id:com.example:id/search') + }) + + it('renders ∈ ancestor context when element has no identity', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl( + 'android.widget.LinearLayout', + { 'content-desc': 'Search section' }, + [ + mobileEl('android.widget.EditText', { + clickable: 'true', + 'content-desc': '', + 'resource-id': 'com.example:id/search', + text: '' + }) + ] + ) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain( + 'textbox "search" ∈ "Search section" → id:com.example:id/search' + ) + }) + + it('renders iOS element with accessibility-id', () => { + const root = mobileEl('XCUIElementTypeApplication', {}, [ + mobileEl('XCUIElementTypeButton', { + accessible: 'true', + name: 'Accept All Cookies', + label: 'Accept All Cookies' + }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'ios' }) + expect(out).toContain('button "Accept All Cookies" → ~Accept All Cookies') + }) + + it('simplifies iOS XCUIElementType prefix', () => { + const root = mobileEl('XCUIElementTypeApplication', {}, [ + mobileEl('XCUIElementTypeScrollView', {}) + ]) + const out = serializeMobileSnapshot(root, { platform: 'ios' }) + expect(out).toContain('ScrollView') + expect(out).not.toContain('XCUIElementType') + }) + + it('shows container without selector', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl('android.widget.FrameLayout', { 'content-desc': '' }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain('FrameLayout') + expect(out).not.toContain('→') + }) +}) diff --git a/packages/elements/tsconfig.json b/packages/elements/tsconfig.json new file mode 100644 index 00000000..3bcc9638 --- /dev/null +++ b/packages/elements/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "ignoreDeprecations": "6.0", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "noEmit": false, + "allowImportingTsExtensions": false, + "declaration": true, + "skipLibCheck": true, + "types": ["node", "@wdio/globals/types"] + }, + "include": ["src/**/*"] +} diff --git a/packages/nightwatch-devtools/tests/session.test.ts b/packages/nightwatch-devtools/tests/session.test.ts index 19822c97..4f00c948 100644 --- a/packages/nightwatch-devtools/tests/session.test.ts +++ b/packages/nightwatch-devtools/tests/session.test.ts @@ -1,6 +1,8 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { SessionCapturer } from '../src/session.js' -import type { NightwatchBrowser } from '../src/types.js' +import type { CommandLog, NightwatchBrowser } from '../src/types.js' + +type CommandLogWithId = CommandLog & { _id: number } function makeMockBrowser( overrides: Partial> = {} @@ -31,8 +33,8 @@ describe('SessionCapturer.captureCommand', () => { args: ['#btn'], result: { ok: true } }) - expect((cap.commandsLog[0] as { _id: number })._id).not.toBe( - (cap.commandsLog[1] as { _id: number })._id + expect((cap.commandsLog[0] as CommandLogWithId)._id).not.toBe( + (cap.commandsLog[1] as CommandLogWithId)._id ) }) @@ -83,7 +85,7 @@ describe('SessionCapturer.replaceCommand', () => { it('splices the old entry and reissues with a new _id', async () => { const cap = makeCapturer() await cap.captureCommand('click', ['#a'], undefined, undefined) - const oldId = (cap.commandsLog[0] as { _id: number })._id + const oldId = (cap.commandsLog[0] as CommandLogWithId)._id const oldTs = cap.commandsLog[0].timestamp const { entry, oldTimestamp } = cap.replaceCommand( oldId, @@ -94,7 +96,7 @@ describe('SessionCapturer.replaceCommand', () => { ) expect(oldTimestamp).toBe(oldTs) expect(cap.commandsLog).toHaveLength(1) - expect((cap.commandsLog[0] as { _id: number })._id).not.toBe(oldId) + expect((cap.commandsLog[0] as CommandLogWithId)._id).not.toBe(oldId) expect(entry.result).toEqual({ ok: true }) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 55b17f19..6a5c144d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -312,6 +312,31 @@ importers: specifier: ^8.21.0 version: 8.21.0 + packages/elements: + dependencies: + '@xmldom/xmldom': + specifier: ^0.9.8 + version: 0.9.10 + webdriverio: + specifier: ^9.0.0 + version: 9.27.2(puppeteer-core@21.11.0) + xpath: + specifier: ^0.0.34 + version: 0.0.34 + devDependencies: + '@types/node': + specifier: 25.9.1 + version: 25.9.1 + '@wdio/globals': + specifier: 9.27.0 + version: 9.27.0(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0)) + typescript: + specifier: 6.0.2 + version: 6.0.2 + vitest: + specifier: ^4.0.16 + version: 4.1.8(@types/node@25.9.1)(@vitest/coverage-v8@4.1.8)(happy-dom@20.9.0)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) + packages/nightwatch-devtools: dependencies: '@wdio/devtools-backend': @@ -2570,6 +2595,13 @@ packages: resolution: {integrity: sha512-xoBgmACafV4L7e7e3DUN8UM1N+I225oms38JtxtfgrMfvHm8QtcmZWXfycxEGM28Gm2M3NmeV3oso7hZeBk6Ww==} engines: {node: '>=18.20.0'} + '@wdio/globals@9.27.0': + resolution: {integrity: sha512-yT6EAyvEqm+wFD11fg89BMxvFkYLgnIVCihfJx+k73Gm3utL/DfZQpSheQdwrlQzu5p7jHi/JwOD76740F5Peg==} + engines: {node: '>=18.20.0'} + peerDependencies: + expect-webdriverio: ^5.6.5 + webdriverio: ^9.0.0 + '@wdio/globals@9.27.2': resolution: {integrity: sha512-Rx9bqD4/8iR3CNPMWYxywQSCqsR/WGwIYT2Q0uUmrvPxOdYFridDEhVRGO32kQ55UM5+JXzXppxgwGLRQ60fJg==} engines: {node: '>=18.20.0'} @@ -2634,6 +2666,10 @@ packages: resolution: {integrity: sha512-Rj8AP/VYVd5clZFKy+P7zzoXCKshjrog6lcV65nnUzATbUYT/PpUCy6OhEWHTSmLQY2Oc5ztY/IetLSg4nmB3w==} engines: {node: '>=18'} + '@xmldom/xmldom@0.9.10': + resolution: {integrity: sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw==} + engines: {node: '>=14.6'} + '@zip.js/zip.js@2.8.26': resolution: {integrity: sha512-RQ4h9F6DOiHxpdocUDrOl6xBM+yOtz+LkUol47AVWcfebGBDpZ7w7Xvz9PS24JgXvLGiXXzSAfdCdVy1tPlaFA==} engines: {bun: '>=0.7.0', deno: '>=1.0.0', node: '>=18.0.0'} @@ -6739,6 +6775,11 @@ packages: engines: {node: '>=14.17'} hasBin: true + typescript@6.0.2: + resolution: {integrity: sha512-bGdAIrZ0wiGDo5l8c++HWtbaNCWTS4UTv7RaTH/ThVIgjkveJt83m74bBHMJkuCbslY8ixgLBVZJIOiQlQTjfQ==} + engines: {node: '>=14.17'} + hasBin: true + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -7197,6 +7238,10 @@ packages: xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + xpath@0.0.34: + resolution: {integrity: sha512-FxF6+rkr1rNSQrhUNYrAFJpRXNzlDoMxeXN5qI84939ylEv3qqPFKa85Oxr6tDaJKqwW6KKyo2v26TSv3k6LeA==} + engines: {node: '>=0.6.0'} + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -9416,6 +9461,11 @@ snapshots: '@wdio/types': 9.27.2 chalk: 5.6.2 + '@wdio/globals@9.27.0(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0))': + dependencies: + expect-webdriverio: 5.6.7(@wdio/globals@9.27.0)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)) + webdriverio: 9.27.2(puppeteer-core@21.11.0) + '@wdio/globals@9.27.2(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0))': dependencies: expect-webdriverio: 5.6.7(@wdio/globals@9.27.2)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)) @@ -9558,6 +9608,8 @@ snapshots: dependencies: '@wdio/logger': 9.18.0 + '@xmldom/xmldom@0.9.10': {} + '@zip.js/zip.js@2.8.26': {} abort-controller@3.0.0: @@ -11082,6 +11134,16 @@ snapshots: expect-type@1.3.0: {} + expect-webdriverio@5.6.7(@wdio/globals@9.27.0)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)): + dependencies: + '@vitest/snapshot': 4.1.8 + '@wdio/globals': 9.27.0(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0)) + '@wdio/logger': 9.18.0 + deep-eql: 5.0.2 + expect: 30.4.1 + jest-matcher-utils: 30.4.1 + webdriverio: 9.27.2(puppeteer-core@21.11.0) + expect-webdriverio@5.6.7(@wdio/globals@9.27.2)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)): dependencies: '@vitest/snapshot': 4.1.8 @@ -14384,6 +14446,8 @@ snapshots: typescript@5.9.3: optional: true + typescript@6.0.2: {} + typescript@6.0.3: {} ua-parser-js@1.0.41: {} @@ -14833,6 +14897,8 @@ snapshots: xmlchars@2.2.0: {} + xpath@0.0.34: {} + y18n@5.0.8: {} yallist@3.1.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c577713a..038df802 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -2,6 +2,7 @@ packages: - 'packages/shared' - 'packages/core' + - 'packages/elements' - 'packages/backend' - 'packages/script' - 'packages/service' diff --git a/tsconfig.json b/tsconfig.json index 97c599d0..6cd2e8c9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,6 +29,8 @@ "@wdio/devtools-shared/*": ["packages/shared/src/*"], "@wdio/devtools-core": ["packages/core/src/index.ts"], "@wdio/devtools-core/*": ["packages/core/src/*"], + "@wdio/elements": ["packages/elements/src/index.ts"], + "@wdio/elements/*": ["packages/elements/src/*"], "@wdio/devtools-backend": ["packages/backend/src/index.ts"], "@wdio/devtools-backend/*": ["packages/backend/src/*"], "@wdio/devtools-script": ["packages/script/src/index.ts"], From 99ef6cb97db60b75c84929aaada8c47078904655 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 4 Jun 2026 16:31:22 +0530 Subject: [PATCH 03/13] Phase 3: Capture per-action accessibility/element snapshots in trace mode (WDIO) --- packages/core/src/action-mapping.ts | 53 ++++++++++++++++++++++++ packages/core/src/index.ts | 1 + packages/service/package.json | 1 + packages/service/src/action-snapshot.ts | 54 +++++++++++++++++++++++++ packages/service/src/index.ts | 35 ++++++++++++++-- packages/shared/src/types.ts | 14 +++++++ pnpm-lock.yaml | 3 ++ 7 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 packages/core/src/action-mapping.ts create mode 100644 packages/service/src/action-snapshot.ts diff --git a/packages/core/src/action-mapping.ts b/packages/core/src/action-mapping.ts new file mode 100644 index 00000000..0b1d76e5 --- /dev/null +++ b/packages/core/src/action-mapping.ts @@ -0,0 +1,53 @@ +// Allow-list mapping from runner-native command names to Playwright-trace +// vocabulary. Ported from Vince Graics' PR #209 (`@wdio/tracing-service`); the +// existing devtools UI uses its own denylist (`INTERNAL_COMMANDS`) — this map +// is for the trace.zip exporter to filter + rename in one step. + +export interface TraceAction { + class: string + method: string +} + +const ACTION_MAP: Record = { + url: { class: 'Page', method: 'navigate' }, + navigateTo: { class: 'Page', method: 'navigate' }, + back: { class: 'Page', method: 'goBack' }, + forward: { class: 'Page', method: 'goForward' }, + refresh: { class: 'Page', method: 'reload' }, + newWindow: { class: 'Page', method: 'goto' }, + click: { class: 'Element', method: 'click' }, + doubleClick: { class: 'Element', method: 'dblclick' }, + setValue: { class: 'Element', method: 'fill' }, + selectByVisibleText: { class: 'Element', method: 'selectOption' }, + moveTo: { class: 'Element', method: 'hover' }, + scrollIntoView: { class: 'Element', method: 'scrollIntoViewIfNeeded' }, + dragAndDrop: { class: 'Element', method: 'dragTo' }, + keys: { class: 'Keyboard', method: 'press' }, + execute: { class: 'Page', method: 'evaluate' }, + executeAsync: { class: 'Page', method: 'evaluate' }, + executeScript: { class: 'Page', method: 'evaluate' }, + switchToFrame: { class: 'Frame', method: 'goto' }, + touchAction: { class: 'Element', method: 'tap' } +} + +// clearValue / addValue are excluded: WDIO fires them internally inside setValue +// and they would produce duplicate trace entries. + +export function mapCommandToAction(command: string): TraceAction | null { + return ACTION_MAP[command] ?? null +} + +export function formatActionTitle( + action: TraceAction, + args: unknown[], + params?: Record +): string { + const firstArg = args[0] ?? params?.selector + if (firstArg === undefined) { + return `${action.class}.${action.method}()` + } + const label = ( + typeof firstArg === 'object' ? JSON.stringify(firstArg) : String(firstArg) + ).slice(0, 80) + return `${action.class}.${action.method}("${label}")` +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e34b6ac4..a690e243 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,6 +1,7 @@ // Framework-agnostic capture/reporter logic shared by @wdio/devtools-* // adapters. See ARCHITECTURE.md §2 and CLAUDE.md §2.2. +export * from './action-mapping.js' export * from './assert-patcher.js' export * from './bidi.js' export * from './console.js' diff --git a/packages/service/package.json b/packages/service/package.json index 66461d70..25c25c97 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -40,6 +40,7 @@ "@babel/types": "^7.29.7", "@wdio/devtools-backend": "workspace:^", "@wdio/devtools-script": "workspace:^", + "@wdio/elements": "workspace:^", "@wdio/logger": "9.18.0", "@wdio/reporter": "9.27.2", "@wdio/types": "9.27.2", diff --git a/packages/service/src/action-snapshot.ts b/packages/service/src/action-snapshot.ts new file mode 100644 index 00000000..8763f393 --- /dev/null +++ b/packages/service/src/action-snapshot.ts @@ -0,0 +1,54 @@ +// Per-action snapshot capture — fires only in `mode: 'trace'` for commands +// in the action allow-list (see @wdio/devtools-core/action-mapping). Returns +// null on failure; snapshot errors must not break the user's test. + +import { + getBrowserAccessibilityTree, + getElements, + serializeMobileSnapshot, + serializeWebSnapshot +} from '@wdio/elements' +import type { ActionSnapshot } from '@wdio/devtools-shared' + +export async function captureActionSnapshot( + browser: WebdriverIO.Browser, + command: string +): Promise { + try { + const timestamp = Date.now() + const isMobile = !!(browser.isAndroid || browser.isIOS) + const [screenshot, url, title] = await Promise.all([ + browser.takeScreenshot().catch(() => undefined), + browser.getUrl().catch(() => undefined), + browser.getTitle().catch(() => undefined) + ]) + let elements: unknown[] = [] + let snapshotText: string | undefined + if (isMobile) { + const result = await getElements(browser, { inViewportOnly: true }) + elements = result.elements + if (result.tree) { + const platform = browser.isAndroid ? 'android' : 'ios' + snapshotText = serializeMobileSnapshot(result.tree, { platform }) + } + } else { + const [tree, flatResult] = await Promise.all([ + getBrowserAccessibilityTree(browser, { inViewportOnly: true }), + getElements(browser, { inViewportOnly: true }) + ]) + elements = flatResult.elements + snapshotText = serializeWebSnapshot(tree, { url, title }) + } + return { + timestamp, + command, + url, + title, + screenshot, + elements, + snapshotText + } + } catch { + return null + } +} diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 1f54eba1..9e8e2ec2 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -3,7 +3,9 @@ import fs from 'node:fs/promises' import path from 'node:path' import logger from '@wdio/logger' -import { errorMessage } from '@wdio/devtools-core' +import { errorMessage, mapCommandToAction } from '@wdio/devtools-core' +import { captureActionSnapshot } from './action-snapshot.js' +import type { ActionSnapshot } from '@wdio/devtools-shared' import { SevereServiceError } from 'webdriverio' import type { Services, Reporters, Capabilities, Options } from '@wdio/types' import type { WebDriverCommands } from '@wdio/protocols' @@ -47,8 +49,12 @@ export default class DevToolsHookService implements Services.ServiceInstance { #bidiListenersSetup = false #screencastRecorder?: ScreencastRecorder #screencastOptions?: ScreencastOptions + #options: ServiceOptions + #actionSnapshots: ActionSnapshot[] = [] + #snapshotCaptures: Promise[] = [] constructor(serviceOptions: ServiceOptions = {}) { + this.#options = serviceOptions this.#screencastOptions = serviceOptions.screencast } @@ -275,7 +281,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { if (frame?.command === command) { this.#commandStack.pop() if (this.#browser) { - return this.#sessionCapturer.afterCommand( + const captured = this.#sessionCapturer.afterCommand( this.#browser, command, args, @@ -283,6 +289,21 @@ export default class DevToolsHookService implements Services.ServiceInstance { error, frame.callSource ) + if ( + this.#options.mode === 'trace' && + !error && + mapCommandToAction(command) + ) { + const browser = this.#browser + this.#snapshotCaptures.push( + captureActionSnapshot(browser, command).then((snap) => { + if (snap) { + this.#actionSnapshots.push(snap) + } + }) + ) + } + return captured } } @@ -304,6 +325,11 @@ export default class DevToolsHookService implements Services.ServiceInstance { // Stop and encode the screencast for the current session. await this.#finalizeScreencast(this.#browser.sessionId) + // Drain in-flight per-action snapshots before writing the trace. + if (this.#snapshotCaptures.length) { + await Promise.allSettled(this.#snapshotCaptures) + } + const outputDir = this.#outputDir const { ...options } = this.#browser.options const traceLog: TraceLog = { @@ -319,7 +345,10 @@ export default class DevToolsHookService implements Services.ServiceInstance { }, commands: this.#sessionCapturer.commandsLog, sources: Object.fromEntries(this.#sessionCapturer.sources), - suites: this.#testReporters.map((reporter) => reporter.report) + suites: this.#testReporters.map((reporter) => reporter.report), + ...(this.#actionSnapshots.length + ? { actionSnapshots: this.#actionSnapshots } + : {}) } const traceFilePath = path.join( diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index bd23a501..6712f2dc 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -247,6 +247,20 @@ export interface TraceMutation { url?: string } +/** + * Captured at each user-facing action boundary in `trace` mode. Feeds the + * downstream trace.zip exporter (Phase 4). `screenshot` is base64-encoded JPEG. + */ +export interface ActionSnapshot { + timestamp: number + command: string + url?: string + title?: string + screenshot?: string + elements?: unknown[] + snapshotText?: string +} + export interface TraceLog { mutations: TraceMutation[] logs: string[] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6a5c144d..c936044c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -490,6 +490,9 @@ importers: '@wdio/devtools-script': specifier: workspace:^ version: link:../script + '@wdio/elements': + specifier: workspace:^ + version: link:../elements '@wdio/logger': specifier: 9.18.0 version: 9.18.0 From c7f53e08e6fedf277a9a21ab6bc0b6f023297b76 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 4 Jun 2026 16:43:53 +0530 Subject: [PATCH 04/13] Phase 4: Export trace.zip from captured TraceLog (WDIO) --- packages/core/package.json | 4 +- packages/core/src/index.ts | 3 + packages/core/src/trace-exporter.ts | 188 ++++++++++++++++++++++++++ packages/core/src/trace-har.ts | 100 ++++++++++++++ packages/core/src/trace-zip-writer.ts | 35 +++++ packages/service/package.json | 3 +- packages/service/src/index.ts | 18 ++- packages/shared/src/types.ts | 2 + pnpm-lock.yaml | 23 ++++ 9 files changed, 373 insertions(+), 3 deletions(-) create mode 100644 packages/core/src/trace-exporter.ts create mode 100644 packages/core/src/trace-har.ts create mode 100644 packages/core/src/trace-zip-writer.ts diff --git a/packages/core/package.json b/packages/core/package.json index b637cd88..b7b6ad91 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -27,9 +27,11 @@ "license": "MIT", "devDependencies": { "@types/ws": "^8.18.1", + "@types/yazl": "^2.4.6", "@wdio/devtools-script": "workspace:*", "@wdio/devtools-shared": "workspace:^", "stacktrace-parser": "^0.1.11", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" } } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index a690e243..67879f93 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,6 +3,9 @@ export * from './action-mapping.js' export * from './assert-patcher.js' +export * from './trace-exporter.js' +export * from './trace-har.js' +export * from './trace-zip-writer.js' export * from './bidi.js' export * from './console.js' export * from './uid.js' diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts new file mode 100644 index 00000000..b082af5b --- /dev/null +++ b/packages/core/src/trace-exporter.ts @@ -0,0 +1,188 @@ +// Converts a captured TraceLog into a Playwright v8 trace.zip Buffer. +// Stays runner-agnostic so the three adapters can call this directly. + +import type { + ActionSnapshot, + CommandLog, + NetworkRequest, + TraceLog +} from '@wdio/devtools-shared' +import { formatActionTitle, mapCommandToAction } from './action-mapping.js' +import { networkRequestToHar } from './trace-har.js' +import { buildTraceZip, type TraceZipResource } from './trace-zip-writer.js' + +const TRACE_VERSION = 8 +const LIBRARY_NAME = '@wdio/devtools-core' +const LIBRARY_VERSION = '1.0.0' + +interface ContextOptionsEvent { + version: number + type: 'context-options' + origin: 'library' + libraryName: string + libraryVersion: string + browserName: string + platform: string + wallTime: number + monotonicTime: number + sdkLanguage: string + title: string + contextId: string + options: { viewport: { width: number; height: number } } +} + +interface BeforeEvent { + type: 'before' + callId: string + startTime: number + class: string + method: string + pageId: string + params: Record + title: string +} + +interface AfterEvent { + type: 'after' + callId: string + endTime: number + error?: { message: string } +} + +type TraceEvent = ContextOptionsEvent | BeforeEvent | AfterEvent + +function shortId(sessionId?: string): string { + return (sessionId ?? Math.random().toString(36).slice(2, 10)).slice(0, 8) +} + +function buildContextOptions( + trace: TraceLog, + contextId: string, + wallTime: number +): ContextOptionsEvent { + const caps = trace.metadata.capabilities as + | Record + | undefined + const browserName = (caps?.browserName as string) ?? 'chromium' + const viewport = trace.metadata.viewport ?? { width: 1280, height: 720 } + return { + version: TRACE_VERSION, + type: 'context-options', + origin: 'library', + libraryName: LIBRARY_NAME, + libraryVersion: LIBRARY_VERSION, + browserName, + platform: process.platform, + wallTime, + monotonicTime: 0, + sdkLanguage: 'javascript', + title: browserName, + contextId, + options: { + viewport: { width: viewport.width, height: viewport.height } + } + } +} + +function buildActionEvents( + commands: CommandLog[], + pageId: string, + wallTime: number +): TraceEvent[] { + const events: TraceEvent[] = [] + let callCounter = 0 + for (const cmd of commands) { + const action = mapCommandToAction(cmd.command) + if (!action) { + continue + } + callCounter++ + const callId = `call@${callCounter}` + const relativeMs = Math.max(0, cmd.timestamp - wallTime) + const params: Record = Object.fromEntries( + cmd.args.map((a, i) => [String(i), a]) + ) + events.push({ + type: 'before', + callId, + startTime: relativeMs, + class: action.class, + method: action.method, + pageId, + params, + title: formatActionTitle(action, cmd.args, params) + }) + const afterEvent: AfterEvent = { + type: 'after', + callId, + endTime: relativeMs + } + if (cmd.error) { + const err = cmd.error as { message?: string } + afterEvent.error = { message: err.message ?? String(cmd.error) } + } + events.push(afterEvent) + } + return events +} + +function buildNetworkNdjson(requests: NetworkRequest[]): Buffer { + if (!requests.length) { + return Buffer.alloc(0) + } + const lines = requests.map((r) => JSON.stringify(networkRequestToHar(r))) + return Buffer.from(lines.join('\n'), 'utf8') +} + +function buildSnapshotResources( + snapshots: ActionSnapshot[], + pageId: string +): TraceZipResource[] { + const out: TraceZipResource[] = [] + for (const snap of snapshots) { + const base = `${pageId}-${snap.timestamp}` + if (snap.screenshot) { + out.push({ + resourceName: `${base}.jpeg`, + data: Buffer.from(snap.screenshot, 'base64') + }) + } + if (snap.elements && snap.elements.length) { + out.push({ + resourceName: `elements-${base}.json`, + data: Buffer.from(JSON.stringify(snap.elements), 'utf8') + }) + } + if (snap.snapshotText) { + out.push({ + resourceName: `snapshot-${base}.txt`, + data: Buffer.from(snap.snapshotText, 'utf8') + }) + } + } + return out +} + +/** + * Build a Playwright v8 trace.zip buffer from the captured TraceLog. + * Filters commands through ACTION_MAP and renames to Playwright vocabulary; + * network entries become HAR resource-snapshots; per-action screenshots, + * element JSON, and snapshot text are written under `resources/`. + */ +export async function exportTraceZip( + trace: TraceLog, + opts: { sessionId?: string; wallTimeOverride?: number } = {} +): Promise { + const wallTime = opts.wallTimeOverride ?? Date.now() + const idPrefix = shortId(opts.sessionId) + const contextId = `context@${idPrefix}` + const pageId = `page@${idPrefix}` + const events: TraceEvent[] = [ + buildContextOptions(trace, contextId, wallTime), + ...buildActionEvents(trace.commands, pageId, wallTime) + ] + const traceNdjson = events.map((e) => JSON.stringify(e)).join('\n') + const networkNdjson = buildNetworkNdjson(trace.networkRequests) + const resources = buildSnapshotResources(trace.actionSnapshots ?? [], pageId) + return buildTraceZip({ traceNdjson, networkNdjson, resources }) +} diff --git a/packages/core/src/trace-har.ts b/packages/core/src/trace-har.ts new file mode 100644 index 00000000..06fae1b8 --- /dev/null +++ b/packages/core/src/trace-har.ts @@ -0,0 +1,100 @@ +// Convert the existing NetworkRequest shape into Playwright v8 +// `resource-snapshot` NDJSON entries (HAR-flavoured) for trace.zip. + +import type { NetworkRequest } from '@wdio/devtools-shared' + +export interface ResourceSnapshotEntry { + type: 'resource-snapshot' + snapshot: { + startedDateTime: string + time: number + request: { + method: string + url: string + httpVersion: string + cookies: unknown[] + headers: { name: string; value: string }[] + queryString: { name: string; value: string }[] + headersSize: number + bodySize: number + } + response: { + status: number + statusText: string + httpVersion: string + cookies: unknown[] + headers: { name: string; value: string }[] + content: { size: number; mimeType: string } + redirectURL: string + headersSize: number + bodySize: number + } + cache: Record + timings: { send: number; wait: number; receive: number } + } +} + +function toHeaderArray( + h: Record | undefined +): { name: string; value: string }[] { + if (!h) { + return [] + } + return Object.entries(h).map(([name, value]) => ({ name, value })) +} + +function toQueryString(url: string): { name: string; value: string }[] { + try { + const u = new URL(url) + const out: { name: string; value: string }[] = [] + u.searchParams.forEach((value, name) => out.push({ name, value })) + return out + } catch { + return [] + } +} + +export function networkRequestToHar( + entry: NetworkRequest +): ResourceSnapshotEntry { + const startedDateTime = new Date(entry.timestamp).toISOString() + const duration = + entry.time ?? (entry.endTime ?? entry.startTime) - entry.startTime + const status = entry.response?.status ?? entry.status ?? 0 + const mimeType = entry.response?.mimeType ?? '' + const responseHeaders = entry.response?.headers ?? entry.responseHeaders + return { + type: 'resource-snapshot', + snapshot: { + startedDateTime, + time: Math.max(0, duration), + request: { + method: entry.method, + url: entry.url, + httpVersion: 'HTTP/1.1', + cookies: [], + headers: toHeaderArray(entry.requestHeaders ?? entry.headers), + queryString: toQueryString(entry.url), + headersSize: -1, + bodySize: entry.requestBody ? entry.requestBody.length : -1 + }, + response: { + status, + statusText: entry.statusText ?? '', + httpVersion: 'HTTP/1.1', + cookies: [], + headers: toHeaderArray(responseHeaders), + content: { size: entry.size ?? 0, mimeType }, + redirectURL: '', + headersSize: -1, + bodySize: entry.size ?? -1 + }, + cache: {}, + timings: { + send: -1, + wait: Math.max(0, duration), + receive: -1 + } + } + } +} diff --git a/packages/core/src/trace-zip-writer.ts b/packages/core/src/trace-zip-writer.ts new file mode 100644 index 00000000..581790e7 --- /dev/null +++ b/packages/core/src/trace-zip-writer.ts @@ -0,0 +1,35 @@ +// Thin yazl wrapper that packages a Playwright v8 trace into a single Buffer. +// Ported from Vince Graics' PR #209. + +import yazl from 'yazl' + +export interface TraceZipResource { + /** Path inside the zip, e.g. `resources/page@xxx-12345.jpeg`. */ + resourceName: string + data: Buffer +} + +export interface TraceZipInputs { + /** NDJSON action events (one JSON object per line). */ + traceNdjson: string + /** NDJSON HAR resource-snapshot entries. Empty buffer when omitted. */ + networkNdjson: Buffer + /** Files written under `resources/` — typically screenshots + element snapshots. */ + resources: TraceZipResource[] +} + +export function buildTraceZip(inputs: TraceZipInputs): Promise { + return new Promise((resolve, reject) => { + const zipFile = new yazl.ZipFile() + zipFile.addBuffer(Buffer.from(inputs.traceNdjson, 'utf8'), 'trace.trace') + zipFile.addBuffer(inputs.networkNdjson, 'trace.network') + for (const resource of inputs.resources) { + zipFile.addBuffer(resource.data, `resources/${resource.resourceName}`) + } + zipFile.end() + const chunks: Buffer[] = [] + zipFile.outputStream.on('data', (chunk: Buffer) => chunks.push(chunk)) + zipFile.outputStream.on('end', () => resolve(Buffer.concat(chunks))) + zipFile.outputStream.on('error', reject) + }) +} diff --git a/packages/service/package.json b/packages/service/package.json index 25c25c97..b00eb468 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -48,7 +48,8 @@ "import-meta-resolve": "^4.2.0", "stack-trace": "^1.0.0", "stacktrace-parser": "^0.1.11", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" }, "license": "MIT", "devDependencies": { diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 9e8e2ec2..0bfa06a9 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -3,7 +3,11 @@ import fs from 'node:fs/promises' import path from 'node:path' import logger from '@wdio/logger' -import { errorMessage, mapCommandToAction } from '@wdio/devtools-core' +import { + errorMessage, + exportTraceZip, + mapCommandToAction +} from '@wdio/devtools-core' import { captureActionSnapshot } from './action-snapshot.js' import type { ActionSnapshot } from '@wdio/devtools-shared' import { SevereServiceError } from 'webdriverio' @@ -358,6 +362,18 @@ export default class DevToolsHookService implements Services.ServiceInstance { await fs.writeFile(traceFilePath, JSON.stringify(traceLog)) log.info(`DevTools trace saved to ${traceFilePath}`) + if (this.#options.mode === 'trace') { + const zip = await exportTraceZip(traceLog, { + sessionId: this.#browser.sessionId + }) + const zipPath = path.join( + outputDir, + `trace-${this.#browser.sessionId}.zip` + ) + await fs.writeFile(zipPath, zip) + log.info(`Playwright v8 trace.zip saved to ${zipPath}`) + } + // Clean up console patching this.#sessionCapturer.cleanup() } diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 6712f2dc..ca7bc2d8 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -272,6 +272,8 @@ export interface TraceLog { suites?: Record[] screencast?: ScreencastInfo config?: { configFile?: string } + /** Per-action snapshots captured in `mode: 'trace'` for the trace.zip exporter. */ + actionSnapshots?: ActionSnapshot[] } // ─── Preserve-and-rerun ───────────────────────────────────────────────────── diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c936044c..6079c368 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -299,6 +299,9 @@ importers: '@types/ws': specifier: ^8.18.1 version: 8.18.1 + '@types/yazl': + specifier: ^2.4.6 + version: 2.4.6 '@wdio/devtools-script': specifier: workspace:* version: link:../script @@ -311,6 +314,9 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 packages/elements: dependencies: @@ -523,6 +529,9 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 devDependencies: '@types/babel__core': specifier: ^7.20.5 @@ -2347,6 +2356,9 @@ packages: '@types/yauzl@2.10.3': resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} + '@types/yazl@2.4.6': + resolution: {integrity: sha512-/ifFjQtcKaoZOjl5NNCQRR0fAKafB3Foxd7J/WvFPTMea46zekapcR30uzkwIkKAAuq5T6d0dkwz754RFH27hg==} + '@typescript-eslint/eslint-plugin@8.60.1': resolution: {integrity: sha512-JQ4S5GB0tfjO8BuJ4fcX+HodkzJjYBV+7OJ+wLygaX7OGQ7FudyHL4NSCA6ob+w3Yn+5MkKIozOwQhXeM7opVg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -7283,6 +7295,9 @@ packages: yauzl@2.10.0: resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + yazl@2.5.1: + resolution: {integrity: sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw==} + yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} @@ -9118,6 +9133,10 @@ snapshots: '@types/node': 25.9.1 optional: true + '@types/yazl@2.4.6': + dependencies: + '@types/node': 25.9.1 + '@typescript-eslint/eslint-plugin@8.60.1(@typescript-eslint/parser@8.60.1(eslint@10.4.1(jiti@2.7.0))(typescript@6.0.3))(eslint@10.4.1(jiti@2.7.0))(typescript@6.0.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -14946,6 +14965,10 @@ snapshots: buffer-crc32: 0.2.13 fd-slicer: 1.1.0 + yazl@2.5.1: + dependencies: + buffer-crc32: 0.2.13 + yn@3.1.1: {} yocto-queue@0.1.0: {} From f5abbf4f0d0690239094acf10a5d74e7a0dda4bf Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Mon, 8 Jun 2026 15:41:43 +0530 Subject: [PATCH 05/13] Phase 5: Emit trace.zip from Selenium + Nightwatch adapters via shared core helper; Restrict screencast feature only for the live mode --- .gitignore | 15 +- examples/nightwatch/nightwatch.conf.cjs | 1 + examples/selenium/mocha-test/test/example.js | 1 + examples/wdio/wdio.conf.ts | 16 +- packages/core/src/action-mapping.ts | 12 +- packages/core/src/trace-exporter.ts | 143 ++++++++++++++++-- packages/core/src/trace-har.ts | 2 +- packages/core/src/trace-zip-writer.ts | 2 +- packages/nightwatch-devtools/package.json | 3 +- packages/nightwatch-devtools/src/index.ts | 49 +++++- .../nightwatch-devtools/src/session-init.ts | 6 +- packages/selenium-devtools/package.json | 3 +- packages/selenium-devtools/src/index.ts | 29 +++- .../src/session-lifecycle.ts | 46 +++++- packages/service/src/index.ts | 27 ++-- pnpm-lock.yaml | 6 + 16 files changed, 300 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index 6197c849..117af42f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,20 @@ dist-ssr /*.ts /*.json *.tgz -example/wdio-*.json +examples/wdio/wdio-*.json +examples/wdio/wdio-*.webm +examples/nightwatch/logs/ + +# Adapter-encoded screencasts (written next to the project root by default) +selenium-video-*.webm +nightwatch-video-*.webm +packages/nightwatch-devtools/nightwatch-video-*.webm + +# trace.zip output (mode: 'trace') +trace-*.zip + +# vitest --coverage output +coverage/ # pnpm state, cache, logs, and debug files /packages/**/*.mjs diff --git a/examples/nightwatch/nightwatch.conf.cjs b/examples/nightwatch/nightwatch.conf.cjs index a817bf57..619d1a76 100644 --- a/examples/nightwatch/nightwatch.conf.cjs +++ b/examples/nightwatch/nightwatch.conf.cjs @@ -45,6 +45,7 @@ module.exports = { // off to avoid duplicate entries. globals: nightwatchDevtools({ port: 3000, + mode: 'trace', screencast: { enabled: true, pollIntervalMs: 200 }, bidi: true }) diff --git a/examples/selenium/mocha-test/test/example.js b/examples/selenium/mocha-test/test/example.js index be2a79e4..a0d27e50 100644 --- a/examples/selenium/mocha-test/test/example.js +++ b/examples/selenium/mocha-test/test/example.js @@ -9,6 +9,7 @@ import { Builder, By, until } from 'selenium-webdriver' import { DevTools } from '@wdio/selenium-devtools' DevTools.configure({ + mode: 'trace', screencast: { enabled: true, quality: 70, maxWidth: 1280, maxHeight: 720 }, headless: true }) diff --git a/examples/wdio/wdio.conf.ts b/examples/wdio/wdio.conf.ts index 73e88052..71830cf2 100644 --- a/examples/wdio/wdio.conf.ts +++ b/examples/wdio/wdio.conf.ts @@ -127,21 +127,7 @@ export const config: Options.Testrunner = { // Services take over a specific job you don't want to take care of. They enhance // your test setup with almost no effort. Unlike plugins, they don't add new // commands. Instead, they hook themselves up into the test process. - services: [ - 'devtools' - // [ - // 'devtools', - // { - // screencast: { - // enabled: true, - // captureFormat: 'jpeg', // 'jpeg' or 'png' — frame format sent by Chrome over CDP - // quality: 70, // JPEG quality 0–100 - // maxWidth: 1280, // max frame width in px - // maxHeight: 720 // max frame height in px - // } - // } - // ] - ], + services: [['devtools', { mode: 'trace' as const }]], // // Framework you want to run your specs with. // The following are supported: Mocha, Jasmine, and Cucumber diff --git a/packages/core/src/action-mapping.ts b/packages/core/src/action-mapping.ts index 0b1d76e5..5c530f9c 100644 --- a/packages/core/src/action-mapping.ts +++ b/packages/core/src/action-mapping.ts @@ -1,4 +1,4 @@ -// Allow-list mapping from runner-native command names to Playwright-trace +// Allow-list mapping from runner-native command names to trace // vocabulary. Ported from Vince Graics' PR #209 (`@wdio/tracing-service`); the // existing devtools UI uses its own denylist (`INTERNAL_COMMANDS`) — this map // is for the trace.zip exporter to filter + rename in one step. @@ -9,12 +9,17 @@ export interface TraceAction { } const ACTION_MAP: Record = { + // WDIO browser-level url: { class: 'Page', method: 'navigate' }, navigateTo: { class: 'Page', method: 'navigate' }, back: { class: 'Page', method: 'goBack' }, forward: { class: 'Page', method: 'goForward' }, refresh: { class: 'Page', method: 'reload' }, newWindow: { class: 'Page', method: 'goto' }, + // Selenium WebDriver navigation (driver.get, driver.navigate().to/back/forward/refresh) + get: { class: 'Page', method: 'navigate' }, + to: { class: 'Page', method: 'navigate' }, + // WDIO element-level click: { class: 'Element', method: 'click' }, doubleClick: { class: 'Element', method: 'dblclick' }, setValue: { class: 'Element', method: 'fill' }, @@ -22,6 +27,11 @@ const ACTION_MAP: Record = { moveTo: { class: 'Element', method: 'hover' }, scrollIntoView: { class: 'Element', method: 'scrollIntoViewIfNeeded' }, dragAndDrop: { class: 'Element', method: 'dragTo' }, + // Selenium WebElement actions + sendKeys: { class: 'Element', method: 'fill' }, + clear: { class: 'Element', method: 'clear' }, + submit: { class: 'Element', method: 'submit' }, + // Cross-runner keys: { class: 'Keyboard', method: 'press' }, execute: { class: 'Page', method: 'evaluate' }, executeAsync: { class: 'Page', method: 'evaluate' }, diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts index b082af5b..cfcaef9c 100644 --- a/packages/core/src/trace-exporter.ts +++ b/packages/core/src/trace-exporter.ts @@ -1,11 +1,16 @@ -// Converts a captured TraceLog into a Playwright v8 trace.zip Buffer. +// Converts a captured TraceLog into a trace.zip Buffer. // Stays runner-agnostic so the three adapters can call this directly. +import fs from 'node:fs/promises' +import path from 'node:path' import type { ActionSnapshot, CommandLog, + ConsoleLog, + Metadata, NetworkRequest, - TraceLog + TraceLog, + TraceMutation } from '@wdio/devtools-shared' import { formatActionTitle, mapCommandToAction } from './action-mapping.js' import { networkRequestToHar } from './trace-har.js' @@ -49,7 +54,21 @@ interface AfterEvent { error?: { message: string } } -type TraceEvent = ContextOptionsEvent | BeforeEvent | AfterEvent +interface ScreencastFrameEvent { + type: 'screencast-frame' + pageId: string + sha1: string + elements?: string + width: number + height: number + timestamp: number +} + +type TraceEvent = + | ContextOptionsEvent + | BeforeEvent + | AfterEvent + | ScreencastFrameEvent function shortId(sessionId?: string): string { return (sessionId ?? Math.random().toString(36).slice(2, 10)).slice(0, 8) @@ -90,6 +109,9 @@ function buildActionEvents( wallTime: number ): TraceEvent[] { const events: TraceEvent[] = [] + // cmd.timestamp records command completion, so the *previous* mapped + // command's timestamp is a usable startTime for the next one. + let prevEndMs = 0 let callCounter = 0 for (const cmd of commands) { const action = mapCommandToAction(cmd.command) @@ -98,14 +120,14 @@ function buildActionEvents( } callCounter++ const callId = `call@${callCounter}` - const relativeMs = Math.max(0, cmd.timestamp - wallTime) + const endMs = Math.max(prevEndMs, cmd.timestamp - wallTime) const params: Record = Object.fromEntries( cmd.args.map((a, i) => [String(i), a]) ) events.push({ type: 'before', callId, - startTime: relativeMs, + startTime: prevEndMs, class: action.class, method: action.method, pageId, @@ -115,13 +137,14 @@ function buildActionEvents( const afterEvent: AfterEvent = { type: 'after', callId, - endTime: relativeMs + endTime: endMs } if (cmd.error) { const err = cmd.error as { message?: string } afterEvent.error = { message: err.message ?? String(cmd.error) } } events.push(afterEvent) + prevEndMs = endMs } return events } @@ -163,9 +186,34 @@ function buildSnapshotResources( return out } +function buildScreencastFrames( + snapshots: ActionSnapshot[], + pageId: string, + wallTime: number, + viewport: { width: number; height: number } +): ScreencastFrameEvent[] { + return snapshots + .filter((s) => s.screenshot) + .map((s) => { + const base = `${pageId}-${s.timestamp}` + const frame: ScreencastFrameEvent = { + type: 'screencast-frame', + pageId, + sha1: `${base}.jpeg`, + width: viewport.width, + height: viewport.height, + timestamp: Math.max(0, s.timestamp - wallTime) + } + if (s.elements && s.elements.length) { + frame.elements = `elements-${base}.json` + } + return frame + }) +} + /** - * Build a Playwright v8 trace.zip buffer from the captured TraceLog. - * Filters commands through ACTION_MAP and renames to Playwright vocabulary; + * Build a trace.zip buffer from the captured TraceLog. + * Filters commands through ACTION_MAP and renames to trace vocabulary; * network entries become HAR resource-snapshots; per-action screenshots, * element JSON, and snapshot text are written under `resources/`. */ @@ -173,16 +221,91 @@ export async function exportTraceZip( trace: TraceLog, opts: { sessionId?: string; wallTimeOverride?: number } = {} ): Promise { - const wallTime = opts.wallTimeOverride ?? Date.now() + // wallTime anchors monotonic offsets at the first captured command so + // subsequent actions render at positive deltas in the trace viewer. + const firstCommandTs = trace.commands[0]?.timestamp + const wallTime = opts.wallTimeOverride ?? firstCommandTs ?? Date.now() const idPrefix = shortId(opts.sessionId) const contextId = `context@${idPrefix}` const pageId = `page@${idPrefix}` + const viewport = trace.metadata.viewport ?? { width: 1280, height: 720 } + const snapshots = trace.actionSnapshots ?? [] const events: TraceEvent[] = [ buildContextOptions(trace, contextId, wallTime), + ...buildScreencastFrames(snapshots, pageId, wallTime, viewport), ...buildActionEvents(trace.commands, pageId, wallTime) ] const traceNdjson = events.map((e) => JSON.stringify(e)).join('\n') const networkNdjson = buildNetworkNdjson(trace.networkRequests) - const resources = buildSnapshotResources(trace.actionSnapshots ?? [], pageId) + const resources = buildSnapshotResources(snapshots, pageId) return buildTraceZip({ traceNdjson, networkNdjson, resources }) } + +/** Minimum capturer surface needed to assemble a TraceLog. */ +export interface TraceCapturer { + mutations: TraceMutation[] + traceLogs: string[] + consoleLogs: ConsoleLog[] + networkRequests: NetworkRequest[] + commandsLog: CommandLog[] + sources: Map + metadata?: Metadata +} + +export interface WriteTraceZipOptions { + outputDir: string + sessionId: string + capabilities?: unknown + /** + * Per-action snapshots from a Phase-3-style hook. When omitted, snapshots + * are synthesized from CommandLog entries that carry a screenshot so the + * viewer still renders thumbnails for adapters without an action hook. + */ + actionSnapshots?: ActionSnapshot[] +} + +/** + * Build a TraceLog from a SessionCapturer-shaped source and write a + * Trace.zip. Returns the absolute path written. + */ +export async function writeTraceZip( + capturer: TraceCapturer, + opts: WriteTraceZipOptions +): Promise { + const baseMetadata = capturer.metadata ?? ({} as Metadata) + const actionSnapshots = + opts.actionSnapshots ?? + synthesizeSnapshotsFromCommands(capturer.commandsLog) + const traceLog: TraceLog = { + mutations: capturer.mutations, + logs: capturer.traceLogs, + consoleLogs: capturer.consoleLogs, + networkRequests: capturer.networkRequests, + metadata: { + ...baseMetadata, + ...(opts.capabilities + ? { capabilities: opts.capabilities as Metadata['capabilities'] } + : {}) + }, + commands: capturer.commandsLog, + sources: Object.fromEntries(capturer.sources), + ...(actionSnapshots.length ? { actionSnapshots } : {}) + } + const zip = await exportTraceZip(traceLog, { sessionId: opts.sessionId }) + await fs.mkdir(opts.outputDir, { recursive: true }) + const zipPath = path.join(opts.outputDir, `trace-${opts.sessionId}.zip`) + await fs.writeFile(zipPath, zip) + return zipPath +} + +function synthesizeSnapshotsFromCommands( + commands: CommandLog[] +): ActionSnapshot[] { + return commands + .filter((c) => c.screenshot && mapCommandToAction(c.command)) + .map((c) => ({ + timestamp: c.timestamp, + command: c.command, + screenshot: c.screenshot + })) +} diff --git a/packages/core/src/trace-har.ts b/packages/core/src/trace-har.ts index 06fae1b8..565053ff 100644 --- a/packages/core/src/trace-har.ts +++ b/packages/core/src/trace-har.ts @@ -1,4 +1,4 @@ -// Convert the existing NetworkRequest shape into Playwright v8 +// Convert the existing NetworkRequest shape into trace format // `resource-snapshot` NDJSON entries (HAR-flavoured) for trace.zip. import type { NetworkRequest } from '@wdio/devtools-shared' diff --git a/packages/core/src/trace-zip-writer.ts b/packages/core/src/trace-zip-writer.ts index 581790e7..f3dfac69 100644 --- a/packages/core/src/trace-zip-writer.ts +++ b/packages/core/src/trace-zip-writer.ts @@ -1,4 +1,4 @@ -// Thin yazl wrapper that packages a Playwright v8 trace into a single Buffer. +// Thin yazl wrapper that packages a trace into a single Buffer. // Ported from Vince Graics' PR #209. import yazl from 'yazl' diff --git a/packages/nightwatch-devtools/package.json b/packages/nightwatch-devtools/package.json index 0a92afb0..c8ad4f92 100644 --- a/packages/nightwatch-devtools/package.json +++ b/packages/nightwatch-devtools/package.json @@ -48,7 +48,8 @@ "import-meta-resolve": "^4.2.0", "stacktrace-parser": "^0.1.11", "webdriverio": "^9.27.2", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" }, "devDependencies": { "@types/node": "25.9.1", diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index dcb49dd9..245a8210 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -6,7 +6,12 @@ */ import { fileURLToPath } from 'node:url' -import { errorMessage } from '@wdio/devtools-core' +import { + errorMessage, + resolveAdapterOutputDir, + writeTraceZip +} from '@wdio/devtools-core' +import { stop as stopBackend } from '@wdio/devtools-backend' import { REUSE_ENV, SCREENCAST_DEFAULTS } from '@wdio/devtools-shared' import logger from '@wdio/logger' import { @@ -98,17 +103,20 @@ class NightwatchDevToolsPlugin { #bidiAttachAttempted = false constructor(options: DevToolsOptions = {}) { + const mode = options.mode ?? 'live' + const ignore = mode === 'trace' && options.screencast?.enabled === true + if (ignore) { + log.warn('trace mode: ignoring screencast option (live-mode feature)') + } + const screencast = ignore ? {} : (options.screencast ?? {}) this.options = { port: options.port ?? 3000, hostname: options.hostname ?? 'localhost', - screencast: options.screencast ?? {}, + screencast, bidi: options.bidi ?? false, - mode: options.mode ?? 'live' - } - this.#screencastOptions = { - ...SCREENCAST_DEFAULTS, - ...(options.screencast ?? {}) + mode } + this.#screencastOptions = { ...SCREENCAST_DEFAULTS, ...screencast } this.#bidiEnabled = options.bidi === true } @@ -442,6 +450,12 @@ class NightwatchDevToolsPlugin { try { await this.#finalizeAllSuites(browser) this.#logRunSummary() + if (this.options.mode === 'trace') { + await this.#writeTraceZipIfNeeded() + await this.sessionCapturer?.closeWebSocket() + await stopBackend() + return + } if (!this.#devtoolsBrowser) { // Reuse mode: force one final suites broadcast so the UI reflects the // actual outcome before the process exits. @@ -465,6 +479,27 @@ class NightwatchDevToolsPlugin { logRunSummary(this.#getInternals()) } + async #writeTraceZipIfNeeded(): Promise { + if (this.options.mode !== 'trace' || !this.sessionCapturer) { + return + } + const sessionId = this.sessionCapturer.metadata?.sessionId + if (!sessionId) { + return + } + try { + const zipPath = await writeTraceZip(this.sessionCapturer, { + outputDir: resolveAdapterOutputDir({ + configPath: this.#configPath + }), + sessionId + }) + log.info(`Trace.zip saved to ${zipPath}`) + } catch (err) { + log.warn(`trace.zip write failed: ${errorMessage(err)}`) + } + } + async #waitForDevtoolsBrowserClose(): Promise { await waitForDevtoolsBrowserClose(this.#getInternals()) } diff --git a/packages/nightwatch-devtools/src/session-init.ts b/packages/nightwatch-devtools/src/session-init.ts index 5b7b29c9..32df9d91 100644 --- a/packages/nightwatch-devtools/src/session-init.ts +++ b/packages/nightwatch-devtools/src/session-init.ts @@ -124,7 +124,7 @@ function broadcastSessionMetadata( ctx.srcFolders = Array.isArray(sf) ? sf : sf ? [sf] : [] } - ctx.sessionCapturer.sendUpstream('metadata', { + const metadata = { type: TraceType.Testrunner, capabilities, desiredCapabilities, @@ -133,7 +133,9 @@ function broadcastSessionMetadata( host: opts.webdriver?.host, options: ctx.buildMetadataOptions(), url: '' - }) + } + ctx.sessionCapturer.metadata = metadata + ctx.sessionCapturer.sendUpstream('metadata', metadata) const browserName = capabilities.browserName || desiredCapabilities.browserName || 'unknown' diff --git a/packages/selenium-devtools/package.json b/packages/selenium-devtools/package.json index 0b118b7f..869c25d8 100644 --- a/packages/selenium-devtools/package.json +++ b/packages/selenium-devtools/package.json @@ -47,7 +47,8 @@ "@wdio/logger": "^9.18.0", "stacktrace-parser": "^0.1.11", "webdriverio": "^9.27.2", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" }, "optionalDependencies": { "fluent-ffmpeg": "^2.1.3" diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index 63a091d1..e02204fa 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -49,6 +49,7 @@ import { } from './constants.js' import { type CapturedCommand, + type DevToolsMode, type DevToolsOptions, type ScreencastOptions, type SeleniumDriverLike, @@ -228,24 +229,32 @@ class SeleniumDevToolsPlugin { screencast?: ScreencastOptions headless?: boolean openUi?: boolean + mode?: DevToolsMode } = {} ) { if ('rerunCommand' in opts) { this.#rerunManager.configure(opts.rerunCommand) this.#options.rerunCommand = opts.rerunCommand } - if (opts.screencast) { - this.#screencastOptions = { - ...this.#screencastOptions, - ...opts.screencast - } - } if (typeof opts.headless === 'boolean') { this.#options.headless = opts.headless } if (typeof opts.openUi === 'boolean') { this.#options.openUi = opts.openUi } + if (opts.mode) { + this.#options.mode = opts.mode + } + if (opts.screencast) { + if (this.#options.mode === 'trace' && opts.screencast.enabled) { + log.warn('trace mode: ignoring screencast option (live-mode feature)') + } else { + this.#screencastOptions = { + ...this.#screencastOptions, + ...opts.screencast + } + } + } } get options() { @@ -564,7 +573,13 @@ if (!registerHooks()) { registerProcessHooks(plugin) export const DevTools = { - configure: (opts: { rerunCommand?: string }) => plugin.configure(opts), + configure: (opts: { + rerunCommand?: string + screencast?: ScreencastOptions + headless?: boolean + openUi?: boolean + mode?: DevToolsMode + }) => plugin.configure(opts), startTest: (name: string, meta?: { file?: string }) => plugin.startTest(name, meta), endTest: (state: TestStats['state'] = 'passed') => plugin.endTest(state) diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index 59c823e5..f3c8c483 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -12,7 +12,8 @@ import logger from '@wdio/logger' import { errorMessage, finalizeScreencast, - resolveAdapterOutputDir + resolveAdapterOutputDir, + writeTraceZip } from '@wdio/devtools-core' import { TIMING } from './constants.js' import { SessionCapturer } from './session.js' @@ -22,7 +23,12 @@ import { ScreencastRecorder } from './screencast.js' import { buildDriverMetadata } from './helpers/driverMetadata.js' import { attachBidiHandlers, buildBidiSinks } from './bidi.js' import { gracefulShutdown } from './helpers/processHooks.js' -import type { ScreencastOptions, SeleniumDriverLike } from './types.js' +import type { + DevToolsMode, + Metadata, + ScreencastOptions, + SeleniumDriverLike +} from './types.js' import type { TestManager } from './helpers/testManager.js' const log = logger('@wdio/selenium-devtools:session-lifecycle') @@ -34,6 +40,7 @@ export interface SessionLifecycleCtx { openUi: boolean captureScreenshots: boolean rerunCommand?: string + mode?: DevToolsMode } readonly screencastOptions: ScreencastOptions readonly runner: string @@ -136,6 +143,10 @@ async function initPerDriverCapture( }) ctx.sessionId = sessionId if (metadata) { + // buildDriverMetadata returns a Record-shaped payload; the relevant + // Metadata fields (sessionId, capabilities, viewport, ...) are present + // at runtime but TS can't prove the discriminant `type`. + ctx.sessionCapturer.metadata = metadata as unknown as Metadata ctx.sessionCapturer.sendUpstream('metadata', metadata) } @@ -198,6 +209,9 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { } ctx.setFinalized(true) const shutdownStart = Date.now() + // Capture for the trace.zip write before onDriverEnd clears ctx state. + const capturerAtStart = ctx.sessionCapturer + const testFilePathAtStart = ctx.testFilePath try { await onDriverEnd(ctx).catch(() => {}) @@ -211,14 +225,40 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { ctx.testManager?.finalizeSession() ctx.testReporter?.updateSuites() + const sessionId = capturerAtStart?.metadata?.sessionId + if (ctx.options.mode === 'trace' && capturerAtStart && sessionId) { + try { + const zipPath = await writeTraceZip(capturerAtStart, { + outputDir: resolveAdapterOutputDir({ + testFilePath: testFilePathAtStart + }), + sessionId + }) + log.info(`Trace.zip saved to ${zipPath}`) + } catch (err) { + log.warn(`trace.zip write failed: ${errorMessage(err)}`) + } + } + logSessionSummary(ctx) ctx.sessionCapturer?.cleanup() - if (ctx.options.openUi && !ctx.isReuse) { + if ( + ctx.options.openUi && + ctx.options.mode !== 'trace' && + !ctx.isReuse + ) { handleInteractivePath(ctx, shutdownStart) return } + // trace mode: no UI to wait for; force-shutdown so the backend HTTP + // server stops keeping the event loop alive. + if (ctx.options.mode === 'trace' && !ctx.isReuse) { + await completeShutdown(ctx, shutdownStart) + return + } + // Non-interactive path (no dashboard or rerun child). Don't close the // WS yet: this `onSessionEnd` is reached via the patched `driver.quit()` // (cucumber's per-scenario `After` hook), but the runner's diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 0bfa06a9..558f0755 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -5,8 +5,8 @@ import path from 'node:path' import logger from '@wdio/logger' import { errorMessage, - exportTraceZip, - mapCommandToAction + mapCommandToAction, + writeTraceZip } from '@wdio/devtools-core' import { captureActionSnapshot } from './action-snapshot.js' import type { ActionSnapshot } from '@wdio/devtools-shared' @@ -59,7 +59,12 @@ export default class DevToolsHookService implements Services.ServiceInstance { constructor(serviceOptions: ServiceOptions = {}) { this.#options = serviceOptions - this.#screencastOptions = serviceOptions.screencast + if (serviceOptions.mode === 'trace' && serviceOptions.screencast?.enabled) { + log.warn('trace mode: ignoring screencast option (live-mode feature)') + this.#screencastOptions = undefined + } else { + this.#screencastOptions = serviceOptions.screencast + } } /** @@ -363,15 +368,15 @@ export default class DevToolsHookService implements Services.ServiceInstance { log.info(`DevTools trace saved to ${traceFilePath}`) if (this.#options.mode === 'trace') { - const zip = await exportTraceZip(traceLog, { - sessionId: this.#browser.sessionId - }) - const zipPath = path.join( + const zipPath = await writeTraceZip(this.#sessionCapturer, { outputDir, - `trace-${this.#browser.sessionId}.zip` - ) - await fs.writeFile(zipPath, zip) - log.info(`Playwright v8 trace.zip saved to ${zipPath}`) + sessionId: this.#browser.sessionId, + capabilities: this.#browser.capabilities, + actionSnapshots: this.#actionSnapshots.length + ? this.#actionSnapshots + : undefined + }) + log.info(`Trace.zip saved to ${zipPath}`) } // Clean up console patching diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6079c368..4b3cee2f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -372,6 +372,9 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 devDependencies: '@types/node': specifier: 25.9.1 @@ -437,6 +440,9 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 devDependencies: '@cucumber/cucumber': specifier: ^13.0.0 From 2d8f3a8a1dd0b29d92dbce7b7f4d1a294633b798 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Mon, 8 Jun 2026 16:30:39 +0530 Subject: [PATCH 06/13] Phase 6: Per-action element/snapshot parity for Selenium and Nightwatch --- examples/selenium/jest-test/test/example.js | 2 +- examples/selenium/mocha-test/test/example.js | 1 - packages/core/src/action-mapping.ts | 8 ++- packages/nightwatch-devtools/package.json | 1 + .../src/action-snapshot.ts | 67 +++++++++++++++++++ packages/nightwatch-devtools/src/index.ts | 10 ++- .../src/plugin-internals.ts | 1 + .../nightwatch-devtools/src/session-init.ts | 3 + packages/nightwatch-devtools/src/session.ts | 33 ++++++++- packages/nightwatch-devtools/src/types.ts | 1 + packages/selenium-devtools/package.json | 1 + .../selenium-devtools/src/action-snapshot.ts | 62 +++++++++++++++++ .../src/helpers/commandPostActions.ts | 23 ++++++- packages/selenium-devtools/src/index.ts | 9 +++ .../selenium-devtools/src/plugin-internals.ts | 12 +++- .../src/session-lifecycle.ts | 25 +++++-- packages/selenium-devtools/src/types.ts | 1 + pnpm-lock.yaml | 6 ++ 18 files changed, 253 insertions(+), 13 deletions(-) create mode 100644 packages/nightwatch-devtools/src/action-snapshot.ts create mode 100644 packages/selenium-devtools/src/action-snapshot.ts diff --git a/examples/selenium/jest-test/test/example.js b/examples/selenium/jest-test/test/example.js index 15e9d21a..a59b252b 100644 --- a/examples/selenium/jest-test/test/example.js +++ b/examples/selenium/jest-test/test/example.js @@ -12,7 +12,7 @@ const VALID_USERNAME = 'tomsmith' const VALID_PASSWORD = 'SuperSecretPassword!' DevTools.configure({ - screencast: { enabled: true, quality: 70, maxWidth: 1280, maxHeight: 720 }, + mode: 'trace', headless: true }) diff --git a/examples/selenium/mocha-test/test/example.js b/examples/selenium/mocha-test/test/example.js index a0d27e50..be2a79e4 100644 --- a/examples/selenium/mocha-test/test/example.js +++ b/examples/selenium/mocha-test/test/example.js @@ -9,7 +9,6 @@ import { Builder, By, until } from 'selenium-webdriver' import { DevTools } from '@wdio/selenium-devtools' DevTools.configure({ - mode: 'trace', screencast: { enabled: true, quality: 70, maxWidth: 1280, maxHeight: 720 }, headless: true }) diff --git a/packages/core/src/action-mapping.ts b/packages/core/src/action-mapping.ts index 5c530f9c..43a39802 100644 --- a/packages/core/src/action-mapping.ts +++ b/packages/core/src/action-mapping.ts @@ -35,13 +35,15 @@ const ACTION_MAP: Record = { keys: { class: 'Keyboard', method: 'press' }, execute: { class: 'Page', method: 'evaluate' }, executeAsync: { class: 'Page', method: 'evaluate' }, - executeScript: { class: 'Page', method: 'evaluate' }, switchToFrame: { class: 'Frame', method: 'goto' }, touchAction: { class: 'Element', method: 'tap' } } -// clearValue / addValue are excluded: WDIO fires them internally inside setValue -// and they would produce duplicate trace entries. +// Excluded by design: +// clearValue / addValue — WDIO fires these inside setValue (duplicate events). +// executeScript — Selenium's `until` polling fires it ~50ms; also recurses +// because @wdio/elements uses executeScript inside captureActionSnapshot. +// WDIO's user-facing `execute`/`executeAsync` are still captured. export function mapCommandToAction(command: string): TraceAction | null { return ACTION_MAP[command] ?? null diff --git a/packages/nightwatch-devtools/package.json b/packages/nightwatch-devtools/package.json index c8ad4f92..21f4f17a 100644 --- a/packages/nightwatch-devtools/package.json +++ b/packages/nightwatch-devtools/package.json @@ -43,6 +43,7 @@ "dependencies": { "@wdio/devtools-backend": "workspace:*", "@wdio/devtools-script": "workspace:*", + "@wdio/elements": "workspace:^", "@wdio/logger": "^9.18.0", "fluent-ffmpeg": "^2.1.3", "import-meta-resolve": "^4.2.0", diff --git a/packages/nightwatch-devtools/src/action-snapshot.ts b/packages/nightwatch-devtools/src/action-snapshot.ts new file mode 100644 index 00000000..171cb9d0 --- /dev/null +++ b/packages/nightwatch-devtools/src/action-snapshot.ts @@ -0,0 +1,67 @@ +// Per-action snapshot capture for Nightwatch — fires only in `mode: 'trace'` +// for commands in ACTION_MAP. Wraps NightwatchBrowser in a minimal +// WebdriverIO.Browser-shaped shim so @wdio/elements can run its in-page +// scripts. Returns null on failure; capture errors must not break the test. + +import { + getBrowserAccessibilityTree, + getInteractableBrowserElements, + serializeWebSnapshot +} from '@wdio/elements' +import type { ActionSnapshot } from '@wdio/devtools-shared' +import type { NightwatchBrowser } from './types.js' + +interface BrowserWithUrl extends NightwatchBrowser { + getCurrentUrl?: () => Promise + getTitle?: () => Promise +} + +function shimAsWdioBrowser(browser: NightwatchBrowser): unknown { + return { + capabilities: browser.capabilities ?? {}, + isAndroid: false, + isIOS: false, + execute: (script: unknown, ...args: unknown[]) => + browser.execute( + script as string, + args.length === 1 && Array.isArray(args[0]) + ? (args[0] as unknown[]) + : args + ) + } +} + +export async function captureActionSnapshot( + browser: NightwatchBrowser, + command: string, + takeScreenshot?: () => Promise +): Promise { + try { + const timestamp = Date.now() + const b = browser as BrowserWithUrl + const browserLike = shimAsWdioBrowser(browser) as WebdriverIO.Browser + const [shot, url, title, tree, elements] = await Promise.all([ + takeScreenshot?.().catch(() => null) ?? Promise.resolve(null), + b.getCurrentUrl?.().catch(() => undefined) ?? Promise.resolve(undefined), + b.getTitle?.().catch(() => undefined) ?? Promise.resolve(undefined), + getBrowserAccessibilityTree(browserLike, { inViewportOnly: true }).catch( + () => [] + ), + getInteractableBrowserElements(browserLike, { + inViewportOnly: true + }).catch(() => []) + ]) + const snapshotText = serializeWebSnapshot(tree, { url, title }) + return { + timestamp, + command, + url, + title, + screenshot: shot ?? undefined, + elements, + snapshotText + } + } catch { + return null + } +} diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index 245a8210..d8033aee 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -141,6 +141,9 @@ class NightwatchDevToolsPlugin { get port() { return self.options.port }, + get mode() { + return self.options.mode + }, get screencastOptions() { return self.#screencastOptions }, @@ -488,11 +491,16 @@ class NightwatchDevToolsPlugin { return } try { + if (this.sessionCapturer.snapshotCaptures.length) { + await Promise.allSettled(this.sessionCapturer.snapshotCaptures) + } + const snapshots = this.sessionCapturer.actionSnapshots const zipPath = await writeTraceZip(this.sessionCapturer, { outputDir: resolveAdapterOutputDir({ configPath: this.#configPath }), - sessionId + sessionId, + actionSnapshots: snapshots.length ? snapshots : undefined }) log.info(`Trace.zip saved to ${zipPath}`) } catch (err) { diff --git a/packages/nightwatch-devtools/src/plugin-internals.ts b/packages/nightwatch-devtools/src/plugin-internals.ts index ce0165bf..afce58b4 100644 --- a/packages/nightwatch-devtools/src/plugin-internals.ts +++ b/packages/nightwatch-devtools/src/plugin-internals.ts @@ -26,6 +26,7 @@ export interface PluginInternals { options: { hostname: string; port: number; mode?: DevToolsMode } readonly hostname: string readonly port: number + readonly mode: DevToolsMode readonly screencastOptions: ScreencastOptions readonly bidiEnabled: boolean diff --git a/packages/nightwatch-devtools/src/session-init.ts b/packages/nightwatch-devtools/src/session-init.ts index 32df9d91..438132c8 100644 --- a/packages/nightwatch-devtools/src/session-init.ts +++ b/packages/nightwatch-devtools/src/session-init.ts @@ -25,6 +25,7 @@ import { SuiteManager } from './helpers/suiteManager.js' import { BrowserProxy } from './helpers/browserProxy.js' import { ScreencastRecorder } from './screencast.js' import type { + DevToolsMode, NightwatchBrowser, ScreencastOptions, SuiteStats @@ -37,6 +38,7 @@ export interface SessionInitCtx { readonly port: number readonly screencastOptions: ScreencastOptions readonly bidiEnabled: boolean + readonly mode: DevToolsMode sessionCapturer: SessionCapturer testReporter: TestReporter @@ -232,6 +234,7 @@ export async function ensureSessionInitialized( { port: ctx.port, hostname: ctx.hostname }, browser ) + ctx.sessionCapturer.traceMode = ctx.mode const connected = await ctx.sessionCapturer.waitForConnection(3000) if (!connected) { log.error('❌ Worker WebSocket failed to connect!') diff --git a/packages/nightwatch-devtools/src/session.ts b/packages/nightwatch-devtools/src/session.ts index d316f95e..87b3cde5 100644 --- a/packages/nightwatch-devtools/src/session.ts +++ b/packages/nightwatch-devtools/src/session.ts @@ -10,6 +10,8 @@ import { serializeError, type LogSource } from '@wdio/devtools-core' +import { mapCommandToAction } from '@wdio/devtools-core' +import { captureActionSnapshot } from './action-snapshot.js' import { NAVIGATION_COMMANDS } from './constants.js' import { parseNetworkFromPerfLogs, @@ -22,7 +24,13 @@ import { type CapturedPerformancePayload, applyPerformanceData } from '@wdio/devtools-core' -import type { CommandLog, LogLevel, NightwatchBrowser } from './types.js' +import type { + ActionSnapshot, + CommandLog, + DevToolsMode, + LogLevel, + NightwatchBrowser +} from './types.js' const log = logger('@wdio/nightwatch-devtools:SessionCapturer') @@ -105,6 +113,11 @@ export class SessionCapturer extends SessionCapturerBase { // capture path skips when set, so we don't double-emit network requests. bidiActive = false + // Populated by captureCommand when mode === 'trace' (set by the plugin). + traceMode: DevToolsMode = 'live' + readonly actionSnapshots: ActionSnapshot[] = [] + readonly snapshotCaptures: Promise[] = [] + constructor( devtoolsOptions: { hostname?: string; port?: number } = {}, browser?: NightwatchBrowser @@ -176,6 +189,24 @@ export class SessionCapturer extends SessionCapturerBase { }) } + if ( + this.traceMode === 'trace' && + !error && + this.#browser && + mapCommandToAction(command) + ) { + const browser = this.#browser + this.snapshotCaptures.push( + captureActionSnapshot(browser, command, () => + this.takeScreenshotViaHttp(browser) + ).then((snap) => { + if (snap) { + this.actionSnapshots.push(snap) + } + }) + ) + } + return true } diff --git a/packages/nightwatch-devtools/src/types.ts b/packages/nightwatch-devtools/src/types.ts index 9a81c03c..cb519808 100644 --- a/packages/nightwatch-devtools/src/types.ts +++ b/packages/nightwatch-devtools/src/types.ts @@ -2,6 +2,7 @@ export { TraceType, + type ActionSnapshot, type CommandLog, type ConsoleLog, type DevToolsMode, diff --git a/packages/selenium-devtools/package.json b/packages/selenium-devtools/package.json index 869c25d8..93ec7223 100644 --- a/packages/selenium-devtools/package.json +++ b/packages/selenium-devtools/package.json @@ -44,6 +44,7 @@ "dependencies": { "@wdio/devtools-backend": "workspace:*", "@wdio/devtools-script": "workspace:*", + "@wdio/elements": "workspace:^", "@wdio/logger": "^9.18.0", "stacktrace-parser": "^0.1.11", "webdriverio": "^9.27.2", diff --git a/packages/selenium-devtools/src/action-snapshot.ts b/packages/selenium-devtools/src/action-snapshot.ts new file mode 100644 index 00000000..84a081a4 --- /dev/null +++ b/packages/selenium-devtools/src/action-snapshot.ts @@ -0,0 +1,62 @@ +// Per-action snapshot capture for Selenium — fires only in `mode: 'trace'` +// for commands in ACTION_MAP. Wraps the SeleniumDriverLike in a minimal +// WebdriverIO.Browser-shaped shim so @wdio/elements can run its in-page +// scripts via driver.executeScript. Returns null on failure; capture errors +// must not break the user's test. + +import { + getBrowserAccessibilityTree, + getInteractableBrowserElements, + serializeWebSnapshot +} from '@wdio/elements' +import type { ActionSnapshot } from '@wdio/devtools-shared' +import type { SeleniumDriverLike } from './types.js' + +interface DriverWithUrl extends SeleniumDriverLike { + getCurrentUrl?: () => Promise + getTitle?: () => Promise +} + +function shimAsWdioBrowser(driver: SeleniumDriverLike): unknown { + return { + capabilities: {}, + isAndroid: false, + isIOS: false, + execute: (script: unknown, ...args: unknown[]) => + driver.executeScript(script as string, ...args) + } +} + +export async function captureActionSnapshot( + driver: SeleniumDriverLike, + command: string +): Promise { + try { + const timestamp = Date.now() + const d = driver as DriverWithUrl + const browserLike = shimAsWdioBrowser(driver) as WebdriverIO.Browser + const [screenshot, url, title, tree, elements] = await Promise.all([ + d.takeScreenshot?.().catch(() => undefined) ?? Promise.resolve(undefined), + d.getCurrentUrl?.().catch(() => undefined) ?? Promise.resolve(undefined), + d.getTitle?.().catch(() => undefined) ?? Promise.resolve(undefined), + getBrowserAccessibilityTree(browserLike, { inViewportOnly: true }).catch( + () => [] + ), + getInteractableBrowserElements(browserLike, { + inViewportOnly: true + }).catch(() => []) + ]) + const snapshotText = serializeWebSnapshot(tree, { url, title }) + return { + timestamp, + command, + url, + title, + screenshot, + elements, + snapshotText + } + } catch { + return null + } +} diff --git a/packages/selenium-devtools/src/helpers/commandPostActions.ts b/packages/selenium-devtools/src/helpers/commandPostActions.ts index d90b9880..ad90666c 100644 --- a/packages/selenium-devtools/src/helpers/commandPostActions.ts +++ b/packages/selenium-devtools/src/helpers/commandPostActions.ts @@ -3,17 +3,21 @@ import { CAPTURE_PERFORMANCE_SCRIPT, applyPerformanceData, errorMessage, + mapCommandToAction, toError, type CapturedPerformancePayload, type RetryTracker } from '@wdio/devtools-core' import { getDriverOriginals, getElementOriginals } from '../driverPatcher.js' import { captureOrReplaceCommand } from './captureOrReplaceCommand.js' +import { captureActionSnapshot } from '../action-snapshot.js' import type { SessionCapturer } from '../session.js' import type { TestManager } from './testManager.js' import type { + ActionSnapshot, CapturedCommand, CommandLog, + DevToolsMode, SeleniumDriverLike } from '../types.js' @@ -144,10 +148,12 @@ export interface OnCommandCtx { readonly sessionCapturer: SessionCapturer | undefined readonly testManager: TestManager | undefined readonly retryTracker: RetryTracker - readonly options: { captureScreenshots: boolean } + readonly options: { captureScreenshots: boolean; mode?: DevToolsMode } readonly scriptInjected: boolean readonly finalized: boolean readonly driver: SeleniumDriverLike | undefined + readonly actionSnapshots: ActionSnapshot[] + readonly snapshotCaptures: Promise[] setScriptInjected(v: boolean): void } @@ -214,4 +220,19 @@ export async function handleOnCommand( ctx.driver ) } + if ( + ctx.options.mode === 'trace' && + !error && + ctx.driver && + mapCommandToAction(cmd.command) + ) { + const driver = ctx.driver + ctx.snapshotCaptures.push( + captureActionSnapshot(driver, cmd.command).then((snap) => { + if (snap) { + ctx.actionSnapshots.push(snap) + } + }) + ) + } } diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index e02204fa..4a673549 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -48,6 +48,7 @@ import { NAVIGATION_COMMANDS } from './constants.js' import { + type ActionSnapshot, type CapturedCommand, type DevToolsMode, type DevToolsOptions, @@ -86,6 +87,8 @@ class SeleniumDevToolsPlugin { #retryTracker = new RetryTracker() #screencast?: ScreencastRecorder #screencastOptions: ScreencastOptions + #actionSnapshots: ActionSnapshot[] = [] + #snapshotCaptures: Promise[] = [] #sessionId?: string #uiUrlOpened = false #testFilePath?: string @@ -375,6 +378,12 @@ class SeleniumDevToolsPlugin { setScriptInjected: (v) => { self.#scriptInjected = v }, + get actionSnapshots() { + return self.#actionSnapshots + }, + get snapshotCaptures() { + return self.#snapshotCaptures + }, ensureBackendStarted: () => self.ensureBackendStarted(), flushPendingTestActions: () => self.#flushPendingTestActions(), resetRetryTracker: () => self.#retryTracker.reset(), diff --git a/packages/selenium-devtools/src/plugin-internals.ts b/packages/selenium-devtools/src/plugin-internals.ts index 9484d07b..6bd73e99 100644 --- a/packages/selenium-devtools/src/plugin-internals.ts +++ b/packages/selenium-devtools/src/plugin-internals.ts @@ -12,7 +12,12 @@ import type { TestReporter } from './reporter.js' import type { SuiteManager } from './helpers/suiteManager.js' import type { TestManager } from './helpers/testManager.js' import type { ScreencastRecorder } from './screencast.js' -import type { ScreencastOptions, SeleniumDriverLike } from './types.js' +import type { + ActionSnapshot, + DevToolsMode, + ScreencastOptions, + SeleniumDriverLike +} from './types.js' import type { RetryTracker } from '@wdio/devtools-core' import type { PendingTestAction, PendingScenario } from './test-management.js' @@ -24,6 +29,7 @@ export interface PluginInternals { openUi: boolean captureScreenshots: boolean rerunCommand?: string + mode?: DevToolsMode } readonly screencastOptions: ScreencastOptions readonly runner: string @@ -49,6 +55,10 @@ export interface PluginInternals { pendingTestActions: PendingTestAction[] pendingScenario: PendingScenario | null + // trace-mode snapshot accumulators (mode === 'trace' only) + readonly actionSnapshots: ActionSnapshot[] + readonly snapshotCaptures: Promise[] + // Plugin-side delegates setFinalized(v: boolean): void setScriptInjected(v: boolean): void diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index f3c8c483..b13306b0 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -24,6 +24,7 @@ import { buildDriverMetadata } from './helpers/driverMetadata.js' import { attachBidiHandlers, buildBidiSinks } from './bidi.js' import { gracefulShutdown } from './helpers/processHooks.js' import type { + ActionSnapshot, DevToolsMode, Metadata, ScreencastOptions, @@ -60,6 +61,10 @@ export interface SessionLifecycleCtx { testFilePath: string | undefined keepAliveTimer: ReturnType | undefined + // Populated by handleOnCommand when mode === 'trace'. + readonly actionSnapshots: ActionSnapshot[] + readonly snapshotCaptures: Promise[] + setFinalized(v: boolean): void ensureBackendStarted(): Promise flushPendingTestActions(): void @@ -228,11 +233,17 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { const sessionId = capturerAtStart?.metadata?.sessionId if (ctx.options.mode === 'trace' && capturerAtStart && sessionId) { try { + if (ctx.snapshotCaptures.length) { + await Promise.allSettled(ctx.snapshotCaptures) + } const zipPath = await writeTraceZip(capturerAtStart, { outputDir: resolveAdapterOutputDir({ testFilePath: testFilePathAtStart }), - sessionId + sessionId, + actionSnapshots: ctx.actionSnapshots.length + ? ctx.actionSnapshots + : undefined }) log.info(`Trace.zip saved to ${zipPath}`) } catch (err) { @@ -252,10 +263,16 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { return } - // trace mode: no UI to wait for; force-shutdown so the backend HTTP - // server stops keeping the event loop alive. + // trace mode: no UI to wait for; close the WS so the backend can wind + // down naturally. process.exit is avoided — Jest/runners may treat + // forced exits as failures. if (ctx.options.mode === 'trace' && !ctx.isReuse) { - await completeShutdown(ctx, shutdownStart) + try { + await ctx.sessionCapturer?.closeWebSocket() + } catch { + /* best-effort */ + } + log.info(`🛑 Shutdown complete (${Date.now() - shutdownStart}ms)`) return } diff --git a/packages/selenium-devtools/src/types.ts b/packages/selenium-devtools/src/types.ts index 2e60f850..dc451c45 100644 --- a/packages/selenium-devtools/src/types.ts +++ b/packages/selenium-devtools/src/types.ts @@ -2,6 +2,7 @@ export { TraceType, + type ActionSnapshot, type CommandLog, type ConsoleLog, type DevToolsMode, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4b3cee2f..cd5c1104 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -351,6 +351,9 @@ importers: '@wdio/devtools-script': specifier: workspace:* version: link:../script + '@wdio/elements': + specifier: workspace:^ + version: link:../elements '@wdio/logger': specifier: ^9.18.0 version: 9.18.0 @@ -428,6 +431,9 @@ importers: '@wdio/devtools-script': specifier: workspace:* version: link:../script + '@wdio/elements': + specifier: workspace:^ + version: link:../elements '@wdio/logger': specifier: ^9.18.0 version: 9.18.0 From b756583a8755cacad45f10dc9784f24ded20906e Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Mon, 8 Jun 2026 16:41:23 +0530 Subject: [PATCH 07/13] Update README --- README.md | 21 +++++++++++++++++++++ packages/nightwatch-devtools/README.md | 3 ++- packages/selenium-devtools/README.md | 9 +++++++++ packages/service/README.md | 3 ++- 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 62a6448f..a68f3e83 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,27 @@ Real-time capture of browser-side events through the WebDriver BiDi protocol — When BiDi is active in Selenium or Nightwatch, the per-command Chrome performance-log network-capture path is gated off so requests don't appear twice in the dashboard. The attach + sink logic lives in `@wdio/devtools-core`'s `bidi.ts` — same module both adapters consume. +### 📦 Trace mode (trace.zip) + +Headless capture path — no DevTools UI window opens. At session end the adapter writes a `trace-.zip` next to the user's spec / config file, suitable for offline replay, AI-agent diffing, or any consumer that prefers a portable artifact over a live UI. + +| Adapter | How to enable | +|---|---| +| **WebdriverIO** | `services: [['devtools', { mode: 'trace' }]]` | +| **Selenium** | `DevTools.configure({ mode: 'trace' })` (before importing `selenium-webdriver`) | +| **Nightwatch** | `globals: nightwatchDevtools({ mode: 'trace' })` | + +The zip contains: +- `trace.trace` — NDJSON `context-options` + `before`/`after` action events +- `trace.network` — HAR-style network entries derived from the existing capture +- `resources/page@-.jpeg` — screenshot per user-facing action +- `resources/elements-page@-.json` — flat interactable element list from `@wdio/elements` +- `resources/snapshot-page@-.txt` — depth-indented accessibility-tree snapshot (AI-friendly) + +What counts as a user-facing action is filtered through an allow-list in `@wdio/devtools-core/action-mapping.ts` (`url`, `click`, `setValue`, `sendKeys`, `get`, etc.). Internal commands like `findElement`/`waitUntil`/`executeScript` don't produce trace entries. + +Trace mode and live mode are **mutually exclusive** — `screencast` options are ignored in trace mode (live-mode feature). Live and trace serve different audiences (humans debugging vs. agents diffing), and stacking them only costs perf. + ### 🔍︎ TestLens - **Code Intelligence**: View test definitions directly in your editor - **Run/Debug Actions**: Execute individual tests or suites with inline CodeLens actions diff --git a/packages/nightwatch-devtools/README.md b/packages/nightwatch-devtools/README.md index 00da385d..5f404f5d 100644 --- a/packages/nightwatch-devtools/README.md +++ b/packages/nightwatch-devtools/README.md @@ -82,8 +82,9 @@ module.exports = { |--------|------|---------|-------------| | `port` | `number` | `3000` | Port for the DevTools backend server. Auto-incremented if already in use. | | `hostname` | `string` | `'localhost'` | Hostname the backend server binds to. | -| `screencast` | `ScreencastOptions` | `{ enabled: false }` | Session video recording (see [Screencast](#screencast)). | +| `screencast` | `ScreencastOptions` | `{ enabled: false }` | Session video recording — live mode only (see [Screencast](#screencast)). | | `bidi` | `boolean` | `false` | Opt into WebDriver BiDi capture for browser console + JS exceptions + network. Requires `webSocketUrl: true` in your capabilities and a BiDi-capable chromedriver. When attached, the per-command Chrome perf-log network path is gated off so requests don't duplicate. | +| `mode` | `'live' \| 'trace'` | `'live'` | `'live'` opens the DevTools UI window; `'trace'` skips the UI and writes a `trace-.zip` next to your `nightwatch.conf.cjs` at run end. See [Trace mode](../../README.md#-trace-mode-tracezip). | ```javascript globals: nightwatchDevtools({ diff --git a/packages/selenium-devtools/README.md b/packages/selenium-devtools/README.md index 7722139c..89d099a9 100644 --- a/packages/selenium-devtools/README.md +++ b/packages/selenium-devtools/README.md @@ -311,6 +311,15 @@ DevTools.configure({ captureScreenshots: false }) DevTools.configure({ rerunCommand: 'npm test -- --grep "{{testName}}"' }) ``` +#### `mode` — live UI vs. headless trace.zip +**Default:** `'live'` (launches the dashboard window). Set to `'trace'` to skip the dashboard entirely and write a `trace-.zip` next to your test file at session end — meant for CI / offline replay / agentic diffing. `screencast` is ignored in trace mode (live-mode-only feature). + +```javascript +DevTools.configure({ mode: 'trace' }) +``` + +See [Trace mode](../../README.md#-trace-mode-tracezip) in the root README for the trace.zip layout and consumer notes. + --- ## Common recipes diff --git a/packages/service/README.md b/packages/service/README.md index 887f885d..d5532798 100644 --- a/packages/service/README.md +++ b/packages/service/README.md @@ -46,7 +46,8 @@ services: [['devtools', options]] | `port` | `number` | random | Port the DevTools UI server listens on | | `hostname` | `string` | `'localhost'` | Hostname the DevTools UI server binds to | | `devtoolsCapabilities` | `Capabilities` | Chrome 1600×1200 | Capabilities used to open the DevTools UI window | -| `screencast` | `ScreencastOptions` | — | Session video recording (see below) | +| `screencast` | `ScreencastOptions` | — | Session video recording (live mode only — see below) | +| `mode` | `'live' \| 'trace'` | `'live'` | `'live'` opens the DevTools UI window; `'trace'` skips the UI and writes a `trace-.zip` at session end. See [Trace mode](../../README.md#-trace-mode-tracezip) | ## Screencast Recording From 4dc81e1a59a443a3cbda8a6120e50d571c13e19f Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Mon, 8 Jun 2026 17:19:52 +0530 Subject: [PATCH 08/13] fix: CodeQl and lint fix --- packages/elements/src/accessibility-tree.ts | 85 ++++---- packages/elements/src/aria-roles.ts | 68 ++++++ packages/elements/src/browser-elements.ts | 56 ++--- .../src/locators/locator-generation.ts | 196 ++++++++++-------- packages/elements/src/locators/xml-parsing.ts | 4 + packages/elements/src/snapshot.ts | 17 ++ .../src/session-lifecycle.ts | 6 +- 7 files changed, 272 insertions(+), 160 deletions(-) create mode 100644 packages/elements/src/aria-roles.ts diff --git a/packages/elements/src/accessibility-tree.ts b/packages/elements/src/accessibility-tree.ts index a5e593bb..eab1aeb1 100644 --- a/packages/elements/src/accessibility-tree.ts +++ b/packages/elements/src/accessibility-tree.ts @@ -6,6 +6,8 @@ * It must be self-contained with no external dependencies */ +import { CONTAINER_ROLES, INPUT_TYPE_ROLES } from './aria-roles.js' + export interface AccessibilityNode { role: string name: string @@ -23,44 +25,27 @@ export interface AccessibilityNode { isInViewport?: boolean } -const accessibilityTreeScript = (inViewportOnly: boolean) => +// Page-injected script: WDIO's `browser.execute` stringifies the arrow body +// and runs it in the browser. Module-level closure values are lost in +// stringification, so ARIA tables defined in aria-roles.ts are passed in +// as execute() args and re-bound here. Same constraint forces every helper +// (getRole/getAccessibleName/getSelector/...) to live inside the IIFE. +// eslint-disable-next-line max-lines-per-function +const accessibilityTreeScript = ( + inViewportOnly: boolean, + inputTypeRoles: Record, + containerRolesArr: readonly string[] +) => + // eslint-disable-next-line max-lines-per-function -- see comment above (function () { - const INPUT_TYPE_ROLES: Record = { - text: 'textbox', - search: 'searchbox', - email: 'textbox', - url: 'textbox', - tel: 'textbox', - password: 'textbox', - number: 'spinbutton', - checkbox: 'checkbox', - radio: 'radio', - range: 'slider', - submit: 'button', - reset: 'button', - image: 'button', - file: 'button', - color: 'button' - } - - // Container roles: named only via aria-label/aria-labelledby, not textContent - const CONTAINER_ROLES = new Set([ - 'navigation', - 'banner', - 'contentinfo', - 'complementary', - 'main', - 'form', - 'region', - 'group', - 'list', - 'listitem', - 'table', - 'row', - 'rowgroup', - 'generic' - ]) - + const INPUT_TYPE_ROLES = inputTypeRoles + const CONTAINER_ROLES = new Set(containerRolesArr) + + // ARIA role-resolution decision tree: each `if (tag === ...)` branch + // maps an HTML element to its implicit role per the WAI-ARIA spec. + // Splitting per-tag would scatter spec-equivalence groups across + // helpers without improving clarity. + // eslint-disable-next-line max-lines-per-function function getRole(el: HTMLElement): string | null { const explicit = el.getAttribute('role') if (explicit) { @@ -154,6 +139,13 @@ const accessibilityTreeScript = (inViewportOnly: boolean) => return null } + // Implements the WAI-ARIA Accessible Name Computation algorithm. + // The 6 sequential fallback steps (aria-label → aria-labelledby → + // tag-specific → placeholder → title → childImg.alt) are spec-ordered; + // each step must run only if prior steps yielded nothing, so they don't + // factor into independent helpers without re-threading the "found" + // signal through every call. + // eslint-disable-next-line max-lines-per-function function getAccessibleName(el: HTMLElement, role: string | null): string { const ariaLabel = el.getAttribute('aria-label') if (ariaLabel) { @@ -229,6 +221,11 @@ const accessibilityTreeScript = (inViewportOnly: boolean) => return (el.textContent?.trim().replace(/\s+/g, ' ') || '').slice(0, 200) } + // Selector synthesis: tries 6 strategies in priority order (text → aria-label + // → data-testid → name attr → id → CSS path) and returns the first one that + // is unique on the page. Splitting per strategy would obscure the priority + // ordering and force re-uniqueness-checking across helpers. + // eslint-disable-next-line max-lines-per-function function getSelector(element: HTMLElement): string { const tag = element.tagName.toLowerCase() @@ -468,13 +465,21 @@ const accessibilityTreeScript = (inViewportOnly: boolean) => * @param options {@link inViewportOnly} defaults to `true` — only nodes * whose bounding rect intersects the viewport are included. */ +// WDIO's typed Browser.execute overloads don't accept our generic injected +// script — narrow to a permissive execute() shape at the boundary instead. +type ExecuteLike = { + execute: (script: unknown, ...args: unknown[]) => Promise +} + export async function getBrowserAccessibilityTree( browser: WebdriverIO.Browser, options: { inViewportOnly?: boolean } = {} ): Promise { const { inViewportOnly = true } = options - return (browser as any).execute( + return (browser as unknown as ExecuteLike).execute( accessibilityTreeScript, - inViewportOnly - ) as unknown as Promise + inViewportOnly, + INPUT_TYPE_ROLES, + CONTAINER_ROLES + ) as Promise } diff --git a/packages/elements/src/aria-roles.ts b/packages/elements/src/aria-roles.ts new file mode 100644 index 00000000..b78736a9 --- /dev/null +++ b/packages/elements/src/aria-roles.ts @@ -0,0 +1,68 @@ +// WAI-ARIA role data tables shared between the accessibility-tree and +// browser-elements page-injected scripts. Defined at module level so they +// can be type-checked + reused; the consuming scripts receive them as +// arguments to `browser.execute()` since values declared at module scope +// don't survive the function-source stringification that injects the script. + +/** HTML → implicit WAI-ARIA role. */ +export const INPUT_TYPE_ROLES: Record = { + text: 'textbox', + search: 'searchbox', + email: 'textbox', + url: 'textbox', + tel: 'textbox', + password: 'textbox', + number: 'spinbutton', + checkbox: 'checkbox', + radio: 'radio', + range: 'slider', + submit: 'button', + reset: 'button', + image: 'button', + file: 'button', + color: 'button' +} + +/** ARIA roles whose accessible name comes only from aria-label/labelledby, + * never from textContent (otherwise the section text leaks into the name). */ +export const CONTAINER_ROLES: readonly string[] = [ + 'navigation', + 'banner', + 'contentinfo', + 'complementary', + 'main', + 'form', + 'region', + 'group', + 'list', + 'listitem', + 'table', + 'row', + 'rowgroup', + 'generic' +] + +/** CSS selector matching all elements treated as interactable by the page-side + * element walker. Includes native form/anchor elements plus ARIA-role aliases. */ +export const INTERACTABLE_SELECTORS: readonly string[] = [ + 'a[href]', + 'button', + 'input:not([type="hidden"])', + 'select', + 'textarea', + '[role="button"]', + '[role="link"]', + '[role="checkbox"]', + '[role="radio"]', + '[role="tab"]', + '[role="menuitem"]', + '[role="combobox"]', + '[role="option"]', + '[role="switch"]', + '[role="slider"]', + '[role="textbox"]', + '[role="searchbox"]', + '[role="spinbutton"]', + '[contenteditable="true"]', + '[tabindex]:not([tabindex="-1"])' +] diff --git a/packages/elements/src/browser-elements.ts b/packages/elements/src/browser-elements.ts index 0e38e5ac..208d7627 100644 --- a/packages/elements/src/browser-elements.ts +++ b/packages/elements/src/browser-elements.ts @@ -6,6 +6,8 @@ * It must be self-contained with no external dependencies */ +import { INTERACTABLE_SELECTORS } from './aria-roles.js' + export interface BrowserElementInfo { tagName: string name: string // computed accessible name (ARIA spec) @@ -23,30 +25,20 @@ export interface GetBrowserElementsOptions { inViewportOnly?: boolean } -const elementsScript = (includeBounds: boolean, inViewportOnly: boolean) => +// Page-injected script: WDIO's `browser.execute` stringifies the arrow body +// and runs it in the browser. The selector list lives in aria-roles.ts at +// module scope so it stays type-checked, but it's passed in via execute() +// args because module-level values don't survive script stringification. +// Same constraint forces every helper below to live inside the IIFE. +// eslint-disable-next-line max-lines-per-function +const elementsScript = ( + includeBounds: boolean, + inViewportOnly: boolean, + interactableSelectorsArr: readonly string[] +) => + // eslint-disable-next-line max-lines-per-function -- see comment above (function () { - const interactableSelectors = [ - 'a[href]', - 'button', - 'input:not([type="hidden"])', - 'select', - 'textarea', - '[role="button"]', - '[role="link"]', - '[role="checkbox"]', - '[role="radio"]', - '[role="tab"]', - '[role="menuitem"]', - '[role="combobox"]', - '[role="option"]', - '[role="switch"]', - '[role="slider"]', - '[role="textbox"]', - '[role="searchbox"]', - '[role="spinbutton"]', - '[contenteditable="true"]', - '[tabindex]:not([tabindex="-1"])' - ].join(',') + const interactableSelectors = interactableSelectorsArr.join(',') function isVisible(element: HTMLElement): boolean { if (typeof element.checkVisibility === 'function') { @@ -66,6 +58,9 @@ const elementsScript = (includeBounds: boolean, inViewportOnly: boolean) => ) } + // WAI-ARIA Accessible Name Computation algorithm — see accessibility-tree.ts + // for the longer rationale. Sequential spec-ordered fallback steps. + // eslint-disable-next-line max-lines-per-function function getAccessibleName(el: HTMLElement): string { // 1. aria-label const ariaLabel = el.getAttribute('aria-label') @@ -137,6 +132,9 @@ const elementsScript = (includeBounds: boolean, inViewportOnly: boolean) => return (el.textContent?.trim().replace(/\s+/g, ' ') || '').slice(0, 200) } + // Selector synthesis — see accessibility-tree.ts for the longer rationale. + // Tries 6 priority-ordered strategies; each must re-check uniqueness. + // eslint-disable-next-line max-lines-per-function function getSelector(element: HTMLElement): string { const tag = element.tagName.toLowerCase() @@ -295,9 +293,15 @@ export async function getInteractableBrowserElements( options: GetBrowserElementsOptions = {} ): Promise { const { includeBounds = false, inViewportOnly = true } = options - return (browser as any).execute( + // WDIO's typed Browser.execute overloads don't accept the injected script — + // narrow to a permissive execute() shape at the boundary. + type ExecuteLike = { + execute: (script: unknown, ...args: unknown[]) => Promise + } + return (browser as unknown as ExecuteLike).execute( elementsScript, includeBounds, - inViewportOnly - ) as unknown as Promise + inViewportOnly, + INTERACTABLE_SELECTORS + ) as Promise } diff --git a/packages/elements/src/locators/locator-generation.ts b/packages/elements/src/locators/locator-generation.ts index cba05cc8..5433e280 100644 --- a/packages/elements/src/locators/locator-generation.ts +++ b/packages/elements/src/locators/locator-generation.ts @@ -94,7 +94,10 @@ function checkUniqueness( return checkXPathUniqueness(ctx.parsedDOM, xpath, targetNode) } - const match = xpath.match(/\/\/\*\[@([^=]+)="([^"]+)"\]/) + // Bounded + disjoint negated classes: attr names can never contain `=`, + // `"` or `]`; values can't contain `"`. Bounds prevent polynomial + // backtracking on adversarial input (CodeQL: js/polynomial-redos). + const match = xpath.match(/\/\/\*\[@([^="\]]{1,200})="([^"]{1,2000})"\]/) if (match) { const [, attr, value] = match return { isUnique: isAttributeUnique(ctx.sourceXML, attr, value) } @@ -410,108 +413,123 @@ function buildXPath( /** * Get simple locators based on single attributes */ -function getSimpleSuggestedLocators( - element: JSONElement, +function appendAndroidSimpleLocators( + attrs: JSONElement['attributes'], ctx: LocatorContext, - automationName: string, + inUiAutomatorScope: boolean, + results: [LocatorStrategy, string][], targetNode?: XMLNode -): [LocatorStrategy, string][] { - const results: [LocatorStrategy, string][] = [] - const isAndroid = automationName.toLowerCase().includes('uiautomator') - const attrs = element.attributes - const inUiAutomatorScope = isAndroid - ? isInUiAutomatorScope(element, ctx.parsedDOM) - : true - - if (isAndroid) { - // Resource ID - const resourceId = attrs['resource-id'] - if (isValidValue(resourceId)) { - const xpath = `//*[@resource-id="${resourceId}"]` - const uniqueness = checkUniqueness(ctx, xpath, targetNode) - - if (uniqueness.isUnique && inUiAutomatorScope) { - results.push([ - 'id', - `android=new UiSelector().resourceId("${resourceId}")` - ]) - } else if (uniqueness.index && inUiAutomatorScope) { - const base = `android=new UiSelector().resourceId("${resourceId}")` - results.push(['id', generateIndexedUiAutomator(base, uniqueness.index)]) - } +): void { + const resourceId = attrs['resource-id'] + if (isValidValue(resourceId)) { + const uniqueness = checkUniqueness( + ctx, + `//*[@resource-id="${resourceId}"]`, + targetNode + ) + const base = `android=new UiSelector().resourceId("${resourceId}")` + if (uniqueness.isUnique && inUiAutomatorScope) { + results.push(['id', base]) + } else if (uniqueness.index && inUiAutomatorScope) { + results.push(['id', generateIndexedUiAutomator(base, uniqueness.index)]) } + } - // Content Description - const contentDesc = attrs['content-desc'] - if (isValidValue(contentDesc)) { - const xpath = `//*[@content-desc="${contentDesc}"]` - const uniqueness = checkUniqueness(ctx, xpath, targetNode) - - if (uniqueness.isUnique) { - results.push(['accessibility-id', `~${contentDesc}`]) - } + const contentDesc = attrs['content-desc'] + if (isValidValue(contentDesc)) { + const uniqueness = checkUniqueness( + ctx, + `//*[@content-desc="${contentDesc}"]`, + targetNode + ) + if (uniqueness.isUnique) { + results.push(['accessibility-id', `~${contentDesc}`]) } + } - // Text - const text = attrs.text - if (isValidValue(text) && text.length < 100) { - const xpath = `//*[@text="${escapeText(text)}"]` - const uniqueness = checkUniqueness(ctx, xpath, targetNode) - - if (uniqueness.isUnique && inUiAutomatorScope) { - results.push([ - 'text', - `android=new UiSelector().text("${escapeText(text)}")` - ]) - } else if (uniqueness.index && inUiAutomatorScope) { - const base = `android=new UiSelector().text("${escapeText(text)}")` - results.push([ - 'text', - generateIndexedUiAutomator(base, uniqueness.index) - ]) - } + const text = attrs.text + if (isValidValue(text) && text.length < 100) { + const uniqueness = checkUniqueness( + ctx, + `//*[@text="${escapeText(text)}"]`, + targetNode + ) + const base = `android=new UiSelector().text("${escapeText(text)}")` + if (uniqueness.isUnique && inUiAutomatorScope) { + results.push(['text', base]) + } else if (uniqueness.index && inUiAutomatorScope) { + results.push(['text', generateIndexedUiAutomator(base, uniqueness.index)]) } - } else { - // iOS: Accessibility ID (name) - const name = attrs.name - if (isValidValue(name)) { - const xpath = `//*[@name="${name}"]` - const uniqueness = checkUniqueness(ctx, xpath, targetNode) - - if (uniqueness.isUnique) { - results.push(['accessibility-id', `~${name}`]) - } + } +} + +function appendIOSSimpleLocators( + attrs: JSONElement['attributes'], + ctx: LocatorContext, + results: [LocatorStrategy, string][], + targetNode?: XMLNode +): void { + const name = attrs.name + if (isValidValue(name)) { + const uniqueness = checkUniqueness(ctx, `//*[@name="${name}"]`, targetNode) + if (uniqueness.isUnique) { + results.push(['accessibility-id', `~${name}`]) } + } - // iOS: Label - const label = attrs.label - if (isValidValue(label) && label !== attrs.name) { - const xpath = `//*[@label="${escapeText(label)}"]` - const uniqueness = checkUniqueness(ctx, xpath, targetNode) - - if (uniqueness.isUnique) { - results.push([ - 'predicate-string', - `-ios predicate string:label == "${escapeText(label)}"` - ]) - } + const label = attrs.label + if (isValidValue(label) && label !== attrs.name) { + const uniqueness = checkUniqueness( + ctx, + `//*[@label="${escapeText(label)}"]`, + targetNode + ) + if (uniqueness.isUnique) { + results.push([ + 'predicate-string', + `-ios predicate string:label == "${escapeText(label)}"` + ]) } + } - // iOS: Value - const value = attrs.value - if (isValidValue(value)) { - const xpath = `//*[@value="${escapeText(value)}"]` - const uniqueness = checkUniqueness(ctx, xpath, targetNode) - - if (uniqueness.isUnique) { - results.push([ - 'predicate-string', - `-ios predicate string:value == "${escapeText(value)}"` - ]) - } + const value = attrs.value + if (isValidValue(value)) { + const uniqueness = checkUniqueness( + ctx, + `//*[@value="${escapeText(value)}"]`, + targetNode + ) + if (uniqueness.isUnique) { + results.push([ + 'predicate-string', + `-ios predicate string:value == "${escapeText(value)}"` + ]) } } +} +function getSimpleSuggestedLocators( + element: JSONElement, + ctx: LocatorContext, + automationName: string, + targetNode?: XMLNode +): [LocatorStrategy, string][] { + const results: [LocatorStrategy, string][] = [] + const isAndroid = automationName.toLowerCase().includes('uiautomator') + const inUiAutomatorScope = isAndroid + ? isInUiAutomatorScope(element, ctx.parsedDOM) + : true + if (isAndroid) { + appendAndroidSimpleLocators( + element.attributes, + ctx, + inUiAutomatorScope, + results, + targetNode + ) + } else { + appendIOSSimpleLocators(element.attributes, ctx, results, targetNode) + } return results } diff --git a/packages/elements/src/locators/xml-parsing.ts b/packages/elements/src/locators/xml-parsing.ts index a100a042..09b1c13a 100644 --- a/packages/elements/src/locators/xml-parsing.ts +++ b/packages/elements/src/locators/xml-parsing.ts @@ -312,6 +312,10 @@ export function countAttributeOccurrences( value: string ): number { const escapedValue = value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + // `attribute` is a fixed set of XML attr names (resource-id, content-desc, + // text, name, label, value) controlled by the locator pipeline; `value` + // is regex-escaped above. No user-controlled regex source. + // eslint-disable-next-line security/detect-non-literal-regexp const pattern = new RegExp(`${attribute}=["']${escapedValue}["']`, 'g') const matches = sourceXML.match(pattern) return matches ? matches.length : 0 diff --git a/packages/elements/src/snapshot.ts b/packages/elements/src/snapshot.ts index ad45c1c4..b49de4c7 100644 --- a/packages/elements/src/snapshot.ts +++ b/packages/elements/src/snapshot.ts @@ -69,6 +69,12 @@ export interface WebSnapshotOptions { * @param context Optional page context for the header line * @param options {@link WebSnapshotOptions} */ +// Single linear pass over the flat node list — per-node decisions (skip-by- +// viewport, role classification, statictext echo dedup, interactive vs +// structural rendering) must stay together so the inferred-purpose lookup +// can see siblings. ROADMAP P2: collapse with mobile pipeline into one +// `serializeSnapshot()`; until then this is the canonical web walker. +// eslint-disable-next-line max-lines-per-function export function serializeWebSnapshot( nodes: AccessibilityNode[], context?: { url?: string; title?: string }, @@ -429,6 +435,12 @@ interface MobileFlatNode { * First pass: walk the JSONElement tree, apply viewport filtering and * collect every node into a flat array with semantic roles and selectors. */ +// Recursive tree walker — splitting the viewport filter, locator pipeline, +// fallback, and node-emit branches would require threading the accumulator +// + walk options through 4 helpers. ROADMAP P0/P1: this whole pipeline gets +// merged with generateAllElementLocators; until that consolidation lands, +// keep as one walker. +// eslint-disable-next-line max-lines-per-function function collectMobileNodes( element: JSONElement, platform: 'android' | 'ios', @@ -609,6 +621,11 @@ function countSelectors(nodes: MobileFlatNode[]): Map { return counts } +// Render pass mirrors serializeWebSnapshot's structure (linear node-by-node +// emission with parent-context lookback, statictext echo dedup, layout-noise +// collapse, .instance(N) indexing). Splitting per-decision would lose the +// shared per-node state. ROADMAP P2: collapse with web pipeline. +// eslint-disable-next-line max-lines-per-function function renderMobileNodes(nodes: MobileFlatNode[]): string[] { const lines: string[] = [] const selectorCounts = countSelectors(nodes) diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index b13306b0..13934512 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -254,11 +254,7 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { logSessionSummary(ctx) ctx.sessionCapturer?.cleanup() - if ( - ctx.options.openUi && - ctx.options.mode !== 'trace' && - !ctx.isReuse - ) { + if (ctx.options.openUi && ctx.options.mode !== 'trace' && !ctx.isReuse) { handleInteractivePath(ctx, shutdownStart) return } From 60f76f99b24c5bf32e05b96a1446ba1ef23c9741 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Tue, 9 Jun 2026 14:07:21 +0530 Subject: [PATCH 09/13] Mobile-aware context-options, snapshot probe timeouts, chronological event ordering --- packages/core/src/index.ts | 1 + packages/core/src/trace-exporter.ts | 65 ++++++++++++++++- packages/core/src/with-timeout.ts | 27 +++++++ .../src/action-snapshot.ts | 19 +++-- .../selenium-devtools/src/action-snapshot.ts | 19 +++-- packages/service/src/action-snapshot.ts | 70 ++++++++++++++----- 6 files changed, 171 insertions(+), 30 deletions(-) create mode 100644 packages/core/src/with-timeout.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 67879f93..5ec95386 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,6 +2,7 @@ // adapters. See ARCHITECTURE.md §2 and CLAUDE.md §2.2. export * from './action-mapping.js' +export * from './with-timeout.js' export * from './assert-patcher.js' export * from './trace-exporter.js' export * from './trace-har.js' diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts index cfcaef9c..375c5807 100644 --- a/packages/core/src/trace-exporter.ts +++ b/packages/core/src/trace-exporter.ts @@ -74,6 +74,29 @@ function shortId(sessionId?: string): string { return (sessionId ?? Math.random().toString(36).slice(2, 10)).slice(0, 8) } +function resolveContextNaming(caps: Record | undefined): { + browserName: string + title: string +} { + const platformName = + typeof caps?.platformName === 'string' + ? caps.platformName.toLowerCase() + : undefined + const deviceName = + typeof caps?.['appium:deviceName'] === 'string' + ? (caps['appium:deviceName'] as string) + : undefined + if (platformName === 'android' || platformName === 'ios') { + return { + browserName: 'chromium', + title: deviceName ? `${platformName} — ${deviceName}` : platformName + } + } + const browserName = + typeof caps?.browserName === 'string' ? caps.browserName : 'chromium' + return { browserName, title: browserName } +} + function buildContextOptions( trace: TraceLog, contextId: string, @@ -82,7 +105,7 @@ function buildContextOptions( const caps = trace.metadata.capabilities as | Record | undefined - const browserName = (caps?.browserName as string) ?? 'chromium' + const { browserName, title } = resolveContextNaming(caps) const viewport = trace.metadata.viewport ?? { width: 1280, height: 720 } return { version: TRACE_VERSION, @@ -95,7 +118,7 @@ function buildContextOptions( wallTime, monotonicTime: 0, sdkLanguage: 'javascript', - title: browserName, + title, contextId, options: { viewport: { width: viewport.width, height: viewport.height } @@ -217,6 +240,42 @@ function buildScreencastFrames( * network entries become HAR resource-snapshots; per-action screenshots, * element JSON, and snapshot text are written under `resources/`. */ +/** Chronological sort key — see `compareEvents` for the tie-breaker rationale. */ +function eventTime(e: TraceEvent): number { + switch (e.type) { + case 'context-options': + return -Infinity + case 'before': + return e.startTime + case 'after': + return e.endTime + case 'screencast-frame': + return e.timestamp + } +} + +/** At the same timestamp T: an action's `after` ends first, then the + * snapshot captured at the action boundary, then the next action's `before`. + * Matches the viewer's expectation that the screencast frame shows the + * state between the previous action's completion and the next one's start. */ +function eventOrder(e: TraceEvent): number { + switch (e.type) { + case 'context-options': + return 0 + case 'after': + return 1 + case 'screencast-frame': + return 2 + case 'before': + return 3 + } +} + +function compareEvents(a: TraceEvent, b: TraceEvent): number { + const dt = eventTime(a) - eventTime(b) + return dt !== 0 ? dt : eventOrder(a) - eventOrder(b) +} + export async function exportTraceZip( trace: TraceLog, opts: { sessionId?: string; wallTimeOverride?: number } = {} @@ -234,7 +293,7 @@ export async function exportTraceZip( buildContextOptions(trace, contextId, wallTime), ...buildScreencastFrames(snapshots, pageId, wallTime, viewport), ...buildActionEvents(trace.commands, pageId, wallTime) - ] + ].sort(compareEvents) const traceNdjson = events.map((e) => JSON.stringify(e)).join('\n') const networkNdjson = buildNetworkNdjson(trace.networkRequests) const resources = buildSnapshotResources(snapshots, pageId) diff --git a/packages/core/src/with-timeout.ts b/packages/core/src/with-timeout.ts new file mode 100644 index 00000000..bdd37469 --- /dev/null +++ b/packages/core/src/with-timeout.ts @@ -0,0 +1,27 @@ +// Resolves `promise` if it settles before `ms`; otherwise resolves to the +// supplied fallback value. Used by per-action snapshot capture to guard +// against hung in-page scripts (heavy pages, infinite-render loops) without +// stalling the user's test. + +export function withTimeout( + promise: Promise, + ms: number, + fallback: T +): Promise { + let timer: ReturnType | undefined + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(fallback), ms) + }) + return Promise.race([ + promise.then((v) => { + if (timer) { + clearTimeout(timer) + } + return v + }), + timeout + ]) +} + +/** Default ceiling for a single in-page snapshot probe. */ +export const SNAPSHOT_PROBE_TIMEOUT_MS = 2500 diff --git a/packages/nightwatch-devtools/src/action-snapshot.ts b/packages/nightwatch-devtools/src/action-snapshot.ts index 171cb9d0..4ca5eee2 100644 --- a/packages/nightwatch-devtools/src/action-snapshot.ts +++ b/packages/nightwatch-devtools/src/action-snapshot.ts @@ -8,6 +8,7 @@ import { getInteractableBrowserElements, serializeWebSnapshot } from '@wdio/elements' +import { SNAPSHOT_PROBE_TIMEOUT_MS, withTimeout } from '@wdio/devtools-core' import type { ActionSnapshot } from '@wdio/devtools-shared' import type { NightwatchBrowser } from './types.js' @@ -44,12 +45,20 @@ export async function captureActionSnapshot( takeScreenshot?.().catch(() => null) ?? Promise.resolve(null), b.getCurrentUrl?.().catch(() => undefined) ?? Promise.resolve(undefined), b.getTitle?.().catch(() => undefined) ?? Promise.resolve(undefined), - getBrowserAccessibilityTree(browserLike, { inViewportOnly: true }).catch( - () => [] + withTimeout( + getBrowserAccessibilityTree(browserLike, { + inViewportOnly: true + }).catch(() => []), + SNAPSHOT_PROBE_TIMEOUT_MS, + [] ), - getInteractableBrowserElements(browserLike, { - inViewportOnly: true - }).catch(() => []) + withTimeout( + getInteractableBrowserElements(browserLike, { + inViewportOnly: true + }).catch(() => []), + SNAPSHOT_PROBE_TIMEOUT_MS, + [] + ) ]) const snapshotText = serializeWebSnapshot(tree, { url, title }) return { diff --git a/packages/selenium-devtools/src/action-snapshot.ts b/packages/selenium-devtools/src/action-snapshot.ts index 84a081a4..1bfa83b9 100644 --- a/packages/selenium-devtools/src/action-snapshot.ts +++ b/packages/selenium-devtools/src/action-snapshot.ts @@ -9,6 +9,7 @@ import { getInteractableBrowserElements, serializeWebSnapshot } from '@wdio/elements' +import { SNAPSHOT_PROBE_TIMEOUT_MS, withTimeout } from '@wdio/devtools-core' import type { ActionSnapshot } from '@wdio/devtools-shared' import type { SeleniumDriverLike } from './types.js' @@ -39,12 +40,20 @@ export async function captureActionSnapshot( d.takeScreenshot?.().catch(() => undefined) ?? Promise.resolve(undefined), d.getCurrentUrl?.().catch(() => undefined) ?? Promise.resolve(undefined), d.getTitle?.().catch(() => undefined) ?? Promise.resolve(undefined), - getBrowserAccessibilityTree(browserLike, { inViewportOnly: true }).catch( - () => [] + withTimeout( + getBrowserAccessibilityTree(browserLike, { + inViewportOnly: true + }).catch(() => []), + SNAPSHOT_PROBE_TIMEOUT_MS, + [] ), - getInteractableBrowserElements(browserLike, { - inViewportOnly: true - }).catch(() => []) + withTimeout( + getInteractableBrowserElements(browserLike, { + inViewportOnly: true + }).catch(() => []), + SNAPSHOT_PROBE_TIMEOUT_MS, + [] + ) ]) const snapshotText = serializeWebSnapshot(tree, { url, title }) return { diff --git a/packages/service/src/action-snapshot.ts b/packages/service/src/action-snapshot.ts index 8763f393..eef71da5 100644 --- a/packages/service/src/action-snapshot.ts +++ b/packages/service/src/action-snapshot.ts @@ -8,8 +8,58 @@ import { serializeMobileSnapshot, serializeWebSnapshot } from '@wdio/elements' +import { SNAPSHOT_PROBE_TIMEOUT_MS, withTimeout } from '@wdio/devtools-core' import type { ActionSnapshot } from '@wdio/devtools-shared' +type ElementsResult = Awaited> +const EMPTY_ELEMENTS: ElementsResult = { + total: 0, + showing: 0, + hasMore: false, + elements: [] +} + +function probeElements(browser: WebdriverIO.Browser): Promise { + return withTimeout( + getElements(browser, { inViewportOnly: true }), + SNAPSHOT_PROBE_TIMEOUT_MS, + EMPTY_ELEMENTS + ) +} + +async function captureMobile( + browser: WebdriverIO.Browser +): Promise<{ elements: unknown[]; snapshotText?: string }> { + const result = await probeElements(browser) + if (!result.tree) { + return { elements: result.elements } + } + const platform = browser.isAndroid ? 'android' : 'ios' + return { + elements: result.elements, + snapshotText: serializeMobileSnapshot(result.tree, { platform }) + } +} + +async function captureWeb( + browser: WebdriverIO.Browser, + url: string | undefined, + title: string | undefined +): Promise<{ elements: unknown[]; snapshotText?: string }> { + const [tree, flat] = await Promise.all([ + withTimeout( + getBrowserAccessibilityTree(browser, { inViewportOnly: true }), + SNAPSHOT_PROBE_TIMEOUT_MS, + [] + ), + probeElements(browser) + ]) + return { + elements: flat.elements, + snapshotText: serializeWebSnapshot(tree, { url, title }) + } +} + export async function captureActionSnapshot( browser: WebdriverIO.Browser, command: string @@ -22,23 +72,9 @@ export async function captureActionSnapshot( browser.getUrl().catch(() => undefined), browser.getTitle().catch(() => undefined) ]) - let elements: unknown[] = [] - let snapshotText: string | undefined - if (isMobile) { - const result = await getElements(browser, { inViewportOnly: true }) - elements = result.elements - if (result.tree) { - const platform = browser.isAndroid ? 'android' : 'ios' - snapshotText = serializeMobileSnapshot(result.tree, { platform }) - } - } else { - const [tree, flatResult] = await Promise.all([ - getBrowserAccessibilityTree(browser, { inViewportOnly: true }), - getElements(browser, { inViewportOnly: true }) - ]) - elements = flatResult.elements - snapshotText = serializeWebSnapshot(tree, { url, title }) - } + const { elements, snapshotText } = isMobile + ? await captureMobile(browser) + : await captureWeb(browser, url, title) return { timestamp, command, From 7cda9ad8d45fe2d47295f88ff3678347777074af Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Tue, 9 Jun 2026 15:18:47 +0530 Subject: [PATCH 10/13] =?UTF-8?q?core:=20@wdio/elements=20=E2=86=92=20webd?= =?UTF-8?q?riverio=20peer=20dep=20leak=20through=20nightwatch/selenium=20r?= =?UTF-8?q?esolved=20by=20core=20extraction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/package.json | 18 + packages/core/src/action-snapshot.ts | 66 ++ packages/core/src/element-scripts.ts | 377 +++++++++ packages/core/src/element-snapshot.ts | 758 +++++++++++++++++ packages/core/src/element-types.ts | 45 + packages/core/src/index.ts | 4 + .../src/locators/constants.ts | 0 .../src/locators/element-filter.ts | 0 packages/core/src/locators/index.ts | 282 +++++++ .../src/locators/locator-generation.ts | 196 ++--- .../{elements => core}/src/locators/types.ts | 0 .../src/locators/xml-parsing.ts | 4 - packages/core/src/trace-exporter.ts | 5 +- packages/elements/package.json | 6 +- packages/elements/src/accessibility-tree.ts | 480 +---------- packages/elements/src/aria-roles.ts | 68 -- packages/elements/src/browser-elements.ts | 309 +------ packages/elements/src/get-elements.ts | 2 +- packages/elements/src/index.ts | 6 +- packages/elements/src/locators/index.ts | 254 +----- packages/elements/src/snapshot.ts | 781 +----------------- packages/nightwatch-devtools/package.json | 1 - .../src/action-snapshot.ts | 79 +- packages/selenium-devtools/package.json | 1 - .../selenium-devtools/src/action-snapshot.ts | 78 +- packages/service/src/action-snapshot.ts | 104 +-- pnpm-lock.yaml | 21 +- 27 files changed, 1766 insertions(+), 2179 deletions(-) create mode 100644 packages/core/src/action-snapshot.ts create mode 100644 packages/core/src/element-scripts.ts create mode 100644 packages/core/src/element-snapshot.ts create mode 100644 packages/core/src/element-types.ts rename packages/{elements => core}/src/locators/constants.ts (100%) rename packages/{elements => core}/src/locators/element-filter.ts (100%) create mode 100644 packages/core/src/locators/index.ts rename packages/{elements => core}/src/locators/locator-generation.ts (81%) rename packages/{elements => core}/src/locators/types.ts (100%) rename packages/{elements => core}/src/locators/xml-parsing.ts (96%) delete mode 100644 packages/elements/src/aria-roles.ts diff --git a/packages/core/package.json b/packages/core/package.json index b7b6ad91..2e2325ae 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -18,6 +18,22 @@ "./*": { "types": "./src/*.ts", "default": "./src/*.ts" + }, + "./locators": { + "types": "./src/locators/index.ts", + "default": "./src/locators/index.ts" + }, + "./element-snapshot": { + "types": "./src/element-snapshot.ts", + "default": "./src/element-snapshot.ts" + }, + "./element-scripts": { + "types": "./src/element-scripts.ts", + "default": "./src/element-scripts.ts" + }, + "./element-types": { + "types": "./src/element-types.ts", + "default": "./src/element-types.ts" } }, "types": "./src/index.ts", @@ -30,8 +46,10 @@ "@types/yazl": "^2.4.6", "@wdio/devtools-script": "workspace:*", "@wdio/devtools-shared": "workspace:^", + "@xmldom/xmldom": "^0.9.8", "stacktrace-parser": "^0.1.11", "ws": "^8.21.0", + "xpath": "^0.0.34", "yazl": "^2.5.1" } } diff --git a/packages/core/src/action-snapshot.ts b/packages/core/src/action-snapshot.ts new file mode 100644 index 00000000..43affebd --- /dev/null +++ b/packages/core/src/action-snapshot.ts @@ -0,0 +1,66 @@ +// Adapter-agnostic per-action snapshot capture. Each adapter wires its own +// `runScript`, `takeScreenshot`, etc. shim so the actual capture pipeline +// (timeouts, fallbacks, snapshot serialization) lives in one place. + +import { accessibilityTreeScript, elementsScript } from './element-scripts.js' +import { serializeWebSnapshot } from './element-snapshot.js' +import type { AccessibilityNode, BrowserElementInfo } from './element-types.js' +import { SNAPSHOT_PROBE_TIMEOUT_MS, withTimeout } from './with-timeout.js' +import type { ActionSnapshot } from '@wdio/devtools-shared' + +export type ScriptRunner = (scriptSrc: string) => Promise + +export interface CaptureActionSnapshotInput { + command: string + runScript: ScriptRunner + takeScreenshot?: () => Promise + getUrl?: () => Promise + getTitle?: () => Promise +} + +async function runWith( + runScript: ScriptRunner, + scriptSrc: string, + fallback: T +): Promise { + return withTimeout( + runScript(scriptSrc).then((r) => r as T), + SNAPSHOT_PROBE_TIMEOUT_MS, + fallback + ).catch(() => fallback) +} + +export async function captureActionSnapshot( + input: CaptureActionSnapshotInput +): Promise { + try { + const timestamp = Date.now() + const [shot, url, title, tree, elements] = await Promise.all([ + input.takeScreenshot?.().catch(() => null) ?? Promise.resolve(null), + input.getUrl?.().catch(() => undefined) ?? Promise.resolve(undefined), + input.getTitle?.().catch(() => undefined) ?? Promise.resolve(undefined), + runWith( + input.runScript, + accessibilityTreeScript(true), + [] + ), + runWith( + input.runScript, + elementsScript(false, true), + [] + ) + ]) + const snapshotText = serializeWebSnapshot(tree, { url, title }) + return { + timestamp, + command: input.command, + url, + title, + screenshot: shot ?? undefined, + elements, + snapshotText + } + } catch { + return null + } +} diff --git a/packages/core/src/element-scripts.ts b/packages/core/src/element-scripts.ts new file mode 100644 index 00000000..9b9b33d9 --- /dev/null +++ b/packages/core/src/element-scripts.ts @@ -0,0 +1,377 @@ +/** + * Browser-injectable script strings for element extraction. + * + * Each function returns a self-contained JavaScript string designed to run + * inside a browser page via `browser.execute(script)`. The scripts have no + * external dependencies and must be ES5-compatible. + * + * WDIO-dependent wrappers that call `browser.execute(script)` live in + * `@wdio/elements` — these are just the script bodies. + */ + +/** + * Accessibility tree walk — returns a flat array of AccessibilityNode. + * + * Walks the DOM from `document.body`, assigning semantic roles (button, link, + * textbox, heading, img, statictext, …) based on tag name, ARIA attributes, + * and visibility. Each node carries a unique CSS selector. + */ +export function accessibilityTreeScript(inViewportOnly: boolean): string { + return `(function () { + var INPUT_TYPE_ROLES = { + text: 'textbox', search: 'searchbox', email: 'textbox', url: 'textbox', + tel: 'textbox', password: 'textbox', number: 'spinbutton', + checkbox: 'checkbox', radio: 'radio', range: 'slider', + submit: 'button', reset: 'button', image: 'button', file: 'button', color: 'button' + } + + var CONTAINER_ROLES = new Set([ + 'navigation', 'banner', 'contentinfo', 'complementary', 'main', + 'form', 'region', 'group', 'list', 'listitem', 'table', 'row', 'rowgroup', 'generic' + ]) + + function getRole(el) { + var explicit = el.getAttribute('role') + if (explicit) { return explicit.split(' ')[0] } + var tag = el.tagName.toLowerCase() + switch (tag) { + case 'button': return 'button' + case 'a': return el.hasAttribute('href') ? 'link' : null + case 'input': { + var type = (el.getAttribute('type') || 'text').toLowerCase() + if (type === 'hidden') { return null } + return INPUT_TYPE_ROLES[type] || 'textbox' + } + case 'select': return 'combobox' + case 'textarea': return 'textbox' + case 'h1': case 'h2': case 'h3': case 'h4': case 'h5': case 'h6': return 'heading' + case 'img': return 'img' + case 'nav': return 'navigation' + case 'main': return 'main' + case 'header': return !el.closest('article,aside,main,nav,section') ? 'banner' : null + case 'footer': return !el.closest('article,aside,main,nav,section') ? 'contentinfo' : null + case 'aside': return 'complementary' + case 'dialog': return 'dialog' + case 'form': return 'form' + case 'section': return el.hasAttribute('aria-label') || el.hasAttribute('aria-labelledby') ? 'region' : null + case 'summary': return 'button' + case 'details': return 'group' + case 'progress': return 'progressbar' + case 'meter': return 'meter' + case 'ul': case 'ol': return 'list' + case 'li': return 'listitem' + case 'table': return 'table' + } + if (el.contentEditable === 'true') { return 'textbox' } + if (el.hasAttribute('tabindex') && parseInt(el.getAttribute('tabindex') || '-1', 10) >= 0) { return 'generic' } + if (getDirectText(el)) { return 'statictext' } + return null + } + + function getAccessibleName(el, role) { + var ariaLabel = el.getAttribute('aria-label') + if (ariaLabel) { return ariaLabel.trim() } + var labelledBy = el.getAttribute('aria-labelledby') + if (labelledBy) { + var texts = labelledBy.split(/\\s+/).map(function(id) { return (document.getElementById(id)?.textContent || '').trim() }).filter(Boolean) + if (texts.length > 0) { return texts.join(' ').slice(0, 200) } + } + var tag = el.tagName.toLowerCase() + if (tag === 'img' || (tag === 'input' && el.getAttribute('type') === 'image')) { + var alt = el.getAttribute('alt') + if (alt !== null) { return alt.trim() } + } + if (['input', 'select', 'textarea'].indexOf(tag) !== -1) { + var id = el.getAttribute('id') + if (id) { + var label = document.querySelector('label[for="' + CSS.escape(id) + '"]') + if (label) { return (label.textContent || '').trim() } + } + var parentLabel = el.closest('label') + if (parentLabel) { + var clone = parentLabel.cloneNode(true) + clone.querySelectorAll('input,select,textarea').forEach(function(n) { n.remove() }) + var lt = (clone.textContent || '').trim() + if (lt) { return lt } + } + } + var ph = el.getAttribute('placeholder') + if (ph) { return ph.trim() } + var title = el.getAttribute('title') + if (title) { return title.trim() } + var childImg = el.querySelector('img') + if (childImg) { + var imgAlt = childImg.getAttribute('alt') + if (imgAlt) { return imgAlt.trim() } + } + if (role && CONTAINER_ROLES.has(role)) { return '' } + return ((el.textContent || '').trim().replace(/\\s+/g, ' ') || '').slice(0, 200) + } + + function getSelector(element) { + var tag = element.tagName.toLowerCase() + var text = (element.textContent || '').trim().replace(/\\s+/g, ' ') + if (text && text.length > 0 && text.length <= 120) { + var sameTagElements = document.querySelectorAll(tag) + var matchCount = 0 + sameTagElements.forEach(function(el) { if (el.textContent.includes(text)) { matchCount++ } }) + if (matchCount === 1) { return tag + '*=' + text } + } + var ariaLabel = element.getAttribute('aria-label') + if (ariaLabel && ariaLabel.length <= 200) { + var sel = '[aria-label="' + CSS.escape(ariaLabel) + '"]' + if (document.querySelectorAll(sel).length === 1) { return sel } + } + var testId = element.getAttribute('data-testid') + if (testId) { + var testSel = '[data-testid="' + CSS.escape(testId) + '"]' + if (document.querySelectorAll(testSel).length === 1) { return testSel } + } + if (element.id) { return '#' + CSS.escape(element.id) } + var nameAttr = element.getAttribute('name') + if (nameAttr) { + var nameSel = tag + '[name="' + CSS.escape(nameAttr) + '"]' + if (document.querySelectorAll(nameSel).length === 1) { return nameSel } + } + if (element.className && typeof element.className === 'string') { + var classes = element.className.trim().split(/\\s+/).filter(Boolean) + for (var i = 0; i < classes.length; i++) { + var clsSel = tag + '.' + CSS.escape(classes[i]) + if (document.querySelectorAll(clsSel).length === 1) { return clsSel } + } + if (classes.length >= 2) { + var twoClsSel = tag + classes.slice(0, 2).map(function(c) { return '.' + CSS.escape(c) }).join('') + if (document.querySelectorAll(twoClsSel).length === 1) { return twoClsSel } + } + } + var current = element + var path = [] + while (current && current !== document.documentElement) { + var seg = current.tagName.toLowerCase() + if (current.id) { path.unshift('#' + CSS.escape(current.id)); break } + var parent = current.parentElement + if (parent) { + var siblings = Array.from(parent.children).filter(function(c) { return c.tagName === current.tagName }) + if (siblings.length > 1) { seg += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')' } + } + path.unshift(seg) + current = current.parentElement + if (path.length >= 4) { break } + } + return path.join(' > ') + } + + function getDirectText(el) { + var text = '' + for (var i = 0; i < el.childNodes.length; i++) { + if (el.childNodes[i].nodeType === 3) { text += el.childNodes[i].textContent } + } + return text.trim().replace(/\\s+/g, ' ') + } + + function isVisible(el) { + if (typeof el.checkVisibility === 'function') { + return el.checkVisibility({ opacityProperty: true, visibilityProperty: true, contentVisibilityAuto: true }) + } + var style = window.getComputedStyle(el) + return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0' && el.offsetWidth > 0 && el.offsetHeight > 0 + } + + function isInViewport(el) { + var rect = el.getBoundingClientRect() + return rect.top >= 0 && rect.left >= 0 && rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) && rect.right <= (window.innerWidth || document.documentElement.clientWidth) + } + + function getLevel(el) { + var m = el.tagName.toLowerCase().match(/^h([1-6])$/) + if (m) { return parseInt(m[1], 10) } + var ariaLevel = el.getAttribute('aria-level') + if (ariaLevel) { return parseInt(ariaLevel, 10) } + return undefined + } + + function getState(el) { + var inputEl = el + var isCheckable = ['input', 'menuitemcheckbox', 'menuitemradio'].indexOf(el.tagName.toLowerCase()) !== -1 || ['checkbox', 'radio', 'switch'].indexOf(el.getAttribute('role') || '') !== -1 + return { + disabled: el.getAttribute('aria-disabled') === 'true' || inputEl.disabled ? 'true' : '', + checked: isCheckable && inputEl.checked ? 'true' : el.getAttribute('aria-checked') || '', + expanded: el.getAttribute('aria-expanded') || '', + selected: el.getAttribute('aria-selected') || '', + pressed: el.getAttribute('aria-pressed') || '', + required: inputEl.required || el.getAttribute('aria-required') === 'true' ? 'true' : '', + readonly: inputEl.readOnly || el.getAttribute('aria-readonly') === 'true' ? 'true' : '' + } + } + + var result = [] + + function walk(el, depth) { + if (depth > 200) { return } + if (!isVisible(el)) { return } + var role = getRole(el) + var inViewport = isInViewport(el) + if (!role) { + for (var i = 0; i < el.children.length; i++) { walk(el.children[i], depth + 1) } + return + } + if (${inViewportOnly} && !inViewport) { + for (var i = 0; i < el.children.length; i++) { walk(el.children[i], depth + 1) } + return + } + var name = getAccessibleName(el, role) + var selector = getSelector(el) + var node = { role: role, name: name, selector: selector, depth: depth, level: getLevel(el) ?? '', isInViewport: inViewport } + var state = getState(el) + for (var k in state) { node[k] = state[k] } + result.push(node) + for (var i = 0; i < el.children.length; i++) { walk(el.children[i], depth + 1) } + } + + for (var i = 0; i < document.body.children.length; i++) { walk(document.body.children[i], 0) } + return result + })()` +} + +/** + * Interactable element query — returns a flat array of BrowserElementInfo. + * + * Uses `querySelectorAll` with a broad interactable-selector list, then + * filters by visibility and (optionally) viewport containment. Each element + * gets a computed accessible name and a unique CSS selector. + */ +export function elementsScript( + includeBounds: boolean, + inViewportOnly: boolean +): string { + return `(function () { + var interactableSelectors = [ + 'a[href]', 'button', 'input:not([type="hidden"])', 'select', 'textarea', + '[role="button"]', '[role="link"]', '[role="checkbox"]', '[role="radio"]', + '[role="tab"]', '[role="menuitem"]', '[role="combobox"]', '[role="option"]', + '[role="switch"]', '[role="slider"]', '[role="textbox"]', '[role="searchbox"]', + '[role="spinbutton"]', '[contenteditable="true"]', '[tabindex]:not([tabindex="-1"])' + ].join(',') + + function isVisible(element) { + if (typeof element.checkVisibility === 'function') { + return element.checkVisibility({ opacityProperty: true, visibilityProperty: true, contentVisibilityAuto: true }) + } + var style = window.getComputedStyle(element) + return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0' && element.offsetWidth > 0 && element.offsetHeight > 0 + } + + function getAccessibleName(el) { + var ariaLabel = el.getAttribute('aria-label') + if (ariaLabel) { return ariaLabel.trim() } + var labelledBy = el.getAttribute('aria-labelledby') + if (labelledBy) { + var texts = labelledBy.split(/\\s+/).map(function(id) { return (document.getElementById(id)?.textContent || '').trim() }).filter(Boolean) + if (texts.length > 0) { return texts.join(' ').slice(0, 200) } + } + var tag = el.tagName.toLowerCase() + if (tag === 'img' || (tag === 'input' && el.getAttribute('type') === 'image')) { + var alt = el.getAttribute('alt') + if (alt !== null) { return alt.trim() } + } + if (['input', 'select', 'textarea'].indexOf(tag) !== -1) { + var id = el.getAttribute('id') + if (id) { + var label = document.querySelector('label[for="' + CSS.escape(id) + '"]') + if (label) { return (label.textContent || '').trim() } + } + var parentLabel = el.closest('label') + if (parentLabel) { + var clone = parentLabel.cloneNode(true) + clone.querySelectorAll('input,select,textarea').forEach(function(n) { n.remove() }) + var lt = (clone.textContent || '').trim() + if (lt) { return lt } + } + } + var ph = el.getAttribute('placeholder') + if (ph) { return ph.trim() } + var title = el.getAttribute('title') + if (title) { return title.trim() } + return ((el.textContent || '').trim().replace(/\\s+/g, ' ') || '').slice(0, 200) + } + + function getSelector(element) { + var tag = element.tagName.toLowerCase() + var text = (element.textContent || '').trim().replace(/\\s+/g, ' ') + if (text && text.length > 0 && text.length <= 120) { + var sameTagElements = document.querySelectorAll(tag) + var matchCount = 0 + sameTagElements.forEach(function(el) { if (el.textContent.includes(text)) { matchCount++ } }) + if (matchCount === 1) { return tag + '*=' + text } + } + var ariaLabel = element.getAttribute('aria-label') + if (ariaLabel && ariaLabel.length <= 200) { + var sel = '[aria-label="' + CSS.escape(ariaLabel) + '"]' + if (document.querySelectorAll(sel).length === 1) { return sel } + } + var testId = element.getAttribute('data-testid') + if (testId) { + var testSel = '[data-testid="' + CSS.escape(testId) + '"]' + if (document.querySelectorAll(testSel).length === 1) { return testSel } + } + if (element.id) { return '#' + CSS.escape(element.id) } + var nameAttr = element.getAttribute('name') + if (nameAttr) { + var nameSel = tag + '[name="' + CSS.escape(nameAttr) + '"]' + if (document.querySelectorAll(nameSel).length === 1) { return nameSel } + } + if (element.className && typeof element.className === 'string') { + var classes = element.className.trim().split(/\\s+/).filter(Boolean) + for (var i = 0; i < classes.length; i++) { + var clsSel = tag + '.' + CSS.escape(classes[i]) + if (document.querySelectorAll(clsSel).length === 1) { return clsSel } + } + if (classes.length >= 2) { + var twoClsSel = tag + classes.slice(0, 2).map(function(c) { return '.' + CSS.escape(c) }).join('') + if (document.querySelectorAll(twoClsSel).length === 1) { return twoClsSel } + } + } + var current = element + var path = [] + while (current && current !== document.documentElement) { + var seg = current.tagName.toLowerCase() + if (current.id) { path.unshift('#' + CSS.escape(current.id)); break } + var parent = current.parentElement + if (parent) { + var siblings = Array.from(parent.children).filter(function(c) { return c.tagName === current.tagName }) + if (siblings.length > 1) { seg += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')' } + } + path.unshift(seg) + current = current.parentElement + if (path.length >= 4) { break } + } + return path.join(' > ') + } + + var elements = [] + var seen = new Set() + + document.querySelectorAll(interactableSelectors).forEach(function(el) { + if (seen.has(el)) { return } + seen.add(el) + var htmlEl = el + if (!isVisible(htmlEl)) { return } + var inputEl = htmlEl + var rect = htmlEl.getBoundingClientRect() + var isInVp = rect.top >= 0 && rect.left >= 0 && rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) && rect.right <= (window.innerWidth || document.documentElement.clientWidth) + if (${inViewportOnly} && !isInVp) { return } + var entry = { + tagName: htmlEl.tagName.toLowerCase(), + name: getAccessibleName(htmlEl), + type: htmlEl.getAttribute('type') || '', + value: inputEl.value || '', + href: htmlEl.getAttribute('href') || '', + selector: getSelector(htmlEl), + isInViewport: isInVp + } + ${includeBounds ? 'entry.boundingBox = { x: rect.x + window.scrollX, y: rect.y + window.scrollY, width: rect.width, height: rect.height }' : ''} + elements.push(entry) + }) + return elements + })()` +} diff --git a/packages/core/src/element-snapshot.ts b/packages/core/src/element-snapshot.ts new file mode 100644 index 00000000..c4e1b2bb --- /dev/null +++ b/packages/core/src/element-snapshot.ts @@ -0,0 +1,758 @@ +/** + * AI-readable snapshot serializers + * + * Converts accessibility trees and mobile element trees into depth-indented + * text files that LLMs can consume without any parsing. + */ + +import type { AccessibilityNode } from './element-types.js' +import type { JSONElement } from './locators/types.js' +import { parseAndroidBounds, parseIOSBounds } from './locators/xml-parsing.js' +import { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS +} from './locators/constants.js' +import { getSuggestedLocators } from './locators/locator-generation.js' + +/** + * Roles that can be interacted with — rendered with `→ selector`. + * Structural roles (heading, img, form, nav, …) are intentionally excluded. + */ +const INTERACTIVE_ROLES = new Set([ + 'button', + 'link', + 'textbox', + 'checkbox', + 'radio', + 'combobox', + 'slider', + 'searchbox', + 'spinbutton', + 'switch', + 'tab', + 'menuitem', + 'option' +]) + +/** + * Walk backwards from `index` to find the nearest ancestor or preceding + * structural sibling with a non-empty name. Same-depth nodes are only + * used when they are structural (img, heading, statictext, …) — never + * another interactive element. + */ +function inferPurpose( + nodes: AccessibilityNode[], + index: number +): string | undefined { + const myDepth = nodes[index].depth + for (let i = index - 1; i >= 0; i--) { + if (nodes[i].depth <= myDepth && nodes[i].name) { + // Same-depth sibling: only structural elements count + if (nodes[i].depth === myDepth && INTERACTIVE_ROLES.has(nodes[i].role)) { + continue + } + return nodes[i].name + } + } + return undefined +} + +export interface WebSnapshotOptions { + /** Only include nodes whose bounding rect intersects the viewport (default true). */ + inViewportOnly?: boolean +} + +/** + * Serialize a web accessibility tree into a depth-indented text snapshot. + * + * @param nodes Flat ordered node list from getBrowserAccessibilityTree() + * @param context Optional page context for the header line + * @param options {@link WebSnapshotOptions} + */ +export function serializeWebSnapshot( + nodes: AccessibilityNode[], + context?: { url?: string; title?: string }, + options: WebSnapshotOptions = {} +): string { + const { inViewportOnly = true } = options + + let header = '[Page' + if (context?.title) { + header += `: ${context.title}` + } + if (context?.url) { + header += ` — ${context.url}` + } + header += ']' + + const lines: string[] = [header] + + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + + // When viewport filtering is on, skip nodes that are known to be off-screen. + // Nodes from a tree captured with inViewportOnly=false will have + // isInViewport populated; nodes from a pre-filtered tree all have + // isInViewport=true (or undefined for pre-existing data). + if (inViewportOnly && node.isInViewport === false) { + continue + } + + const indent = ' '.repeat(node.depth + 1) // +1 indents everything under the header + const isInteractive = INTERACTIVE_ROLES.has(node.role) + + // Skip statictext that merely echoes the parent link/button name. + // Example: link "Highlights" → a*=Highlights doesn't need + // statictext "Highlights" as a child because it adds no information. + if (node.role === 'statictext' && node.name) { + let echoedByParent = false + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + const parentRole = nodes[j].role + const parentName = nodes[j].name + if ( + INTERACTIVE_ROLES.has(parentRole) && + parentName && + parentName.includes(node.name) + ) { + echoedByParent = true + } + break // only check the immediate structural parent + } + } + if (echoedByParent) { + continue + } + } + + // Heading gets level suffix: heading[2] + const roleLabel = + node.role === 'heading' && node.level + ? `heading[${node.level}]` + : node.role + + if (isInteractive) { + // No selector → agent can't act on this node; skip entirely + if (!node.selector) { + continue + } + const purpose = inferPurpose(nodes, i) + if (node.name) { + // Show parent context when available — disambiguates + // duplicate selectors like six "Add to Wishlist" buttons. + lines.push( + purpose + ? `${indent}${roleLabel} "${node.name}" ∈ "${purpose}" → ${node.selector}` + : `${indent}${roleLabel} "${node.name}" → ${node.selector}` + ) + } else if (purpose) { + lines.push(`${indent}${roleLabel} ∈ "${purpose}" → ${node.selector}`) + } else { + lines.push(`${indent}${roleLabel} → ${node.selector}`) + } + } else { + // Container / structural: show role + name when present, no selector + lines.push( + node.name + ? `${indent}${roleLabel} "${node.name}"` + : `${indent}${roleLabel}` + ) + } + } + + return lines.join('\n') +} + +// --------------------------------------------------------------------------- +// Mobile snapshot helpers +// --------------------------------------------------------------------------- + +/** Shorten fully-qualified Android/iOS class names to the last segment. */ +function simplifyTag(tagName: string): string { + const dot = tagName.lastIndexOf('.') + if (dot !== -1) { + return tagName.slice(dot + 1) + } + return tagName.replace(/^XCUIElementType/, '') +} + +// --------------------------------------------------------------------------- +// Mobile role classification — maps raw Android/iOS class names to semantic +// roles so the snapshot reads like the web version (button, textbox, img, …). +// --------------------------------------------------------------------------- + +const ANDROID_ROLE_MAP: Record = { + 'android.widget.Button': 'button', + 'android.widget.ImageButton': 'button', + 'android.widget.ToggleButton': 'button', + 'android.widget.FloatingActionButton': 'button', + 'com.google.android.material.button.MaterialButton': 'button', + 'com.google.android.material.floatingactionbutton.FloatingActionButton': + 'button', + 'android.widget.EditText': 'textbox', + 'android.widget.AutoCompleteTextView': 'textbox', + 'android.widget.MultiAutoCompleteTextView': 'textbox', + 'android.widget.SearchView': 'searchbox', + 'android.widget.ImageView': 'img', + 'android.widget.QuickContactBadge': 'img', + 'android.widget.CheckBox': 'checkbox', + 'android.widget.RadioButton': 'radio', + 'android.widget.Switch': 'switch', + 'android.widget.Spinner': 'combobox', + 'android.widget.SeekBar': 'slider', + 'android.widget.RatingBar': 'slider', + 'android.widget.ProgressBar': 'progressbar', + 'android.widget.TextView': 'statictext', + 'android.widget.CheckedTextView': 'statictext', + 'android.widget.RecyclerView': 'list', + 'android.widget.ListView': 'list', + 'android.widget.GridView': 'list', + 'android.webkit.WebView': 'webview' +} + +const IOS_ROLE_MAP: Record = { + XCUIElementTypeButton: 'button', + XCUIElementTypeLink: 'link', + XCUIElementTypeTextField: 'textbox', + XCUIElementTypeSecureTextField: 'textbox', + XCUIElementTypeTextView: 'textbox', + XCUIElementTypeSearchField: 'searchbox', + XCUIElementTypeImage: 'img', + XCUIElementTypeIcon: 'img', + XCUIElementTypeSwitch: 'switch', + XCUIElementTypeSlider: 'slider', + XCUIElementTypeStepper: 'slider', + XCUIElementTypeCheckBox: 'checkbox', + XCUIElementTypeRadioButton: 'radio', + XCUIElementTypePicker: 'combobox', + XCUIElementTypePickerWheel: 'combobox', + XCUIElementTypeDatePicker: 'combobox', + XCUIElementTypeSegmentedControl: 'combobox', + XCUIElementTypeStaticText: 'statictext', + XCUIElementTypeCell: 'listitem', + XCUIElementTypeTable: 'list', + XCUIElementTypeCollectionView: 'list' +} + +function classifyMobileRole( + tagName: string, + platform: 'android' | 'ios' +): string { + if (platform === 'android') { + return ANDROID_ROLE_MAP[tagName] || simplifyTag(tagName) + } + return IOS_ROLE_MAP[tagName] || simplifyTag(tagName) +} + +// --------------------------------------------------------------------------- +// Locator generation +// --------------------------------------------------------------------------- + +function getBestAndroidLocator( + attrs: JSONElement['attributes'] +): string | undefined { + // Pre-computed by the full locator pipeline (generateAllElementLocators). + // Takes priority over the simplified fallback logic below. + if (attrs._selector) { + return attrs._selector + } + // ~ prefix = accessibility-id shorthand in WebdriverIO ($('~foo')) + if (attrs['content-desc']) { + return `~${attrs['content-desc']}` + } + if (attrs['resource-id']) { + return `id:${attrs['resource-id']}` + } + if (attrs.text) { + return `~${attrs.text}` + } + // Fallback: class-based locator (only useful with :nth-of-type or index) + if (attrs.class) { + return `class:${simplifyTag(attrs.class)}` + } + return undefined +} + +function getBestIOSLocator( + attrs: JSONElement['attributes'] +): string | undefined { + // Pre-computed by the full locator pipeline. + if (attrs._selector) { + return attrs._selector + } + // ~ prefix = accessibility-id shorthand (maps to `name` on iOS) + if (attrs.name) { + return `~${attrs.name}` + } + if (attrs.label) { + return `~${attrs.label}` + } + if (attrs.value) { + return `~${attrs.value}` + } + // Fallback: class-based locator + if (attrs.type) { + return `class:${simplifyTag(attrs.type)}` + } + return undefined +} + +// --------------------------------------------------------------------------- +// Identity +// --------------------------------------------------------------------------- + +function getMobileNodeIdentity( + attrs: JSONElement['attributes'], + platform: 'android' | 'ios' +): string { + if (platform === 'android') { + const contentDesc = attrs['content-desc'] + if (contentDesc) { + return contentDesc + } + if (attrs.text) { + return attrs.text + } + // Fall back to the last segment of the resource-id (e.g. "search_action_bar") + const rid = attrs['resource-id'] + if (rid) { + const slash = rid.lastIndexOf('/') + return slash !== -1 ? rid.slice(slash + 1) : rid + } + return '' + } + return attrs.name || attrs.label || attrs.value || attrs.text || '' +} + +// --------------------------------------------------------------------------- +// Interactivity +// --------------------------------------------------------------------------- + +const ANDROID_INTERACTABLE_SET = new Set(ANDROID_INTERACTABLE_TAGS) +const IOS_INTERACTABLE_SET = new Set(IOS_INTERACTABLE_TAGS) + +/** An element is *explicitly* interactive when it carries a click/focus/check + * attribute — as opposed to being interactive only because its tag is in the + * interactable-tag list. Explicit parents should carry the → selector, not + * their tag-interactive children. */ +function isExplicitlyInteractive( + attrs: JSONElement['attributes'], + platform: 'android' | 'ios' +): boolean { + if (platform === 'android') { + return ( + attrs.clickable === 'true' || + attrs.focusable === 'true' || + attrs.checkable === 'true' || + attrs['long-clickable'] === 'true' + ) + } + return attrs.accessible === 'true' +} + +function isMobileInteractive( + element: JSONElement, + platform: 'android' | 'ios' +): boolean { + const attrs = element.attributes + if (platform === 'android') { + if (ANDROID_INTERACTABLE_SET.has(element.tagName)) { + return true + } + return ( + attrs.clickable === 'true' || + attrs['long-clickable'] === 'true' || + attrs.focusable === 'true' || + attrs.checkable === 'true' + ) + } + if (IOS_INTERACTABLE_SET.has(element.tagName)) { + return true + } + return attrs.accessible === 'true' +} + +// --------------------------------------------------------------------------- +// Viewport +// --------------------------------------------------------------------------- + +interface WalkMobileOptions { + inViewportOnly: boolean + viewport: { width: number; height: number } + /** Raw page-source XML. When provided, the full locator pipeline is used. */ + sourceXML?: string + /** 'uiautomator2' or 'xcuitest'. Required when sourceXML is set. */ + automationName?: string +} + +function isMobileInViewport( + element: JSONElement, + platform: 'android' | 'ios', + viewport: { width: number; height: number } +): boolean { + const bounds = + platform === 'android' + ? parseAndroidBounds(element.attributes.bounds || '') + : parseIOSBounds(element.attributes) + + if (bounds.width === 0 && bounds.height === 0) { + return true + } + + return ( + bounds.x >= 0 && + bounds.y >= 0 && + bounds.width > 0 && + bounds.height > 0 && + bounds.x + bounds.width <= viewport.width && + bounds.y + bounds.height <= viewport.height + ) +} + +// --------------------------------------------------------------------------- +// Flat-node representation (mirrors AccessibilityNode so both pipelines share +// inferPurpose, dedup, and rendering logic). +// --------------------------------------------------------------------------- + +interface MobileFlatNode { + role: string + name: string + selector: string + depth: number + isInteractive: boolean + /** True when the element has clickable/focusable/checkable — the intended tap target. */ + isExplicitInteractive: boolean + isInViewport: boolean +} + +/** + * First pass: walk the JSONElement tree, apply viewport filtering and + * collect every node into a flat array with semantic roles and selectors. + */ +function collectMobileNodes( + element: JSONElement, + platform: 'android' | 'ios', + depth: number, + nodes: MobileFlatNode[], + walkOpts: WalkMobileOptions +): void { + const attrs = element.attributes + const role = classifyMobileRole(element.tagName, platform) + const name = getMobileNodeIdentity(attrs, platform) + const explicit = isExplicitlyInteractive(attrs, platform) + const interactive = isMobileInteractive(element, platform) + const inViewport = isMobileInViewport(element, platform, walkOpts.viewport) + + // Viewport filtering + if (walkOpts.inViewportOnly) { + if (interactive && !inViewport) { + // Skip this node but still recurse (scroll children may be in view). + for (const child of element.children || []) { + collectMobileNodes(child, platform, depth + 1, nodes, walkOpts) + } + return + } + if (!interactive && !inViewport) { + // Collapse off-screen container to a placeholder. + nodes.push({ + role: 'generic', + name: name ? `${role} "${name}"` : role, + selector: '', + depth, + isInteractive: false, + isExplicitInteractive: false, + isInViewport: false + }) + return + } + } + + // Generate a selector for every interactive element. + // Use the full locator pipeline when source XML is available; + // otherwise fall back to the simplified attribute-based heuristics. + let locator = '' + if (interactive) { + if (walkOpts.sourceXML && walkOpts.automationName) { + // Full pipeline: accessible-id, id, text, uiautomator, xpath, class-name + const suggested = getSuggestedLocators( + element, + walkOpts.sourceXML, + walkOpts.automationName, + { + sourceXML: walkOpts.sourceXML, + parsedDOM: null, + isAndroid: platform === 'android' + } + ) + if (suggested.length > 0) { + locator = suggested[0][1] // first = best priority + } + } + if (!locator) { + // Simplified fallback + locator = + (platform === 'android' + ? getBestAndroidLocator(attrs) + : getBestIOSLocator(attrs)) ?? '' + } + } + + nodes.push({ + role, + name, + selector: locator, + depth, + isInteractive: interactive, + isExplicitInteractive: explicit, + isInViewport: inViewport + }) + + for (const child of element.children || []) { + collectMobileNodes(child, platform, depth + 1, nodes, walkOpts) + } +} + +// --------------------------------------------------------------------------- +// Context inference — shared with the web pipeline. +// Same-depth structural siblings (img, statictext, heading, …) provide +// context for following interactive nodes. +// --------------------------------------------------------------------------- + +const MOBILE_STRUCTURAL_ROLES = new Set([ + 'img', + 'heading', + 'list', + 'listitem', + 'webview', + 'progressbar', + 'slider', + 'switch', + 'generic' +]) + +function mobileInferPurpose( + nodes: MobileFlatNode[], + index: number +): string | undefined { + const myDepth = nodes[index].depth + for (let i = index - 1; i >= 0; i--) { + if (nodes[i].depth <= myDepth && nodes[i].name) { + if ( + nodes[i].depth === myDepth && + !MOBILE_STRUCTURAL_ROLES.has(nodes[i].role) + ) { + continue + } + return nodes[i].name + } + } + return undefined +} + +// --------------------------------------------------------------------------- +// When a tag-only-interactive child (e.g. a statictext TextView) sits +// directly under an explicitly-interactive parent (e.g. a clickable +// LinearLayout row), the *parent* should carry the → selector — the +// child is just a label. Suppress the child's interactivity so the +// parent renders as the actionable element. +// --------------------------------------------------------------------------- + +function suppressTagOnlyChildren(nodes: MobileFlatNode[]): void { + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + if (!node.isInteractive || node.isExplicitInteractive) { + continue + } + // Walk up through ALL ancestors looking for an explicitly-interactive + // parent. The immediate depth-1 parent may just be a layout wrapper; + // the real clickable row could be 2-3 levels up. + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + if (nodes[j].isExplicitInteractive) { + node.isInteractive = false + break // found — suppress and stop + } + // keep looking upward through the ancestor chain + } + } + } +} + +// --------------------------------------------------------------------------- +// Render pass: flat nodes into lines with ∈ context, dedup, noise filter, +// and class-instance indexing. +// --------------------------------------------------------------------------- + +/** Layout roles that carry no semantic meaning by themselves. */ +const NOISY_ROLES = new Set([ + 'FrameLayout', + 'LinearLayout', + 'ViewGroup', + 'RelativeLayout', + 'View', + 'CardView', + 'ConstraintLayout', + 'ScrollView' +]) + +/** + * Pre-count selector occurrences so we can attach .instance(N) suffixes + * to duplicate selectors. + */ +function countSelectors(nodes: MobileFlatNode[]): Map { + const counts = new Map() + for (const node of nodes) { + if (node.selector) { + counts.set(node.selector, (counts.get(node.selector) ?? 0) + 1) + } + } + return counts +} + +function renderMobileNodes(nodes: MobileFlatNode[]): string[] { + const lines: string[] = [] + const selectorCounts = countSelectors(nodes) + const selectorIndex = new Map() + + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + const indent = ' '.repeat(node.depth + 1) + + // Collapse anonymous layout containers at depth ≥ 2. + // Keep depth 0-1 structural chrome and any named container. + if ( + NOISY_ROLES.has(node.role) && + !node.name && + node.depth > 1 && + !node.isInteractive + ) { + continue + } + + // Off-screen containers rendered as collapsed placeholders + if (node.isInViewport === false && !node.isInteractive) { + lines.push(`${indent}⋯ ${node.name} (off-screen)`) + continue + } + + // Dedup: skip statictext whose text is echoed by the parent interactive element + if (node.role === 'statictext' && node.name) { + let echoedByParent = false + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + if ( + nodes[j].isInteractive && + nodes[j].name && + nodes[j].name.includes(node.name) + ) { + echoedByParent = true + } + break + } + } + if (echoedByParent) { + continue + } + } + + if (node.isInteractive && node.selector) { + // Append .instance(N) when the same selector repeats + let selector = node.selector + const total = selectorCounts.get(selector) ?? 1 + if (total > 1) { + const idx = selectorIndex.get(selector) ?? 0 + selectorIndex.set(selector, idx + 1) + selector = `${selector}.instance(${idx})` + } + + const purpose = mobileInferPurpose(nodes, i) + if (node.name) { + lines.push( + purpose + ? `${indent}${node.role} "${node.name}" ∈ "${purpose}" → ${selector}` + : `${indent}${node.role} "${node.name}" → ${selector}` + ) + } else if (purpose) { + lines.push(`${indent}${node.role} ∈ "${purpose}" → ${selector}`) + } else { + lines.push(`${indent}${node.role} → ${selector}`) + } + } else { + // Container / structural / non-locatable + lines.push( + node.name + ? `${indent}${node.role} "${node.name}"` + : `${indent}${node.role}` + ) + } + } + + return lines +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +export interface MobileSnapshotOptions { + /** Only include elements whose bounds intersect the viewport (default true). */ + inViewportOnly?: boolean + /** + * Raw XML page source string. When provided the full locator pipeline + * (getSuggestedLocators) runs on every interactive node, producing the same + * selectors that getElements() returns. Omit to use simplified heuristics. + */ + sourceXML?: string +} + +/** + * Serialize a mobile element tree into a depth-indented text snapshot. + * + * @param root Root JSONElement from the page source XML parse + * @param context Platform, optional device name, viewport, and source XML. + * Include `sourceXML` to use the full locator pipeline. + * @param options {@link MobileSnapshotOptions} + */ +export function serializeMobileSnapshot( + root: JSONElement, + context: { + platform: 'android' | 'ios' + deviceName?: string + viewport?: { width: number; height: number } + /** Raw page-source XML. When set, selectors match getElements() output. */ + sourceXML?: string + }, + options: MobileSnapshotOptions = {} +): string { + const { platform, deviceName, viewport, sourceXML } = context + const { inViewportOnly = true } = options + + // Auto-detect source XML stashed by getMobileVisibleElementsWithTree + const effectiveXML = sourceXML || root.attributes._sourceXML + + const effectiveViewport = viewport ?? { width: 9999, height: 9999 } + const automationName = platform === 'android' ? 'uiautomator2' : 'xcuitest' + + let header = `[${platform}` + if (deviceName) { + header += ` — ${deviceName}` + } + if (viewport) { + header += ` (${viewport.width}×${viewport.height})` + } + header += ']' + + const nodes: MobileFlatNode[] = [] + collectMobileNodes(root, platform, 0, nodes, { + inViewportOnly, + viewport: effectiveViewport, + sourceXML: effectiveXML, + automationName: effectiveXML ? automationName : undefined + }) + + // Let explicitly-interactive parents carry the → selector + suppressTagOnlyChildren(nodes) + + const lines = renderMobileNodes(nodes) + return [header, ...lines].join('\n') +} diff --git a/packages/core/src/element-types.ts b/packages/core/src/element-types.ts new file mode 100644 index 00000000..f4b3fcc6 --- /dev/null +++ b/packages/core/src/element-types.ts @@ -0,0 +1,45 @@ +/** + * Framework-agnostic element types used by element extraction scripts, + * snapshot serializers, and locator generation. + * + * These types describe the data structures returned by browser-injectable + * scripts and mobile page-source parsing. They have no WebdriverIO dependency. + */ + +export interface AccessibilityNode { + role: string + name: string + selector: string + depth: number + level: number | string + disabled: string + checked: string + expanded: string + selected: string + pressed: string + required: string + readonly: string + /** Whether the element's bounding rect intersects the viewport. */ + isInViewport?: boolean +} + +export interface BrowserElementInfo { + tagName: string + name: string // computed accessible name (ARIA spec) + type: string + value: string + href: string + selector: string + isInViewport: boolean + boundingBox?: { x: number; y: number; width: number; height: number } +} + +export interface GetBrowserElementsOptions { + includeBounds?: boolean + /** Only return elements whose bounding rect intersects the viewport (default true). */ + inViewportOnly?: boolean +} + +// Re-export mobile types from locators for convenience. +// Downstream consumers can also import directly from @wdio/devtools-core/locators. +export type { JSONElement } from './locators/types.js' diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5ec95386..5f165d86 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,8 +2,12 @@ // adapters. See ARCHITECTURE.md §2 and CLAUDE.md §2.2. export * from './action-mapping.js' +export * from './action-snapshot.js' export * from './with-timeout.js' export * from './assert-patcher.js' +export * from './element-snapshot.js' +export * from './element-scripts.js' +export * from './element-types.js' export * from './trace-exporter.js' export * from './trace-har.js' export * from './trace-zip-writer.js' diff --git a/packages/elements/src/locators/constants.ts b/packages/core/src/locators/constants.ts similarity index 100% rename from packages/elements/src/locators/constants.ts rename to packages/core/src/locators/constants.ts diff --git a/packages/elements/src/locators/element-filter.ts b/packages/core/src/locators/element-filter.ts similarity index 100% rename from packages/elements/src/locators/element-filter.ts rename to packages/core/src/locators/element-filter.ts diff --git a/packages/core/src/locators/index.ts b/packages/core/src/locators/index.ts new file mode 100644 index 00000000..06e88e22 --- /dev/null +++ b/packages/core/src/locators/index.ts @@ -0,0 +1,282 @@ +/** + * Mobile element locator generation + * + * Main orchestrator module that coordinates XML parsing, element filtering, + * and locator generation for mobile automation. + */ + +// Types +export type { + ElementAttributes, + JSONElement, + Bounds, + FilterOptions, + UniquenessResult, + LocatorStrategy, + LocatorContext, + ElementWithLocators, + GenerateLocatorsOptions +} from './types.js' + +// Constants +export { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS, + ANDROID_LAYOUT_CONTAINERS, + IOS_LAYOUT_CONTAINERS +} from './constants.js' + +// XML Parsing +export { + xmlToJSON, + xmlToDOM, + evaluateXPath, + checkXPathUniqueness, + findDOMNodeByPath, + parseAndroidBounds, + parseIOSBounds, + flattenElementTree, + countAttributeOccurrences, + isAttributeUnique +} from './xml-parsing.js' + +// Element Filtering +export { + isInteractableElement, + isLayoutContainer, + hasMeaningfulContent, + shouldIncludeElement, + getDefaultFilters +} from './element-filter.js' + +// Locator Generation +export { + getSuggestedLocators, + getBestLocator, + locatorsToObject +} from './locator-generation.js' + +import type { + JSONElement, + FilterOptions, + LocatorStrategy, + ElementWithLocators, + GenerateLocatorsOptions, + XMLDocument +} from './types.js' + +import { + xmlToJSON, + xmlToDOM, + parseAndroidBounds, + parseIOSBounds, + findDOMNodeByPath +} from './xml-parsing.js' +import { + shouldIncludeElement, + isLayoutContainer, + hasMeaningfulContent +} from './element-filter.js' +import { getSuggestedLocators, locatorsToObject } from './locator-generation.js' + +interface ProcessingContext { + sourceXML: string + platform: 'android' | 'ios' + automationName: string + isNative: boolean + viewportSize: { width: number; height: number } + filters: FilterOptions + inViewportOnly: boolean + results: ElementWithLocators[] + parsedDOM: XMLDocument | null +} + +/** + * Parse element bounds based on platform + */ +function parseBounds( + element: JSONElement, + platform: 'android' | 'ios' +): { x: number; y: number; width: number; height: number } { + return platform === 'android' + ? parseAndroidBounds(element.attributes.bounds || '') + : parseIOSBounds(element.attributes) +} + +/** + * Check if bounds are within viewport + */ +function isWithinViewport( + bounds: { x: number; y: number; width: number; height: number }, + viewport: { width: number; height: number } +): boolean { + return ( + bounds.x >= 0 && + bounds.y >= 0 && + bounds.width > 0 && + bounds.height > 0 && + bounds.x + bounds.width <= viewport.width && + bounds.y + bounds.height <= viewport.height + ) +} + +/** + * Transform JSONElement to ElementWithLocators + */ +function transformElement( + element: JSONElement, + locators: [LocatorStrategy, string][], + ctx: ProcessingContext +): ElementWithLocators { + const attrs = element.attributes + const bounds = parseBounds(element, ctx.platform) + + return { + tagName: element.tagName, + locators: locatorsToObject(locators), + text: attrs.text || attrs.label || '', + contentDesc: attrs['content-desc'] || '', + resourceId: attrs['resource-id'] || '', + accessibilityId: attrs.name || attrs['content-desc'] || '', + label: attrs.label || '', + value: attrs.value || '', + className: attrs.class || element.tagName, + clickable: + attrs.clickable === 'true' || + attrs.accessible === 'true' || + attrs['long-clickable'] === 'true', + enabled: attrs.enabled !== 'false', + displayed: + ctx.platform === 'android' + ? attrs.displayed !== 'false' + : attrs.visible !== 'false', + bounds, + isInViewport: isWithinViewport(bounds, ctx.viewportSize) + } +} + +/** + * Check if element should be processed + */ +function shouldProcess(element: JSONElement, ctx: ProcessingContext): boolean { + if ( + shouldIncludeElement(element, ctx.filters, ctx.isNative, ctx.automationName) + ) { + return true + } + return ( + isLayoutContainer(element, ctx.platform) && + hasMeaningfulContent(element, ctx.platform) + ) +} + +/** + * Process a single element and add to results if valid + */ +function processElement(element: JSONElement, ctx: ProcessingContext): void { + if (!shouldProcess(element, ctx)) { + return + } + + // Skip off-screen elements early when viewport filtering is on — + // avoids expensive locator generation for elements the caller doesn't want. + if (ctx.inViewportOnly) { + const b = parseBounds(element, ctx.platform) + if (!isWithinViewport(b, ctx.viewportSize)) { + return + } + } + + try { + const targetNode = ctx.parsedDOM + ? findDOMNodeByPath(ctx.parsedDOM, element.path) + : undefined + + const locators = getSuggestedLocators( + element, + ctx.sourceXML, + ctx.automationName, + { + sourceXML: ctx.sourceXML, + parsedDOM: ctx.parsedDOM, + isAndroid: ctx.platform === 'android' + }, + targetNode || undefined + ) + if (locators.length === 0) { + return + } + + // Stash the best locator on the tree node so serializeMobileSnapshot + // can reuse the full locator pipeline instead of recomputing. + element.attributes._selector = locators[0][1] + + const transformed = transformElement(element, locators, ctx) + if (Object.keys(transformed.locators).length === 0) { + return + } + + ctx.results.push(transformed) + } catch (error) { + // Core is logger-free; console.error provides the required + // "enough detail to debug" per the error-handling convention. + // A single bad element never fails the entire page-source walk. + console.error(`[processElement] Error at path ${element.path}:`, error) + } +} + +/** + * Recursively traverse and process element tree + */ +function traverseTree( + element: JSONElement | null, + ctx: ProcessingContext +): void { + if (!element) { + return + } + + processElement(element, ctx) + + for (const child of element.children || []) { + traverseTree(child, ctx) + } +} + +/** + * Generate locators for all elements from page source XML + */ +export function generateAllElementLocators( + sourceXML: string, + options: GenerateLocatorsOptions +): ElementWithLocators[] { + const sourceJSON = xmlToJSON(sourceXML) + + if (!sourceJSON) { + // Core is logger-free; console.error is the only signal that XML + // parsing failed — the caller receives an empty result silently otherwise. + console.error( + '[generateAllElementLocators] Failed to parse page source XML' + ) + return [] + } + + const parsedDOM = xmlToDOM(sourceXML) + + const ctx: ProcessingContext = { + sourceXML, + platform: options.platform, + automationName: + options.platform === 'android' ? 'uiautomator2' : 'xcuitest', + isNative: options.isNative ?? true, + viewportSize: options.viewportSize ?? { width: 9999, height: 9999 }, + filters: options.filters ?? {}, + inViewportOnly: options.inViewportOnly ?? true, + results: [], + parsedDOM + } + + traverseTree(sourceJSON, ctx) + + return ctx.results +} diff --git a/packages/elements/src/locators/locator-generation.ts b/packages/core/src/locators/locator-generation.ts similarity index 81% rename from packages/elements/src/locators/locator-generation.ts rename to packages/core/src/locators/locator-generation.ts index 5433e280..cba05cc8 100644 --- a/packages/elements/src/locators/locator-generation.ts +++ b/packages/core/src/locators/locator-generation.ts @@ -94,10 +94,7 @@ function checkUniqueness( return checkXPathUniqueness(ctx.parsedDOM, xpath, targetNode) } - // Bounded + disjoint negated classes: attr names can never contain `=`, - // `"` or `]`; values can't contain `"`. Bounds prevent polynomial - // backtracking on adversarial input (CodeQL: js/polynomial-redos). - const match = xpath.match(/\/\/\*\[@([^="\]]{1,200})="([^"]{1,2000})"\]/) + const match = xpath.match(/\/\/\*\[@([^=]+)="([^"]+)"\]/) if (match) { const [, attr, value] = match return { isUnique: isAttributeUnique(ctx.sourceXML, attr, value) } @@ -413,101 +410,6 @@ function buildXPath( /** * Get simple locators based on single attributes */ -function appendAndroidSimpleLocators( - attrs: JSONElement['attributes'], - ctx: LocatorContext, - inUiAutomatorScope: boolean, - results: [LocatorStrategy, string][], - targetNode?: XMLNode -): void { - const resourceId = attrs['resource-id'] - if (isValidValue(resourceId)) { - const uniqueness = checkUniqueness( - ctx, - `//*[@resource-id="${resourceId}"]`, - targetNode - ) - const base = `android=new UiSelector().resourceId("${resourceId}")` - if (uniqueness.isUnique && inUiAutomatorScope) { - results.push(['id', base]) - } else if (uniqueness.index && inUiAutomatorScope) { - results.push(['id', generateIndexedUiAutomator(base, uniqueness.index)]) - } - } - - const contentDesc = attrs['content-desc'] - if (isValidValue(contentDesc)) { - const uniqueness = checkUniqueness( - ctx, - `//*[@content-desc="${contentDesc}"]`, - targetNode - ) - if (uniqueness.isUnique) { - results.push(['accessibility-id', `~${contentDesc}`]) - } - } - - const text = attrs.text - if (isValidValue(text) && text.length < 100) { - const uniqueness = checkUniqueness( - ctx, - `//*[@text="${escapeText(text)}"]`, - targetNode - ) - const base = `android=new UiSelector().text("${escapeText(text)}")` - if (uniqueness.isUnique && inUiAutomatorScope) { - results.push(['text', base]) - } else if (uniqueness.index && inUiAutomatorScope) { - results.push(['text', generateIndexedUiAutomator(base, uniqueness.index)]) - } - } -} - -function appendIOSSimpleLocators( - attrs: JSONElement['attributes'], - ctx: LocatorContext, - results: [LocatorStrategy, string][], - targetNode?: XMLNode -): void { - const name = attrs.name - if (isValidValue(name)) { - const uniqueness = checkUniqueness(ctx, `//*[@name="${name}"]`, targetNode) - if (uniqueness.isUnique) { - results.push(['accessibility-id', `~${name}`]) - } - } - - const label = attrs.label - if (isValidValue(label) && label !== attrs.name) { - const uniqueness = checkUniqueness( - ctx, - `//*[@label="${escapeText(label)}"]`, - targetNode - ) - if (uniqueness.isUnique) { - results.push([ - 'predicate-string', - `-ios predicate string:label == "${escapeText(label)}"` - ]) - } - } - - const value = attrs.value - if (isValidValue(value)) { - const uniqueness = checkUniqueness( - ctx, - `//*[@value="${escapeText(value)}"]`, - targetNode - ) - if (uniqueness.isUnique) { - results.push([ - 'predicate-string', - `-ios predicate string:value == "${escapeText(value)}"` - ]) - } - } -} - function getSimpleSuggestedLocators( element: JSONElement, ctx: LocatorContext, @@ -516,20 +418,100 @@ function getSimpleSuggestedLocators( ): [LocatorStrategy, string][] { const results: [LocatorStrategy, string][] = [] const isAndroid = automationName.toLowerCase().includes('uiautomator') + const attrs = element.attributes const inUiAutomatorScope = isAndroid ? isInUiAutomatorScope(element, ctx.parsedDOM) : true + if (isAndroid) { - appendAndroidSimpleLocators( - element.attributes, - ctx, - inUiAutomatorScope, - results, - targetNode - ) + // Resource ID + const resourceId = attrs['resource-id'] + if (isValidValue(resourceId)) { + const xpath = `//*[@resource-id="${resourceId}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique && inUiAutomatorScope) { + results.push([ + 'id', + `android=new UiSelector().resourceId("${resourceId}")` + ]) + } else if (uniqueness.index && inUiAutomatorScope) { + const base = `android=new UiSelector().resourceId("${resourceId}")` + results.push(['id', generateIndexedUiAutomator(base, uniqueness.index)]) + } + } + + // Content Description + const contentDesc = attrs['content-desc'] + if (isValidValue(contentDesc)) { + const xpath = `//*[@content-desc="${contentDesc}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push(['accessibility-id', `~${contentDesc}`]) + } + } + + // Text + const text = attrs.text + if (isValidValue(text) && text.length < 100) { + const xpath = `//*[@text="${escapeText(text)}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique && inUiAutomatorScope) { + results.push([ + 'text', + `android=new UiSelector().text("${escapeText(text)}")` + ]) + } else if (uniqueness.index && inUiAutomatorScope) { + const base = `android=new UiSelector().text("${escapeText(text)}")` + results.push([ + 'text', + generateIndexedUiAutomator(base, uniqueness.index) + ]) + } + } } else { - appendIOSSimpleLocators(element.attributes, ctx, results, targetNode) + // iOS: Accessibility ID (name) + const name = attrs.name + if (isValidValue(name)) { + const xpath = `//*[@name="${name}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push(['accessibility-id', `~${name}`]) + } + } + + // iOS: Label + const label = attrs.label + if (isValidValue(label) && label !== attrs.name) { + const xpath = `//*[@label="${escapeText(label)}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push([ + 'predicate-string', + `-ios predicate string:label == "${escapeText(label)}"` + ]) + } + } + + // iOS: Value + const value = attrs.value + if (isValidValue(value)) { + const xpath = `//*[@value="${escapeText(value)}"]` + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + + if (uniqueness.isUnique) { + results.push([ + 'predicate-string', + `-ios predicate string:value == "${escapeText(value)}"` + ]) + } + } } + return results } diff --git a/packages/elements/src/locators/types.ts b/packages/core/src/locators/types.ts similarity index 100% rename from packages/elements/src/locators/types.ts rename to packages/core/src/locators/types.ts diff --git a/packages/elements/src/locators/xml-parsing.ts b/packages/core/src/locators/xml-parsing.ts similarity index 96% rename from packages/elements/src/locators/xml-parsing.ts rename to packages/core/src/locators/xml-parsing.ts index 09b1c13a..a100a042 100644 --- a/packages/elements/src/locators/xml-parsing.ts +++ b/packages/core/src/locators/xml-parsing.ts @@ -312,10 +312,6 @@ export function countAttributeOccurrences( value: string ): number { const escapedValue = value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - // `attribute` is a fixed set of XML attr names (resource-id, content-desc, - // text, name, label, value) controlled by the locator pipeline; `value` - // is regex-escaped above. No user-controlled regex source. - // eslint-disable-next-line security/detect-non-literal-regexp const pattern = new RegExp(`${attribute}=["']${escapedValue}["']`, 'g') const matches = sourceXML.match(pattern) return matches ? matches.length : 0 diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts index 375c5807..194a8c31 100644 --- a/packages/core/src/trace-exporter.ts +++ b/packages/core/src/trace-exporter.ts @@ -143,7 +143,10 @@ function buildActionEvents( } callCounter++ const callId = `call@${callCounter}` - const endMs = Math.max(prevEndMs, cmd.timestamp - wallTime) + // +1ms minimum duration guarantees endTime > startTime so the viewer + // never sees an `after` whose matching `before` hasn't been parsed yet + // (its action-map lookup crashes on undefined and aborts trace load). + const endMs = Math.max(prevEndMs + 1, cmd.timestamp - wallTime) const params: Record = Object.fromEntries( cmd.args.map((a, i) => [String(i), a]) ) diff --git a/packages/elements/package.json b/packages/elements/package.json index 3c208c09..a334c3fb 100644 --- a/packages/elements/package.json +++ b/packages/elements/package.json @@ -26,12 +26,10 @@ "lint": "eslint . --fix", "test": "vitest run" }, - "dependencies": { - "@xmldom/xmldom": "^0.9.8", - "xpath": "^0.0.34" - }, + "dependencies": {}, "devDependencies": { "@types/node": "25.5.2", + "@wdio/devtools-core": "workspace:^", "@wdio/globals": "9.27.0", "typescript": "6.0.2", "vitest": "^4.0.16" diff --git a/packages/elements/src/accessibility-tree.ts b/packages/elements/src/accessibility-tree.ts index eab1aeb1..01c6a81f 100644 --- a/packages/elements/src/accessibility-tree.ts +++ b/packages/elements/src/accessibility-tree.ts @@ -1,485 +1,27 @@ /** * Browser accessibility tree - * Single browser.execute() call: DOM walk → flat accessibility node list + * Single browser.execute() call: DOM walk → flat accessibility node list. * - * NOTE: This script runs in browser context via browser.execute() - * It must be self-contained with no external dependencies + * The injected script lives in @wdio/devtools-core/element-scripts so it is + * the single source of truth for both the @wdio/elements wrappers and the + * framework-agnostic trace/snapshot pipeline. */ -import { CONTAINER_ROLES, INPUT_TYPE_ROLES } from './aria-roles.js' +import type { AccessibilityNode } from '@wdio/devtools-core/element-types' +import { accessibilityTreeScript as _accessibilityTreeScript } from '@wdio/devtools-core/element-scripts' -export interface AccessibilityNode { - role: string - name: string - selector: string - depth: number - level: number | string - disabled: string - checked: string - expanded: string - selected: string - pressed: string - required: string - readonly: string - /** Whether the element's bounding rect intersects the viewport. */ - isInViewport?: boolean -} - -// Page-injected script: WDIO's `browser.execute` stringifies the arrow body -// and runs it in the browser. Module-level closure values are lost in -// stringification, so ARIA tables defined in aria-roles.ts are passed in -// as execute() args and re-bound here. Same constraint forces every helper -// (getRole/getAccessibleName/getSelector/...) to live inside the IIFE. -// eslint-disable-next-line max-lines-per-function -const accessibilityTreeScript = ( - inViewportOnly: boolean, - inputTypeRoles: Record, - containerRolesArr: readonly string[] -) => - // eslint-disable-next-line max-lines-per-function -- see comment above - (function () { - const INPUT_TYPE_ROLES = inputTypeRoles - const CONTAINER_ROLES = new Set(containerRolesArr) - - // ARIA role-resolution decision tree: each `if (tag === ...)` branch - // maps an HTML element to its implicit role per the WAI-ARIA spec. - // Splitting per-tag would scatter spec-equivalence groups across - // helpers without improving clarity. - // eslint-disable-next-line max-lines-per-function - function getRole(el: HTMLElement): string | null { - const explicit = el.getAttribute('role') - if (explicit) { - return explicit.split(' ')[0] - } - - const tag = el.tagName.toLowerCase() - - switch (tag) { - case 'button': - return 'button' - case 'a': - return el.hasAttribute('href') ? 'link' : null - case 'input': { - const type = (el.getAttribute('type') || 'text').toLowerCase() - if (type === 'hidden') { - return null - } - return INPUT_TYPE_ROLES[type] || 'textbox' - } - case 'select': - return 'combobox' - case 'textarea': - return 'textbox' - case 'h1': - case 'h2': - case 'h3': - case 'h4': - case 'h5': - case 'h6': - return 'heading' - case 'img': - return 'img' - case 'nav': - return 'navigation' - case 'main': - return 'main' - case 'header': - return !el.closest('article,aside,main,nav,section') ? 'banner' : null - case 'footer': - return !el.closest('article,aside,main,nav,section') - ? 'contentinfo' - : null - case 'aside': - return 'complementary' - case 'dialog': - return 'dialog' - case 'form': - return 'form' - case 'section': - return el.hasAttribute('aria-label') || - el.hasAttribute('aria-labelledby') - ? 'region' - : null - case 'summary': - return 'button' - case 'details': - return 'group' - case 'progress': - return 'progressbar' - case 'meter': - return 'meter' - case 'ul': - case 'ol': - return 'list' - case 'li': - return 'listitem' - case 'table': - return 'table' - } - - if ( - (el as HTMLElement & { contentEditable: string }).contentEditable === - 'true' - ) { - return 'textbox' - } - if ( - el.hasAttribute('tabindex') && - parseInt(el.getAttribute('tabindex') || '-1', 10) >= 0 - ) { - return 'generic' - } - - // Capture elements with visible direct text that don't match - // any semantic role — book titles, prices, labels, etc. - if (getDirectText(el)) { - return 'statictext' - } - - return null - } - - // Implements the WAI-ARIA Accessible Name Computation algorithm. - // The 6 sequential fallback steps (aria-label → aria-labelledby → - // tag-specific → placeholder → title → childImg.alt) are spec-ordered; - // each step must run only if prior steps yielded nothing, so they don't - // factor into independent helpers without re-threading the "found" - // signal through every call. - // eslint-disable-next-line max-lines-per-function - function getAccessibleName(el: HTMLElement, role: string | null): string { - const ariaLabel = el.getAttribute('aria-label') - if (ariaLabel) { - return ariaLabel.trim() - } - - const labelledBy = el.getAttribute('aria-labelledby') - if (labelledBy) { - const texts = labelledBy - .split(/\s+/) - .map((id) => document.getElementById(id)?.textContent?.trim() || '') - .filter(Boolean) - if (texts.length > 0) { - return texts.join(' ').slice(0, 200) - } - } - - const tag = el.tagName.toLowerCase() - - if ( - tag === 'img' || - (tag === 'input' && el.getAttribute('type') === 'image') - ) { - const alt = el.getAttribute('alt') - if (alt !== null) { - return alt.trim() - } - } - - if (['input', 'select', 'textarea'].includes(tag)) { - const id = el.getAttribute('id') - if (id) { - const label = document.querySelector(`label[for="${CSS.escape(id)}"]`) - if (label) { - return label.textContent?.trim() || '' - } - } - const parentLabel = el.closest('label') - if (parentLabel) { - const clone = parentLabel.cloneNode(true) as HTMLElement - clone - .querySelectorAll('input,select,textarea') - .forEach((n) => n.remove()) - const lt = clone.textContent?.trim() - if (lt) { - return lt - } - } - } - - const ph = el.getAttribute('placeholder') - if (ph) { - return ph.trim() - } - - const title = el.getAttribute('title') - if (title) { - return title.trim() - } - - // 9. Child — common pattern for image links and buttons - const childImg = el.querySelector('img') - if (childImg) { - const alt = childImg.getAttribute('alt') - if (alt) { - return alt.trim() - } - } - - if (role && CONTAINER_ROLES.has(role)) { - return '' - } - return (el.textContent?.trim().replace(/\s+/g, ' ') || '').slice(0, 200) - } - - // Selector synthesis: tries 6 strategies in priority order (text → aria-label - // → data-testid → name attr → id → CSS path) and returns the first one that - // is unique on the page. Splitting per strategy would obscure the priority - // ordering and force re-uniqueness-checking across helpers. - // eslint-disable-next-line max-lines-per-function - function getSelector(element: HTMLElement): string { - const tag = element.tagName.toLowerCase() - - const text = element.textContent?.trim().replace(/\s+/g, ' ') - if (text && text.length > 0 && text.length <= 120) { - const sameTagElements = document.querySelectorAll(tag) - let matchCount = 0 - sameTagElements.forEach((el) => { - if (el.textContent?.includes(text)) { - matchCount++ - } - }) - if (matchCount === 1) { - return `${tag}*=${text}` - } - } - - const ariaLabel = element.getAttribute('aria-label') - if (ariaLabel && ariaLabel.length <= 200) { - const sel = `[aria-label="${CSS.escape(ariaLabel)}"]` - if (document.querySelectorAll(sel).length === 1) { - return sel - } - } - - const testId = element.getAttribute('data-testid') - if (testId) { - const sel = `[data-testid="${CSS.escape(testId)}"]` - if (document.querySelectorAll(sel).length === 1) { - return sel - } - } - - if (element.id) { - return `#${CSS.escape(element.id)}` - } - - const nameAttr = element.getAttribute('name') - if (nameAttr) { - const sel = `${tag}[name="${CSS.escape(nameAttr)}"]` - if (document.querySelectorAll(sel).length === 1) { - return sel - } - } - - if (element.className && typeof element.className === 'string') { - const classes = element.className.trim().split(/\s+/).filter(Boolean) - for (const cls of classes) { - const sel = `${tag}.${CSS.escape(cls)}` - if (document.querySelectorAll(sel).length === 1) { - return sel - } - } - if (classes.length >= 2) { - const sel = `${tag}${classes - .slice(0, 2) - .map((c) => `.${CSS.escape(c)}`) - .join('')}` - if (document.querySelectorAll(sel).length === 1) { - return sel - } - } - } - - let current: HTMLElement | null = element - const path: string[] = [] - while (current && current !== document.documentElement) { - let seg = current.tagName.toLowerCase() - if (current.id) { - path.unshift(`#${CSS.escape(current.id)}`) - break - } - const parent = current.parentElement - if (parent) { - const siblings = Array.from(parent.children).filter( - (c) => c.tagName === current!.tagName - ) - if (siblings.length > 1) { - seg += `:nth-of-type(${siblings.indexOf(current) + 1})` - } - } - path.unshift(seg) - current = current.parentElement - if (path.length >= 4) { - break - } - } - return path.join(' > ') - } - - /** Extract text from immediate text-node children only (not nested elements). */ - function getDirectText(el: HTMLElement): string { - let text = '' - for (const child of Array.from(el.childNodes)) { - if (child.nodeType === 3 /* TEXT_NODE */) { - text += child.textContent - } - } - return text.trim().replace(/\s+/g, ' ') - } - - function isVisible(el: HTMLElement): boolean { - if (typeof el.checkVisibility === 'function') { - return el.checkVisibility({ - opacityProperty: true, - visibilityProperty: true, - contentVisibilityAuto: true - }) - } - const style = window.getComputedStyle(el) - return ( - style.display !== 'none' && - style.visibility !== 'hidden' && - style.opacity !== '0' && - el.offsetWidth > 0 && - el.offsetHeight > 0 - ) - } - - function isInViewport(el: HTMLElement): boolean { - const rect = el.getBoundingClientRect() - return ( - rect.top >= 0 && - rect.left >= 0 && - rect.bottom <= - (window.innerHeight || document.documentElement.clientHeight) && - rect.right <= - (window.innerWidth || document.documentElement.clientWidth) - ) - } - - function getLevel(el: HTMLElement): number | undefined { - const m = el.tagName.toLowerCase().match(/^h([1-6])$/) - if (m) { - return parseInt(m[1], 10) - } - const ariaLevel = el.getAttribute('aria-level') - if (ariaLevel) { - return parseInt(ariaLevel, 10) - } - return undefined - } - - function getState(el: HTMLElement): Record { - const inputEl = el as HTMLInputElement - const isCheckable = - ['input', 'menuitemcheckbox', 'menuitemradio'].includes( - el.tagName.toLowerCase() - ) || - ['checkbox', 'radio', 'switch'].includes(el.getAttribute('role') || '') - return { - disabled: - el.getAttribute('aria-disabled') === 'true' || inputEl.disabled - ? 'true' - : '', - checked: - isCheckable && inputEl.checked - ? 'true' - : el.getAttribute('aria-checked') || '', - expanded: el.getAttribute('aria-expanded') || '', - selected: el.getAttribute('aria-selected') || '', - pressed: el.getAttribute('aria-pressed') || '', - required: - inputEl.required || el.getAttribute('aria-required') === 'true' - ? 'true' - : '', - readonly: - inputEl.readOnly || el.getAttribute('aria-readonly') === 'true' - ? 'true' - : '' - } - } - - type RawNode = Record - - const result: RawNode[] = [] - - function walk(el: HTMLElement, depth = 0): void { - if (depth > 200) { - return - } - if (!isVisible(el)) { - return - } - - const role = getRole(el) - const inViewport = isInViewport(el) - - if (!role) { - for (const child of Array.from(el.children)) { - walk(child as HTMLElement, depth + 1) - } - return - } - - // When viewport filtering is on, skip nodes outside the viewport. - // Still recurse into children — they may have different positioning - // (e.g. position:fixed elements inside an off-screen container). - if (inViewportOnly && !inViewport) { - for (const child of Array.from(el.children)) { - walk(child as HTMLElement, depth + 1) - } - return - } - - const name = getAccessibleName(el, role) - // Always generate a selector — even elements without an accessible - // name need a CSS-path fallback so the snapshot doesn't lose them. - const selector = getSelector(el) - const node: RawNode = { - role, - name, - selector, - depth, - level: getLevel(el) ?? '', - isInViewport: inViewport, - ...getState(el) - } - result.push(node) - - for (const child of Array.from(el.children)) { - walk(child as HTMLElement, depth + 1) - } - } - - for (const child of Array.from(document.body.children)) { - walk(child as HTMLElement, 0) - } - - return result - })() +export type { AccessibilityNode } /** * Get browser accessibility tree via a single DOM walk. - * - * @param browser WebdriverIO browser instance - * @param options {@link inViewportOnly} defaults to `true` — only nodes - * whose bounding rect intersects the viewport are included. */ -// WDIO's typed Browser.execute overloads don't accept our generic injected -// script — narrow to a permissive execute() shape at the boundary instead. -type ExecuteLike = { - execute: (script: unknown, ...args: unknown[]) => Promise -} - export async function getBrowserAccessibilityTree( browser: WebdriverIO.Browser, options: { inViewportOnly?: boolean } = {} ): Promise { const { inViewportOnly = true } = options - return (browser as unknown as ExecuteLike).execute( - accessibilityTreeScript, - inViewportOnly, - INPUT_TYPE_ROLES, - CONTAINER_ROLES - ) as Promise + const fn = new Function( + `return (${_accessibilityTreeScript(inViewportOnly)})` + ) as () => unknown + return browser.execute(fn) as unknown as Promise } diff --git a/packages/elements/src/aria-roles.ts b/packages/elements/src/aria-roles.ts deleted file mode 100644 index b78736a9..00000000 --- a/packages/elements/src/aria-roles.ts +++ /dev/null @@ -1,68 +0,0 @@ -// WAI-ARIA role data tables shared between the accessibility-tree and -// browser-elements page-injected scripts. Defined at module level so they -// can be type-checked + reused; the consuming scripts receive them as -// arguments to `browser.execute()` since values declared at module scope -// don't survive the function-source stringification that injects the script. - -/** HTML → implicit WAI-ARIA role. */ -export const INPUT_TYPE_ROLES: Record = { - text: 'textbox', - search: 'searchbox', - email: 'textbox', - url: 'textbox', - tel: 'textbox', - password: 'textbox', - number: 'spinbutton', - checkbox: 'checkbox', - radio: 'radio', - range: 'slider', - submit: 'button', - reset: 'button', - image: 'button', - file: 'button', - color: 'button' -} - -/** ARIA roles whose accessible name comes only from aria-label/labelledby, - * never from textContent (otherwise the section text leaks into the name). */ -export const CONTAINER_ROLES: readonly string[] = [ - 'navigation', - 'banner', - 'contentinfo', - 'complementary', - 'main', - 'form', - 'region', - 'group', - 'list', - 'listitem', - 'table', - 'row', - 'rowgroup', - 'generic' -] - -/** CSS selector matching all elements treated as interactable by the page-side - * element walker. Includes native form/anchor elements plus ARIA-role aliases. */ -export const INTERACTABLE_SELECTORS: readonly string[] = [ - 'a[href]', - 'button', - 'input:not([type="hidden"])', - 'select', - 'textarea', - '[role="button"]', - '[role="link"]', - '[role="checkbox"]', - '[role="radio"]', - '[role="tab"]', - '[role="menuitem"]', - '[role="combobox"]', - '[role="option"]', - '[role="switch"]', - '[role="slider"]', - '[role="textbox"]', - '[role="searchbox"]', - '[role="spinbutton"]', - '[contenteditable="true"]', - '[tabindex]:not([tabindex="-1"])' -] diff --git a/packages/elements/src/browser-elements.ts b/packages/elements/src/browser-elements.ts index 208d7627..a08ed3bd 100644 --- a/packages/elements/src/browser-elements.ts +++ b/packages/elements/src/browser-elements.ts @@ -1,307 +1,34 @@ /** * Browser element detection - * Single browser.execute() call: querySelectorAll → flat interactable element list + * Single browser.execute() call: querySelectorAll → flat interactable element list. * - * NOTE: This script runs in browser context via browser.execute() - * It must be self-contained with no external dependencies + * The injected script lives in @wdio/devtools-core/element-scripts so it is + * the single source of truth for both the @wdio/elements wrappers and the + * framework-agnostic trace/snapshot pipeline. */ -import { INTERACTABLE_SELECTORS } from './aria-roles.js' +import type { + BrowserElementInfo, + GetBrowserElementsOptions +} from '@wdio/devtools-core/element-types' +import { elementsScript as _elementsScript } from '@wdio/devtools-core/element-scripts' -export interface BrowserElementInfo { - tagName: string - name: string // computed accessible name (ARIA spec) - type: string - value: string - href: string - selector: string - isInViewport: boolean - boundingBox?: { x: number; y: number; width: number; height: number } -} - -export interface GetBrowserElementsOptions { - includeBounds?: boolean - /** Only return elements whose bounding rect intersects the viewport (default true). */ - inViewportOnly?: boolean -} - -// Page-injected script: WDIO's `browser.execute` stringifies the arrow body -// and runs it in the browser. The selector list lives in aria-roles.ts at -// module scope so it stays type-checked, but it's passed in via execute() -// args because module-level values don't survive script stringification. -// Same constraint forces every helper below to live inside the IIFE. -// eslint-disable-next-line max-lines-per-function -const elementsScript = ( - includeBounds: boolean, - inViewportOnly: boolean, - interactableSelectorsArr: readonly string[] -) => - // eslint-disable-next-line max-lines-per-function -- see comment above - (function () { - const interactableSelectors = interactableSelectorsArr.join(',') - - function isVisible(element: HTMLElement): boolean { - if (typeof element.checkVisibility === 'function') { - return element.checkVisibility({ - opacityProperty: true, - visibilityProperty: true, - contentVisibilityAuto: true - }) - } - const style = window.getComputedStyle(element) - return ( - style.display !== 'none' && - style.visibility !== 'hidden' && - style.opacity !== '0' && - element.offsetWidth > 0 && - element.offsetHeight > 0 - ) - } - - // WAI-ARIA Accessible Name Computation algorithm — see accessibility-tree.ts - // for the longer rationale. Sequential spec-ordered fallback steps. - // eslint-disable-next-line max-lines-per-function - function getAccessibleName(el: HTMLElement): string { - // 1. aria-label - const ariaLabel = el.getAttribute('aria-label') - if (ariaLabel) { - return ariaLabel.trim() - } - - // 2. aria-labelledby — resolve referenced elements - const labelledBy = el.getAttribute('aria-labelledby') - if (labelledBy) { - const texts = labelledBy - .split(/\s+/) - .map((id) => document.getElementById(id)?.textContent?.trim() || '') - .filter(Boolean) - if (texts.length > 0) { - return texts.join(' ').slice(0, 200) - } - } - - const tag = el.tagName.toLowerCase() - - // 3. alt for images and input[type=image] - if ( - tag === 'img' || - (tag === 'input' && el.getAttribute('type') === 'image') - ) { - const alt = el.getAttribute('alt') - if (alt !== null) { - return alt.trim() - } - } - - // 4. label[for=id] for form elements - if (['input', 'select', 'textarea'].includes(tag)) { - const id = el.getAttribute('id') - if (id) { - const label = document.querySelector(`label[for="${CSS.escape(id)}"]`) - if (label) { - return label.textContent?.trim() || '' - } - } - // 5. Wrapping label — clone, strip inputs, read text - const parentLabel = el.closest('label') - if (parentLabel) { - const clone = parentLabel.cloneNode(true) as HTMLElement - clone - .querySelectorAll('input,select,textarea') - .forEach((n) => n.remove()) - const lt = clone.textContent?.trim() - if (lt) { - return lt - } - } - } - - // 6. placeholder - const ph = el.getAttribute('placeholder') - if (ph) { - return ph.trim() - } - - // 7. title - const title = el.getAttribute('title') - if (title) { - return title.trim() - } - - // 8. text content (truncated, whitespace normalized) - return (el.textContent?.trim().replace(/\s+/g, ' ') || '').slice(0, 200) - } - - // Selector synthesis — see accessibility-tree.ts for the longer rationale. - // Tries 6 priority-ordered strategies; each must re-check uniqueness. - // eslint-disable-next-line max-lines-per-function - function getSelector(element: HTMLElement): string { - const tag = element.tagName.toLowerCase() - - // 1. tag*=Text — best per WebdriverIO docs - const text = element.textContent?.trim().replace(/\s+/g, ' ') - if (text && text.length > 0 && text.length <= 120) { - const sameTagElements = document.querySelectorAll(tag) - let matchCount = 0 - sameTagElements.forEach((el) => { - if (el.textContent?.includes(text)) { - matchCount++ - } - }) - if (matchCount === 1) { - return `${tag}*=${text}` - } - } - - // 2. aria/label - const ariaLabel = element.getAttribute('aria-label') - if (ariaLabel && ariaLabel.length <= 200) { - const sel = `[aria-label="${CSS.escape(ariaLabel)}"]` - if (document.querySelectorAll(sel).length === 1) { - return sel - } - } - - // 3. data-testid - const testId = element.getAttribute('data-testid') - if (testId) { - const sel = `[data-testid="${CSS.escape(testId)}"]` - if (document.querySelectorAll(sel).length === 1) { - return sel - } - } - - // 4. #id - if (element.id) { - return `#${CSS.escape(element.id)}` - } - - // 5. [name] — form elements - const nameAttr = element.getAttribute('name') - if (nameAttr) { - const sel = `${tag}[name="${CSS.escape(nameAttr)}"]` - if (document.querySelectorAll(sel).length === 1) { - return sel - } - } - - // 6. tag.class — try each class individually, then first-two combination - if (element.className && typeof element.className === 'string') { - const classes = element.className.trim().split(/\s+/).filter(Boolean) - for (const cls of classes) { - const sel = `${tag}.${CSS.escape(cls)}` - if (document.querySelectorAll(sel).length === 1) { - return sel - } - } - if (classes.length >= 2) { - const sel = `${tag}${classes - .slice(0, 2) - .map((c) => `.${CSS.escape(c)}`) - .join('')}` - if (document.querySelectorAll(sel).length === 1) { - return sel - } - } - } - - // 7. CSS path fallback - let current: HTMLElement | null = element - const path: string[] = [] - while (current && current !== document.documentElement) { - let seg = current.tagName.toLowerCase() - if (current.id) { - path.unshift(`#${CSS.escape(current.id)}`) - break - } - const parent = current.parentElement - if (parent) { - const siblings = Array.from(parent.children).filter( - (c) => c.tagName === current!.tagName - ) - if (siblings.length > 1) { - seg += `:nth-of-type(${siblings.indexOf(current) + 1})` - } - } - path.unshift(seg) - current = current.parentElement - if (path.length >= 4) { - break - } - } - return path.join(' > ') - } - - const elements: Record[] = [] - const seen = new Set() - - document.querySelectorAll(interactableSelectors).forEach((el) => { - if (seen.has(el)) { - return - } - seen.add(el) - - const htmlEl = el as HTMLElement - if (!isVisible(htmlEl)) { - return - } - - const inputEl = htmlEl as HTMLInputElement - const rect = htmlEl.getBoundingClientRect() - const isInViewport = - rect.top >= 0 && - rect.left >= 0 && - rect.bottom <= - (window.innerHeight || document.documentElement.clientHeight) && - rect.right <= - (window.innerWidth || document.documentElement.clientWidth) - - if (inViewportOnly && !isInViewport) { - return - } - - const entry: Record = { - tagName: htmlEl.tagName.toLowerCase(), - name: getAccessibleName(htmlEl), - type: htmlEl.getAttribute('type') || '', - value: inputEl.value || '', - href: htmlEl.getAttribute('href') || '', - selector: getSelector(htmlEl), - isInViewport - } - - if (includeBounds) { - entry.boundingBox = { - x: rect.x + window.scrollX, - y: rect.y + window.scrollY, - width: rect.width, - height: rect.height - } - } - - elements.push(entry) - }) - - return elements - })() +export type { BrowserElementInfo, GetBrowserElementsOptions } /** * Get interactable browser elements via querySelectorAll. + * + * The script body lives in core but is converted back to a function for + * WDIO's `browser.execute(fn, args)` serialization. Passing a raw string + * to execute() invokes a different code path that may not preserve scope. */ export async function getInteractableBrowserElements( browser: WebdriverIO.Browser, options: GetBrowserElementsOptions = {} ): Promise { const { includeBounds = false, inViewportOnly = true } = options - // WDIO's typed Browser.execute overloads don't accept the injected script — - // narrow to a permissive execute() shape at the boundary. - type ExecuteLike = { - execute: (script: unknown, ...args: unknown[]) => Promise - } - return (browser as unknown as ExecuteLike).execute( - elementsScript, - includeBounds, - inViewportOnly, - INTERACTABLE_SELECTORS - ) as Promise + const fn = new Function( + `return (${_elementsScript(includeBounds, inViewportOnly)})` + ) as () => unknown + return browser.execute(fn) as unknown as Promise } diff --git a/packages/elements/src/get-elements.ts b/packages/elements/src/get-elements.ts index e763a1ff..4aedbf8b 100644 --- a/packages/elements/src/get-elements.ts +++ b/packages/elements/src/get-elements.ts @@ -1,6 +1,6 @@ import { getInteractableBrowserElements } from './browser-elements.js' import { getMobileVisibleElementsWithTree } from './mobile-elements.js' -import type { JSONElement } from './locators/types.js' +import type { JSONElement } from './locators/index.js' export type VisibleElementsResult = { total: number diff --git a/packages/elements/src/index.ts b/packages/elements/src/index.ts index 7aeabf78..37e495bd 100644 --- a/packages/elements/src/index.ts +++ b/packages/elements/src/index.ts @@ -1,3 +1,7 @@ +// WDIO-dependent element extraction wrappers. +// Framework-agnostic types, serializers, scripts, and locator generation live +// in @wdio/devtools-core and are re-exported here for backward compatibility. + export { getInteractableBrowserElements } from './browser-elements.js' export type { BrowserElementInfo, @@ -18,4 +22,4 @@ export type { VisibleElementsResult } from './get-elements.js' export { serializeWebSnapshot, serializeMobileSnapshot } from './snapshot.js' export type { WebSnapshotOptions, MobileSnapshotOptions } from './snapshot.js' -export type { JSONElement } from './locators/types.js' +export type { JSONElement } from './locators/index.js' diff --git a/packages/elements/src/locators/index.ts b/packages/elements/src/locators/index.ts index 20e2330c..09c12bc2 100644 --- a/packages/elements/src/locators/index.ts +++ b/packages/elements/src/locators/index.ts @@ -1,13 +1,7 @@ /** - * Mobile element locator generation - * - * Main orchestrator module that coordinates XML parsing, element filtering, - * and locator generation for mobile automation. - * - * Based on: https://github.com/appium/appium-mcp + * Mobile element locator generation — re-exported from @wdio/devtools-core. */ -// Types export type { ElementAttributes, JSONElement, @@ -18,18 +12,13 @@ export type { LocatorContext, ElementWithLocators, GenerateLocatorsOptions -} from './types.js' +} from '@wdio/devtools-core/locators' -// Constants export { ANDROID_INTERACTABLE_TAGS, IOS_INTERACTABLE_TAGS, ANDROID_LAYOUT_CONTAINERS, - IOS_LAYOUT_CONTAINERS -} from './constants.js' - -// XML Parsing -export { + IOS_LAYOUT_CONTAINERS, xmlToJSON, xmlToDOM, evaluateXPath, @@ -39,241 +28,14 @@ export { parseIOSBounds, flattenElementTree, countAttributeOccurrences, - isAttributeUnique -} from './xml-parsing.js' - -// Element Filtering -export { + isAttributeUnique, isInteractableElement, isLayoutContainer, hasMeaningfulContent, shouldIncludeElement, - getDefaultFilters -} from './element-filter.js' - -// Locator Generation -export { + getDefaultFilters, getSuggestedLocators, getBestLocator, - locatorsToObject -} from './locator-generation.js' - -import type { - JSONElement, - FilterOptions, - LocatorStrategy, - ElementWithLocators, - GenerateLocatorsOptions, - XMLDocument -} from './types.js' - -import { - xmlToJSON, - xmlToDOM, - parseAndroidBounds, - parseIOSBounds, - findDOMNodeByPath -} from './xml-parsing.js' -import { - shouldIncludeElement, - isLayoutContainer, - hasMeaningfulContent -} from './element-filter.js' -import { getSuggestedLocators, locatorsToObject } from './locator-generation.js' - -interface ProcessingContext { - sourceXML: string - platform: 'android' | 'ios' - automationName: string - isNative: boolean - viewportSize: { width: number; height: number } - filters: FilterOptions - inViewportOnly: boolean - results: ElementWithLocators[] - parsedDOM: XMLDocument | null -} - -/** - * Parse element bounds based on platform - */ -function parseBounds( - element: JSONElement, - platform: 'android' | 'ios' -): { x: number; y: number; width: number; height: number } { - return platform === 'android' - ? parseAndroidBounds(element.attributes.bounds || '') - : parseIOSBounds(element.attributes) -} - -/** - * Check if bounds are within viewport - */ -function isWithinViewport( - bounds: { x: number; y: number; width: number; height: number }, - viewport: { width: number; height: number } -): boolean { - return ( - bounds.x >= 0 && - bounds.y >= 0 && - bounds.width > 0 && - bounds.height > 0 && - bounds.x + bounds.width <= viewport.width && - bounds.y + bounds.height <= viewport.height - ) -} - -/** - * Transform JSONElement to ElementWithLocators - */ -function transformElement( - element: JSONElement, - locators: [LocatorStrategy, string][], - ctx: ProcessingContext -): ElementWithLocators { - const attrs = element.attributes - const bounds = parseBounds(element, ctx.platform) - - return { - tagName: element.tagName, - locators: locatorsToObject(locators), - text: attrs.text || attrs.label || '', - contentDesc: attrs['content-desc'] || '', - resourceId: attrs['resource-id'] || '', - accessibilityId: attrs.name || attrs['content-desc'] || '', - label: attrs.label || '', - value: attrs.value || '', - className: attrs.class || element.tagName, - clickable: - attrs.clickable === 'true' || - attrs.accessible === 'true' || - attrs['long-clickable'] === 'true', - enabled: attrs.enabled !== 'false', - displayed: - ctx.platform === 'android' - ? attrs.displayed !== 'false' - : attrs.visible !== 'false', - bounds, - isInViewport: isWithinViewport(bounds, ctx.viewportSize) - } -} - -/** - * Check if element should be processed - */ -function shouldProcess(element: JSONElement, ctx: ProcessingContext): boolean { - if ( - shouldIncludeElement(element, ctx.filters, ctx.isNative, ctx.automationName) - ) { - return true - } - return ( - isLayoutContainer(element, ctx.platform) && - hasMeaningfulContent(element, ctx.platform) - ) -} - -/** - * Process a single element and add to results if valid - */ -function processElement(element: JSONElement, ctx: ProcessingContext): void { - if (!shouldProcess(element, ctx)) { - return - } - - // Skip off-screen elements early when viewport filtering is on — - // avoids expensive locator generation for elements the caller doesn't want. - if (ctx.inViewportOnly) { - const b = parseBounds(element, ctx.platform) - if (!isWithinViewport(b, ctx.viewportSize)) { - return - } - } - - try { - const targetNode = ctx.parsedDOM - ? findDOMNodeByPath(ctx.parsedDOM, element.path) - : undefined - - const locators = getSuggestedLocators( - element, - ctx.sourceXML, - ctx.automationName, - { - sourceXML: ctx.sourceXML, - parsedDOM: ctx.parsedDOM, - isAndroid: ctx.platform === 'android' - }, - targetNode || undefined - ) - if (locators.length === 0) { - return - } - - // Stash the best locator on the tree node so serializeMobileSnapshot - // can reuse the full locator pipeline instead of recomputing. - element.attributes._selector = locators[0][1] - - const transformed = transformElement(element, locators, ctx) - if (Object.keys(transformed.locators).length === 0) { - return - } - - ctx.results.push(transformed) - } catch (error) { - console.error(`[processElement] Error at path ${element.path}:`, error) - } -} - -/** - * Recursively traverse and process element tree - */ -function traverseTree( - element: JSONElement | null, - ctx: ProcessingContext -): void { - if (!element) { - return - } - - processElement(element, ctx) - - for (const child of element.children || []) { - traverseTree(child, ctx) - } -} - -/** - * Generate locators for all elements from page source XML - */ -export function generateAllElementLocators( - sourceXML: string, - options: GenerateLocatorsOptions -): ElementWithLocators[] { - const sourceJSON = xmlToJSON(sourceXML) - - if (!sourceJSON) { - console.error( - '[generateAllElementLocators] Failed to parse page source XML' - ) - return [] - } - - const parsedDOM = xmlToDOM(sourceXML) - - const ctx: ProcessingContext = { - sourceXML, - platform: options.platform, - automationName: - options.platform === 'android' ? 'uiautomator2' : 'xcuitest', - isNative: options.isNative ?? true, - viewportSize: options.viewportSize ?? { width: 9999, height: 9999 }, - filters: options.filters ?? {}, - inViewportOnly: options.inViewportOnly ?? true, - results: [], - parsedDOM - } - - traverseTree(sourceJSON, ctx) - - return ctx.results -} + locatorsToObject, + generateAllElementLocators +} from '@wdio/devtools-core/locators' diff --git a/packages/elements/src/snapshot.ts b/packages/elements/src/snapshot.ts index b49de4c7..ed83ae34 100644 --- a/packages/elements/src/snapshot.ts +++ b/packages/elements/src/snapshot.ts @@ -1,775 +1,12 @@ /** - * AI-readable snapshot serializers - * - * Converts accessibility trees and mobile element trees into depth-indented - * text files that LLMs can consume without any parsing. + * AI-readable snapshot serializers — re-exported from @wdio/devtools-core. */ -import type { AccessibilityNode } from './accessibility-tree.js' -import type { JSONElement } from './locators/types.js' -import { parseAndroidBounds, parseIOSBounds } from './locators/xml-parsing.js' -import { - ANDROID_INTERACTABLE_TAGS, - IOS_INTERACTABLE_TAGS -} from './locators/constants.js' -import { getSuggestedLocators } from './locators/locator-generation.js' - -/** - * Roles that can be interacted with — rendered with `→ selector`. - * Structural roles (heading, img, form, nav, …) are intentionally excluded. - */ -const INTERACTIVE_ROLES = new Set([ - 'button', - 'link', - 'textbox', - 'checkbox', - 'radio', - 'combobox', - 'slider', - 'searchbox', - 'spinbutton', - 'switch', - 'tab', - 'menuitem', - 'option' -]) - -/** - * Walk backwards from `index` to find the nearest ancestor or preceding - * structural sibling with a non-empty name. Same-depth nodes are only - * used when they are structural (img, heading, statictext, …) — never - * another interactive element. - */ -function inferPurpose( - nodes: AccessibilityNode[], - index: number -): string | undefined { - const myDepth = nodes[index].depth - for (let i = index - 1; i >= 0; i--) { - if (nodes[i].depth <= myDepth && nodes[i].name) { - // Same-depth sibling: only structural elements count - if (nodes[i].depth === myDepth && INTERACTIVE_ROLES.has(nodes[i].role)) { - continue - } - return nodes[i].name - } - } - return undefined -} - -export interface WebSnapshotOptions { - /** Only include nodes whose bounding rect intersects the viewport (default true). */ - inViewportOnly?: boolean -} - -/** - * Serialize a web accessibility tree into a depth-indented text snapshot. - * - * @param nodes Flat ordered node list from getBrowserAccessibilityTree() - * @param context Optional page context for the header line - * @param options {@link WebSnapshotOptions} - */ -// Single linear pass over the flat node list — per-node decisions (skip-by- -// viewport, role classification, statictext echo dedup, interactive vs -// structural rendering) must stay together so the inferred-purpose lookup -// can see siblings. ROADMAP P2: collapse with mobile pipeline into one -// `serializeSnapshot()`; until then this is the canonical web walker. -// eslint-disable-next-line max-lines-per-function -export function serializeWebSnapshot( - nodes: AccessibilityNode[], - context?: { url?: string; title?: string }, - options: WebSnapshotOptions = {} -): string { - const { inViewportOnly = true } = options - - let header = '[Page' - if (context?.title) { - header += `: ${context.title}` - } - if (context?.url) { - header += ` — ${context.url}` - } - header += ']' - - const lines: string[] = [header] - - for (let i = 0; i < nodes.length; i++) { - const node = nodes[i] - - // When viewport filtering is on, skip nodes that are known to be off-screen. - // Nodes from a tree captured with inViewportOnly=false will have - // isInViewport populated; nodes from a pre-filtered tree all have - // isInViewport=true (or undefined for pre-existing data). - if (inViewportOnly && node.isInViewport === false) { - continue - } - - const indent = ' '.repeat(node.depth + 1) // +1 indents everything under the header - const isInteractive = INTERACTIVE_ROLES.has(node.role) - - // Skip statictext that merely echoes the parent link/button name. - // Example: link "Highlights" → a*=Highlights doesn't need - // statictext "Highlights" as a child because it adds no information. - if (node.role === 'statictext' && node.name) { - let echoedByParent = false - for (let j = i - 1; j >= 0; j--) { - if (nodes[j].depth < node.depth) { - const parentRole = nodes[j].role - const parentName = nodes[j].name - if ( - INTERACTIVE_ROLES.has(parentRole) && - parentName && - parentName.includes(node.name) - ) { - echoedByParent = true - } - break // only check the immediate structural parent - } - } - if (echoedByParent) { - continue - } - } - - // Heading gets level suffix: heading[2] - const roleLabel = - node.role === 'heading' && node.level - ? `heading[${node.level}]` - : node.role - - if (isInteractive) { - // No selector → agent can't act on this node; skip entirely - if (!node.selector) { - continue - } - const purpose = inferPurpose(nodes, i) - if (node.name) { - // Show parent context when available — disambiguates - // duplicate selectors like six "Add to Wishlist" buttons. - lines.push( - purpose - ? `${indent}${roleLabel} "${node.name}" ∈ "${purpose}" → ${node.selector}` - : `${indent}${roleLabel} "${node.name}" → ${node.selector}` - ) - } else if (purpose) { - lines.push(`${indent}${roleLabel} ∈ "${purpose}" → ${node.selector}`) - } else { - lines.push(`${indent}${roleLabel} → ${node.selector}`) - } - } else { - // Container / structural: show role + name when present, no selector - lines.push( - node.name - ? `${indent}${roleLabel} "${node.name}"` - : `${indent}${roleLabel}` - ) - } - } - - return lines.join('\n') -} - -// --------------------------------------------------------------------------- -// Mobile snapshot helpers -// --------------------------------------------------------------------------- - -/** Shorten fully-qualified Android/iOS class names to the last segment. */ -function simplifyTag(tagName: string): string { - const dot = tagName.lastIndexOf('.') - if (dot !== -1) { - return tagName.slice(dot + 1) - } - return tagName.replace(/^XCUIElementType/, '') -} - -// --------------------------------------------------------------------------- -// Mobile role classification — maps raw Android/iOS class names to semantic -// roles so the snapshot reads like the web version (button, textbox, img, …). -// --------------------------------------------------------------------------- - -const ANDROID_ROLE_MAP: Record = { - 'android.widget.Button': 'button', - 'android.widget.ImageButton': 'button', - 'android.widget.ToggleButton': 'button', - 'android.widget.FloatingActionButton': 'button', - 'com.google.android.material.button.MaterialButton': 'button', - 'com.google.android.material.floatingactionbutton.FloatingActionButton': - 'button', - 'android.widget.EditText': 'textbox', - 'android.widget.AutoCompleteTextView': 'textbox', - 'android.widget.MultiAutoCompleteTextView': 'textbox', - 'android.widget.SearchView': 'searchbox', - 'android.widget.ImageView': 'img', - 'android.widget.QuickContactBadge': 'img', - 'android.widget.CheckBox': 'checkbox', - 'android.widget.RadioButton': 'radio', - 'android.widget.Switch': 'switch', - 'android.widget.Spinner': 'combobox', - 'android.widget.SeekBar': 'slider', - 'android.widget.RatingBar': 'slider', - 'android.widget.ProgressBar': 'progressbar', - 'android.widget.TextView': 'statictext', - 'android.widget.CheckedTextView': 'statictext', - 'android.widget.RecyclerView': 'list', - 'android.widget.ListView': 'list', - 'android.widget.GridView': 'list', - 'android.webkit.WebView': 'webview' -} - -const IOS_ROLE_MAP: Record = { - XCUIElementTypeButton: 'button', - XCUIElementTypeLink: 'link', - XCUIElementTypeTextField: 'textbox', - XCUIElementTypeSecureTextField: 'textbox', - XCUIElementTypeTextView: 'textbox', - XCUIElementTypeSearchField: 'searchbox', - XCUIElementTypeImage: 'img', - XCUIElementTypeIcon: 'img', - XCUIElementTypeSwitch: 'switch', - XCUIElementTypeSlider: 'slider', - XCUIElementTypeStepper: 'slider', - XCUIElementTypeCheckBox: 'checkbox', - XCUIElementTypeRadioButton: 'radio', - XCUIElementTypePicker: 'combobox', - XCUIElementTypePickerWheel: 'combobox', - XCUIElementTypeDatePicker: 'combobox', - XCUIElementTypeSegmentedControl: 'combobox', - XCUIElementTypeStaticText: 'statictext', - XCUIElementTypeCell: 'listitem', - XCUIElementTypeTable: 'list', - XCUIElementTypeCollectionView: 'list' -} - -function classifyMobileRole( - tagName: string, - platform: 'android' | 'ios' -): string { - if (platform === 'android') { - return ANDROID_ROLE_MAP[tagName] || simplifyTag(tagName) - } - return IOS_ROLE_MAP[tagName] || simplifyTag(tagName) -} - -// --------------------------------------------------------------------------- -// Locator generation -// --------------------------------------------------------------------------- - -function getBestAndroidLocator( - attrs: JSONElement['attributes'] -): string | undefined { - // Pre-computed by the full locator pipeline (generateAllElementLocators). - // Takes priority over the simplified fallback logic below. - if (attrs._selector) { - return attrs._selector - } - // ~ prefix = accessibility-id shorthand in WebdriverIO ($('~foo')) - if (attrs['content-desc']) { - return `~${attrs['content-desc']}` - } - if (attrs['resource-id']) { - return `id:${attrs['resource-id']}` - } - if (attrs.text) { - return `~${attrs.text}` - } - // Fallback: class-based locator (only useful with :nth-of-type or index) - if (attrs.class) { - return `class:${simplifyTag(attrs.class)}` - } - return undefined -} - -function getBestIOSLocator( - attrs: JSONElement['attributes'] -): string | undefined { - // Pre-computed by the full locator pipeline. - if (attrs._selector) { - return attrs._selector - } - // ~ prefix = accessibility-id shorthand (maps to `name` on iOS) - if (attrs.name) { - return `~${attrs.name}` - } - if (attrs.label) { - return `~${attrs.label}` - } - if (attrs.value) { - return `~${attrs.value}` - } - // Fallback: class-based locator - if (attrs.type) { - return `class:${simplifyTag(attrs.type)}` - } - return undefined -} - -// --------------------------------------------------------------------------- -// Identity -// --------------------------------------------------------------------------- - -function getMobileNodeIdentity( - attrs: JSONElement['attributes'], - platform: 'android' | 'ios' -): string { - if (platform === 'android') { - const contentDesc = attrs['content-desc'] - if (contentDesc) { - return contentDesc - } - if (attrs.text) { - return attrs.text - } - // Fall back to the last segment of the resource-id (e.g. "search_action_bar") - const rid = attrs['resource-id'] - if (rid) { - const slash = rid.lastIndexOf('/') - return slash !== -1 ? rid.slice(slash + 1) : rid - } - return '' - } - return attrs.name || attrs.label || attrs.value || attrs.text || '' -} - -// --------------------------------------------------------------------------- -// Interactivity -// --------------------------------------------------------------------------- - -const ANDROID_INTERACTABLE_SET = new Set(ANDROID_INTERACTABLE_TAGS) -const IOS_INTERACTABLE_SET = new Set(IOS_INTERACTABLE_TAGS) - -/** An element is *explicitly* interactive when it carries a click/focus/check - * attribute — as opposed to being interactive only because its tag is in the - * interactable-tag list. Explicit parents should carry the → selector, not - * their tag-interactive children. */ -function isExplicitlyInteractive( - attrs: JSONElement['attributes'], - platform: 'android' | 'ios' -): boolean { - if (platform === 'android') { - return ( - attrs.clickable === 'true' || - attrs.focusable === 'true' || - attrs.checkable === 'true' || - attrs['long-clickable'] === 'true' - ) - } - return attrs.accessible === 'true' -} - -function isMobileInteractive( - element: JSONElement, - platform: 'android' | 'ios' -): boolean { - const attrs = element.attributes - if (platform === 'android') { - if (ANDROID_INTERACTABLE_SET.has(element.tagName)) { - return true - } - return ( - attrs.clickable === 'true' || - attrs['long-clickable'] === 'true' || - attrs.focusable === 'true' || - attrs.checkable === 'true' - ) - } - if (IOS_INTERACTABLE_SET.has(element.tagName)) { - return true - } - return attrs.accessible === 'true' -} - -// --------------------------------------------------------------------------- -// Viewport -// --------------------------------------------------------------------------- - -interface WalkMobileOptions { - inViewportOnly: boolean - viewport: { width: number; height: number } - /** Raw page-source XML. When provided, the full locator pipeline is used. */ - sourceXML?: string - /** 'uiautomator2' or 'xcuitest'. Required when sourceXML is set. */ - automationName?: string -} - -function isMobileInViewport( - element: JSONElement, - platform: 'android' | 'ios', - viewport: { width: number; height: number } -): boolean { - const bounds = - platform === 'android' - ? parseAndroidBounds(element.attributes.bounds || '') - : parseIOSBounds(element.attributes) - - if (bounds.width === 0 && bounds.height === 0) { - return true - } - - return ( - bounds.x >= 0 && - bounds.y >= 0 && - bounds.width > 0 && - bounds.height > 0 && - bounds.x + bounds.width <= viewport.width && - bounds.y + bounds.height <= viewport.height - ) -} - -// --------------------------------------------------------------------------- -// Flat-node representation (mirrors AccessibilityNode so both pipelines share -// inferPurpose, dedup, and rendering logic). -// --------------------------------------------------------------------------- - -interface MobileFlatNode { - role: string - name: string - selector: string - depth: number - isInteractive: boolean - /** True when the element has clickable/focusable/checkable — the intended tap target. */ - isExplicitInteractive: boolean - isInViewport: boolean -} - -/** - * First pass: walk the JSONElement tree, apply viewport filtering and - * collect every node into a flat array with semantic roles and selectors. - */ -// Recursive tree walker — splitting the viewport filter, locator pipeline, -// fallback, and node-emit branches would require threading the accumulator -// + walk options through 4 helpers. ROADMAP P0/P1: this whole pipeline gets -// merged with generateAllElementLocators; until that consolidation lands, -// keep as one walker. -// eslint-disable-next-line max-lines-per-function -function collectMobileNodes( - element: JSONElement, - platform: 'android' | 'ios', - depth: number, - nodes: MobileFlatNode[], - walkOpts: WalkMobileOptions -): void { - const attrs = element.attributes - const role = classifyMobileRole(element.tagName, platform) - const name = getMobileNodeIdentity(attrs, platform) - const explicit = isExplicitlyInteractive(attrs, platform) - const interactive = isMobileInteractive(element, platform) - const inViewport = isMobileInViewport(element, platform, walkOpts.viewport) - - // Viewport filtering - if (walkOpts.inViewportOnly) { - if (interactive && !inViewport) { - // Skip this node but still recurse (scroll children may be in view). - for (const child of element.children || []) { - collectMobileNodes(child, platform, depth + 1, nodes, walkOpts) - } - return - } - if (!interactive && !inViewport) { - // Collapse off-screen container to a placeholder. - nodes.push({ - role: 'generic', - name: name ? `${role} "${name}"` : role, - selector: '', - depth, - isInteractive: false, - isExplicitInteractive: false, - isInViewport: false - }) - return - } - } - - // Generate a selector for every interactive element. - // Use the full locator pipeline when source XML is available; - // otherwise fall back to the simplified attribute-based heuristics. - let locator = '' - if (interactive) { - if (walkOpts.sourceXML && walkOpts.automationName) { - // Full pipeline: accessible-id, id, text, uiautomator, xpath, class-name - const suggested = getSuggestedLocators( - element, - walkOpts.sourceXML, - walkOpts.automationName, - { - sourceXML: walkOpts.sourceXML, - parsedDOM: null, - isAndroid: platform === 'android' - } - ) - if (suggested.length > 0) { - locator = suggested[0][1] // first = best priority - } - } - if (!locator) { - // Simplified fallback - locator = - (platform === 'android' - ? getBestAndroidLocator(attrs) - : getBestIOSLocator(attrs)) ?? '' - } - } - - nodes.push({ - role, - name, - selector: locator, - depth, - isInteractive: interactive, - isExplicitInteractive: explicit, - isInViewport: inViewport - }) - - for (const child of element.children || []) { - collectMobileNodes(child, platform, depth + 1, nodes, walkOpts) - } -} - -// --------------------------------------------------------------------------- -// Context inference — shared with the web pipeline. -// Same-depth structural siblings (img, statictext, heading, …) provide -// context for following interactive nodes. -// --------------------------------------------------------------------------- - -const MOBILE_STRUCTURAL_ROLES = new Set([ - 'img', - 'heading', - 'list', - 'listitem', - 'webview', - 'progressbar', - 'slider', - 'switch', - 'generic' -]) - -function mobileInferPurpose( - nodes: MobileFlatNode[], - index: number -): string | undefined { - const myDepth = nodes[index].depth - for (let i = index - 1; i >= 0; i--) { - if (nodes[i].depth <= myDepth && nodes[i].name) { - if ( - nodes[i].depth === myDepth && - !MOBILE_STRUCTURAL_ROLES.has(nodes[i].role) - ) { - continue - } - return nodes[i].name - } - } - return undefined -} - -// --------------------------------------------------------------------------- -// When a tag-only-interactive child (e.g. a statictext TextView) sits -// directly under an explicitly-interactive parent (e.g. a clickable -// LinearLayout row), the *parent* should carry the → selector — the -// child is just a label. Suppress the child's interactivity so the -// parent renders as the actionable element. -// --------------------------------------------------------------------------- - -function suppressTagOnlyChildren(nodes: MobileFlatNode[]): void { - for (let i = 0; i < nodes.length; i++) { - const node = nodes[i] - if (!node.isInteractive || node.isExplicitInteractive) { - continue - } - // Walk up through ALL ancestors looking for an explicitly-interactive - // parent. The immediate depth-1 parent may just be a layout wrapper; - // the real clickable row could be 2-3 levels up. - for (let j = i - 1; j >= 0; j--) { - if (nodes[j].depth < node.depth) { - if (nodes[j].isExplicitInteractive) { - node.isInteractive = false - break // found — suppress and stop - } - // keep looking upward through the ancestor chain - } - } - } -} - -// --------------------------------------------------------------------------- -// Render pass: flat nodes into lines with ∈ context, dedup, noise filter, -// and class-instance indexing. -// --------------------------------------------------------------------------- - -/** Layout roles that carry no semantic meaning by themselves. */ -const NOISY_ROLES = new Set([ - 'FrameLayout', - 'LinearLayout', - 'ViewGroup', - 'RelativeLayout', - 'View', - 'CardView', - 'ConstraintLayout', - 'ScrollView' -]) - -/** - * Pre-count selector occurrences so we can attach .instance(N) suffixes - * to duplicate selectors. - */ -function countSelectors(nodes: MobileFlatNode[]): Map { - const counts = new Map() - for (const node of nodes) { - if (node.selector) { - counts.set(node.selector, (counts.get(node.selector) ?? 0) + 1) - } - } - return counts -} - -// Render pass mirrors serializeWebSnapshot's structure (linear node-by-node -// emission with parent-context lookback, statictext echo dedup, layout-noise -// collapse, .instance(N) indexing). Splitting per-decision would lose the -// shared per-node state. ROADMAP P2: collapse with web pipeline. -// eslint-disable-next-line max-lines-per-function -function renderMobileNodes(nodes: MobileFlatNode[]): string[] { - const lines: string[] = [] - const selectorCounts = countSelectors(nodes) - const selectorIndex = new Map() - - for (let i = 0; i < nodes.length; i++) { - const node = nodes[i] - const indent = ' '.repeat(node.depth + 1) - - // Collapse anonymous layout containers at depth ≥ 2. - // Keep depth 0-1 structural chrome and any named container. - if ( - NOISY_ROLES.has(node.role) && - !node.name && - node.depth > 1 && - !node.isInteractive - ) { - continue - } - - // Off-screen containers rendered as collapsed placedersen - if (node.isInViewport === false && !node.isInteractive) { - lines.push(`${indent}⋯ ${node.name} (off-screen)`) - continue - } - - // Dedup: skip statictext whose text is echoed by the parent interactive element - if (node.role === 'statictext' && node.name) { - let echoedByParent = false - for (let j = i - 1; j >= 0; j--) { - if (nodes[j].depth < node.depth) { - if ( - nodes[j].isInteractive && - nodes[j].name && - nodes[j].name.includes(node.name) - ) { - echoedByParent = true - } - break - } - } - if (echoedByParent) { - continue - } - } - - if (node.isInteractive && node.selector) { - // Append .instance(N) when the same selector repeats - let selector = node.selector - const total = selectorCounts.get(selector) ?? 1 - if (total > 1) { - const idx = selectorIndex.get(selector) ?? 0 - selectorIndex.set(selector, idx + 1) - selector = `${selector}.instance(${idx})` - } - - const purpose = mobileInferPurpose(nodes, i) - if (node.name) { - lines.push( - purpose - ? `${indent}${node.role} "${node.name}" ∈ "${purpose}" → ${selector}` - : `${indent}${node.role} "${node.name}" → ${selector}` - ) - } else if (purpose) { - lines.push(`${indent}${node.role} ∈ "${purpose}" → ${selector}`) - } else { - lines.push(`${indent}${node.role} → ${selector}`) - } - } else { - // Container / structural / non-locatable - lines.push( - node.name - ? `${indent}${node.role} "${node.name}"` - : `${indent}${node.role}` - ) - } - } - - return lines -} - -// --------------------------------------------------------------------------- -// Public API -// --------------------------------------------------------------------------- - -export interface MobileSnapshotOptions { - /** Only include elements whose bounds intersect the viewport (default true). */ - inViewportOnly?: boolean - /** - * Raw XML page source string. When provided the full locator pipeline - * (getSuggestedLocators) runs on every interactive node, producing the same - * selectors that getElements() returns. Omit to use simplified heuristics. - */ - sourceXML?: string -} - -/** - * Serialize a mobile element tree into a depth-indented text snapshot. - * - * @param root Root JSONElement from the page source XML parse - * @param context Platform, optional device name, viewport, and source XML. - * Include `sourceXML` to use the full locator pipeline. - * @param options {@link MobileSnapshotOptions} - */ -export function serializeMobileSnapshot( - root: JSONElement, - context: { - platform: 'android' | 'ios' - deviceName?: string - viewport?: { width: number; height: number } - /** Raw page-source XML. When set, selectors match getElements() output. */ - sourceXML?: string - }, - options: MobileSnapshotOptions = {} -): string { - const { platform, deviceName, viewport, sourceXML } = context - const { inViewportOnly = true } = options - - // Auto-detect source XML stashed by getMobileVisibleElementsWithTree - const effectiveXML = sourceXML || root.attributes._sourceXML - - const effectiveViewport = viewport ?? { width: 9999, height: 9999 } - const automationName = platform === 'android' ? 'uiautomator2' : 'xcuitest' - - let header = `[${platform}` - if (deviceName) { - header += ` — ${deviceName}` - } - if (viewport) { - header += ` (${viewport.width}×${viewport.height})` - } - header += ']' - - const nodes: MobileFlatNode[] = [] - collectMobileNodes(root, platform, 0, nodes, { - inViewportOnly, - viewport: effectiveViewport, - sourceXML: effectiveXML, - automationName: effectiveXML ? automationName : undefined - }) - - // Let explicitly-interactive parents carry the → selector - suppressTagOnlyChildren(nodes) - - const lines = renderMobileNodes(nodes) - return [header, ...lines].join('\n') -} +export { + serializeWebSnapshot, + serializeMobileSnapshot +} from '@wdio/devtools-core/element-snapshot' +export type { + WebSnapshotOptions, + MobileSnapshotOptions +} from '@wdio/devtools-core/element-snapshot' diff --git a/packages/nightwatch-devtools/package.json b/packages/nightwatch-devtools/package.json index 21f4f17a..c8ad4f92 100644 --- a/packages/nightwatch-devtools/package.json +++ b/packages/nightwatch-devtools/package.json @@ -43,7 +43,6 @@ "dependencies": { "@wdio/devtools-backend": "workspace:*", "@wdio/devtools-script": "workspace:*", - "@wdio/elements": "workspace:^", "@wdio/logger": "^9.18.0", "fluent-ffmpeg": "^2.1.3", "import-meta-resolve": "^4.2.0", diff --git a/packages/nightwatch-devtools/src/action-snapshot.ts b/packages/nightwatch-devtools/src/action-snapshot.ts index 4ca5eee2..6d1fc8c1 100644 --- a/packages/nightwatch-devtools/src/action-snapshot.ts +++ b/packages/nightwatch-devtools/src/action-snapshot.ts @@ -1,14 +1,6 @@ -// Per-action snapshot capture for Nightwatch — fires only in `mode: 'trace'` -// for commands in ACTION_MAP. Wraps NightwatchBrowser in a minimal -// WebdriverIO.Browser-shaped shim so @wdio/elements can run its in-page -// scripts. Returns null on failure; capture errors must not break the test. +// Nightwatch adapter: wires NightwatchBrowser into core's captureActionSnapshot. -import { - getBrowserAccessibilityTree, - getInteractableBrowserElements, - serializeWebSnapshot -} from '@wdio/elements' -import { SNAPSHOT_PROBE_TIMEOUT_MS, withTimeout } from '@wdio/devtools-core' +import { captureActionSnapshot as coreCapture } from '@wdio/devtools-core' import type { ActionSnapshot } from '@wdio/devtools-shared' import type { NightwatchBrowser } from './types.js' @@ -17,60 +9,23 @@ interface BrowserWithUrl extends NightwatchBrowser { getTitle?: () => Promise } -function shimAsWdioBrowser(browser: NightwatchBrowser): unknown { - return { - capabilities: browser.capabilities ?? {}, - isAndroid: false, - isIOS: false, - execute: (script: unknown, ...args: unknown[]) => - browser.execute( - script as string, - args.length === 1 && Array.isArray(args[0]) - ? (args[0] as unknown[]) - : args - ) - } -} - -export async function captureActionSnapshot( +export function captureActionSnapshot( browser: NightwatchBrowser, command: string, takeScreenshot?: () => Promise ): Promise { - try { - const timestamp = Date.now() - const b = browser as BrowserWithUrl - const browserLike = shimAsWdioBrowser(browser) as WebdriverIO.Browser - const [shot, url, title, tree, elements] = await Promise.all([ - takeScreenshot?.().catch(() => null) ?? Promise.resolve(null), - b.getCurrentUrl?.().catch(() => undefined) ?? Promise.resolve(undefined), - b.getTitle?.().catch(() => undefined) ?? Promise.resolve(undefined), - withTimeout( - getBrowserAccessibilityTree(browserLike, { - inViewportOnly: true - }).catch(() => []), - SNAPSHOT_PROBE_TIMEOUT_MS, - [] - ), - withTimeout( - getInteractableBrowserElements(browserLike, { - inViewportOnly: true - }).catch(() => []), - SNAPSHOT_PROBE_TIMEOUT_MS, - [] - ) - ]) - const snapshotText = serializeWebSnapshot(tree, { url, title }) - return { - timestamp, - command, - url, - title, - screenshot: shot ?? undefined, - elements, - snapshotText - } - } catch { - return null - } + const b = browser as BrowserWithUrl + return coreCapture({ + command, + runScript: (src) => browser.execute(`return (${src})`) as Promise, + takeScreenshot, + getUrl: () => + b.getCurrentUrl + ? b.getCurrentUrl().catch(() => undefined) + : Promise.resolve(undefined), + getTitle: () => + b.getTitle + ? b.getTitle().catch(() => undefined) + : Promise.resolve(undefined) + }) } diff --git a/packages/selenium-devtools/package.json b/packages/selenium-devtools/package.json index 93ec7223..869c25d8 100644 --- a/packages/selenium-devtools/package.json +++ b/packages/selenium-devtools/package.json @@ -44,7 +44,6 @@ "dependencies": { "@wdio/devtools-backend": "workspace:*", "@wdio/devtools-script": "workspace:*", - "@wdio/elements": "workspace:^", "@wdio/logger": "^9.18.0", "stacktrace-parser": "^0.1.11", "webdriverio": "^9.27.2", diff --git a/packages/selenium-devtools/src/action-snapshot.ts b/packages/selenium-devtools/src/action-snapshot.ts index 1bfa83b9..7644ae1d 100644 --- a/packages/selenium-devtools/src/action-snapshot.ts +++ b/packages/selenium-devtools/src/action-snapshot.ts @@ -1,15 +1,6 @@ -// Per-action snapshot capture for Selenium — fires only in `mode: 'trace'` -// for commands in ACTION_MAP. Wraps the SeleniumDriverLike in a minimal -// WebdriverIO.Browser-shaped shim so @wdio/elements can run its in-page -// scripts via driver.executeScript. Returns null on failure; capture errors -// must not break the user's test. +// Selenium adapter: wires SeleniumDriverLike into core's captureActionSnapshot. -import { - getBrowserAccessibilityTree, - getInteractableBrowserElements, - serializeWebSnapshot -} from '@wdio/elements' -import { SNAPSHOT_PROBE_TIMEOUT_MS, withTimeout } from '@wdio/devtools-core' +import { captureActionSnapshot as coreCapture } from '@wdio/devtools-core' import type { ActionSnapshot } from '@wdio/devtools-shared' import type { SeleniumDriverLike } from './types.js' @@ -18,54 +9,25 @@ interface DriverWithUrl extends SeleniumDriverLike { getTitle?: () => Promise } -function shimAsWdioBrowser(driver: SeleniumDriverLike): unknown { - return { - capabilities: {}, - isAndroid: false, - isIOS: false, - execute: (script: unknown, ...args: unknown[]) => - driver.executeScript(script as string, ...args) - } -} - -export async function captureActionSnapshot( +export function captureActionSnapshot( driver: SeleniumDriverLike, command: string ): Promise { - try { - const timestamp = Date.now() - const d = driver as DriverWithUrl - const browserLike = shimAsWdioBrowser(driver) as WebdriverIO.Browser - const [screenshot, url, title, tree, elements] = await Promise.all([ - d.takeScreenshot?.().catch(() => undefined) ?? Promise.resolve(undefined), - d.getCurrentUrl?.().catch(() => undefined) ?? Promise.resolve(undefined), - d.getTitle?.().catch(() => undefined) ?? Promise.resolve(undefined), - withTimeout( - getBrowserAccessibilityTree(browserLike, { - inViewportOnly: true - }).catch(() => []), - SNAPSHOT_PROBE_TIMEOUT_MS, - [] - ), - withTimeout( - getInteractableBrowserElements(browserLike, { - inViewportOnly: true - }).catch(() => []), - SNAPSHOT_PROBE_TIMEOUT_MS, - [] - ) - ]) - const snapshotText = serializeWebSnapshot(tree, { url, title }) - return { - timestamp, - command, - url, - title, - screenshot, - elements, - snapshotText - } - } catch { - return null - } + const d = driver as DriverWithUrl + return coreCapture({ + command, + runScript: (src) => driver.executeScript(`return (${src})`), + takeScreenshot: () => + d.takeScreenshot + ? d.takeScreenshot().catch(() => undefined) + : Promise.resolve(undefined), + getUrl: () => + d.getCurrentUrl + ? d.getCurrentUrl().catch(() => undefined) + : Promise.resolve(undefined), + getTitle: () => + d.getTitle + ? d.getTitle().catch(() => undefined) + : Promise.resolve(undefined) + }) } diff --git a/packages/service/src/action-snapshot.ts b/packages/service/src/action-snapshot.ts index eef71da5..6512f33a 100644 --- a/packages/service/src/action-snapshot.ts +++ b/packages/service/src/action-snapshot.ts @@ -1,90 +1,32 @@ -// Per-action snapshot capture — fires only in `mode: 'trace'` for commands -// in the action allow-list (see @wdio/devtools-core/action-mapping). Returns -// null on failure; snapshot errors must not break the user's test. +// WDIO adapter: wires WebdriverIO.Browser into core's captureActionSnapshot. +// `browser.execute` is passed a Function reconstructed from the script body +// string (the same trick @wdio/elements uses); a raw string would route +// through a different WDIO path that doesn't preserve the script closure. +// +// `src` is NOT user-controlled: it's one of two compile-time constants +// produced by `@wdio/devtools-core/element-scripts` and shipped with the +// library. No external input reaches new Function() — the lint flag here is +// a false positive given the closed input set. -import { - getBrowserAccessibilityTree, - getElements, - serializeMobileSnapshot, - serializeWebSnapshot -} from '@wdio/elements' -import { SNAPSHOT_PROBE_TIMEOUT_MS, withTimeout } from '@wdio/devtools-core' +import { captureActionSnapshot as coreCapture } from '@wdio/devtools-core' import type { ActionSnapshot } from '@wdio/devtools-shared' -type ElementsResult = Awaited> -const EMPTY_ELEMENTS: ElementsResult = { - total: 0, - showing: 0, - hasMore: false, - elements: [] +function reviveScript(src: string): () => unknown { + // `src` from core/element-scripts is already a self-invoking IIFE + // (`(function () { ... })()`); we just wrap it in a return so it's + // a function browser.execute() can call. + return new Function(`return (${src})`) as () => unknown } -function probeElements(browser: WebdriverIO.Browser): Promise { - return withTimeout( - getElements(browser, { inViewportOnly: true }), - SNAPSHOT_PROBE_TIMEOUT_MS, - EMPTY_ELEMENTS - ) -} - -async function captureMobile( - browser: WebdriverIO.Browser -): Promise<{ elements: unknown[]; snapshotText?: string }> { - const result = await probeElements(browser) - if (!result.tree) { - return { elements: result.elements } - } - const platform = browser.isAndroid ? 'android' : 'ios' - return { - elements: result.elements, - snapshotText: serializeMobileSnapshot(result.tree, { platform }) - } -} - -async function captureWeb( - browser: WebdriverIO.Browser, - url: string | undefined, - title: string | undefined -): Promise<{ elements: unknown[]; snapshotText?: string }> { - const [tree, flat] = await Promise.all([ - withTimeout( - getBrowserAccessibilityTree(browser, { inViewportOnly: true }), - SNAPSHOT_PROBE_TIMEOUT_MS, - [] - ), - probeElements(browser) - ]) - return { - elements: flat.elements, - snapshotText: serializeWebSnapshot(tree, { url, title }) - } -} - -export async function captureActionSnapshot( +export function captureActionSnapshot( browser: WebdriverIO.Browser, command: string ): Promise { - try { - const timestamp = Date.now() - const isMobile = !!(browser.isAndroid || browser.isIOS) - const [screenshot, url, title] = await Promise.all([ - browser.takeScreenshot().catch(() => undefined), - browser.getUrl().catch(() => undefined), - browser.getTitle().catch(() => undefined) - ]) - const { elements, snapshotText } = isMobile - ? await captureMobile(browser) - : await captureWeb(browser, url, title) - return { - timestamp, - command, - url, - title, - screenshot, - elements, - snapshotText - } - } catch { - return null - } + return coreCapture({ + command, + runScript: (src) => browser.execute(reviveScript(src)), + takeScreenshot: () => browser.takeScreenshot().catch(() => undefined), + getUrl: () => browser.getUrl().catch(() => undefined), + getTitle: () => browser.getTitle().catch(() => undefined) + }) } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cd5c1104..9d5b6ce8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -308,31 +308,34 @@ importers: '@wdio/devtools-shared': specifier: workspace:^ version: link:../shared + '@xmldom/xmldom': + specifier: ^0.9.8 + version: 0.9.10 stacktrace-parser: specifier: ^0.1.11 version: 0.1.11 ws: specifier: ^8.21.0 version: 8.21.0 + xpath: + specifier: ^0.0.34 + version: 0.0.34 yazl: specifier: ^2.5.1 version: 2.5.1 packages/elements: dependencies: - '@xmldom/xmldom': - specifier: ^0.9.8 - version: 0.9.10 webdriverio: specifier: ^9.0.0 version: 9.27.2(puppeteer-core@21.11.0) - xpath: - specifier: ^0.0.34 - version: 0.0.34 devDependencies: '@types/node': specifier: 25.9.1 version: 25.9.1 + '@wdio/devtools-core': + specifier: workspace:^ + version: link:../core '@wdio/globals': specifier: 9.27.0 version: 9.27.0(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0)) @@ -351,9 +354,6 @@ importers: '@wdio/devtools-script': specifier: workspace:* version: link:../script - '@wdio/elements': - specifier: workspace:^ - version: link:../elements '@wdio/logger': specifier: ^9.18.0 version: 9.18.0 @@ -431,9 +431,6 @@ importers: '@wdio/devtools-script': specifier: workspace:* version: link:../script - '@wdio/elements': - specifier: workspace:^ - version: link:../elements '@wdio/logger': specifier: ^9.18.0 version: 9.18.0 From 72a88949f27476ab35effa7a49afeb24cdee10fe Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Tue, 9 Jun 2026 18:40:20 +0530 Subject: [PATCH 11/13] core: Trace.zip viewer compat, traceFormat option, mobile example, Selenium race fix; --- README.md | 19 +++++ examples/wdio/package.json | 3 +- examples/wdio/wdio.mobile.conf.ts | 79 +++++++++++++++++++ packages/core/src/trace-exporter.ts | 63 ++++++++++++--- packages/nightwatch-devtools/src/index.ts | 12 +-- packages/nightwatch-devtools/src/types.ts | 10 ++- packages/selenium-devtools/src/index.ts | 3 +- .../selenium-devtools/src/plugin-internals.ts | 4 +- .../src/session-lifecycle.ts | 52 +++++++----- packages/selenium-devtools/src/session.ts | 13 ++- packages/selenium-devtools/src/types.ts | 12 ++- packages/service/src/index.ts | 7 +- packages/service/src/types.ts | 12 ++- packages/shared/src/types.ts | 6 ++ 14 files changed, 241 insertions(+), 54 deletions(-) create mode 100644 examples/wdio/wdio.mobile.conf.ts diff --git a/README.md b/README.md index a68f3e83..3a61908a 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,25 @@ What counts as a user-facing action is filtered through an allow-list in `@wdio/ Trace mode and live mode are **mutually exclusive** — `screencast` options are ignored in trace mode (live-mode feature). Live and trace serve different audiences (humans debugging vs. agents diffing), and stacking them only costs perf. +#### Output layout — `traceFormat` + +`{ mode: 'trace', traceFormat: 'zip' | 'ndjson-directory' }`. Default `'zip'` writes a single `trace-.zip`; `'ndjson-directory'` unpacks the same `trace.trace` + `trace.network` + `resources/` files into a `trace-/` folder. Both render in `npx playwright show-trace `. The unpacked form skips the unzip step for scripted / agentic consumers. + +#### 📱 Mobile testing + +Adapters detect mobile sessions via `platformName: 'android' | 'ios'` (and Appium `bstack:options`) and adjust the per-action snapshot to extract elements from the mobile XML tree instead of the DOM. The trace's `context-options` records `title: 'android'` / `'ios'` so the viewer labels frames correctly. + +A reference WDIO config is at [examples/wdio/wdio.mobile.conf.ts](examples/wdio/wdio.mobile.conf.ts). Prereqs to run it end-to-end with a local emulator: + +1. **Java JDK** — `brew install --cask temurin` +2. **Android SDK** — `brew install --cask android-commandlinetools` then `yes | sdkmanager --licenses && sdkmanager "platform-tools" "emulator" "system-images;android-34;google_apis_playstore;arm64-v8a"`. The brew cask installs sdkmanager under `/opt/homebrew/share/android-commandlinetools/`, and sdkmanager downloads other SDK pieces alongside it — set `ANDROID_HOME` to that path (not `~/Library/Android/sdk/`). +3. **AVD + emulator** — `avdmanager create avd -n devtools-test -k "system-images;android-34;google_apis_playstore;arm64-v8a" -d "pixel_7"`, then `emulator -avd devtools-test &` + `adb wait-for-device`. +4. **Appium + UiAutomator2 driver** — `sudo npm i -g appium && appium driver install uiautomator2`. +5. **Chromedriver pinning** — Appium's autodownload doesn't reach back far enough for the Chrome version that ships with most Android system images (e.g. Chrome 113 on Android 14). Manually download the matching Chromedriver and start Appium with `--default-capabilities '{"appium:chromedriverExecutableDir": ""}'` plus `--allow-insecure=uiautomator2:chromedriver_autodownload`. +6. **Classic WebDriver protocol** — Appium 3's BiDi shim for UiAutomator2 doesn't implement every BiDi command (e.g. `script.addPreloadScript`). Set `'wdio:enforceWebDriverClassic': true` in the capability block so WDIO doesn't attempt the BiDi handshake. + +These are emulator-specific issues; on a physical phone with USB debugging only steps 1, 4, 6 (and the Chromedriver pin if Chrome on the device is old) apply. + ### 🔍︎ TestLens - **Code Intelligence**: View test definitions directly in your editor - **Run/Debug Actions**: Execute individual tests or suites with inline CodeLens actions diff --git a/examples/wdio/package.json b/examples/wdio/package.json index d3cc2104..a3c3644c 100644 --- a/examples/wdio/package.json +++ b/examples/wdio/package.json @@ -16,6 +16,7 @@ "typescript": "^6.0.3" }, "scripts": { - "wdio": "wdio run ./wdio.conf.ts" + "wdio": "wdio run ./wdio.conf.ts", + "mobile": "wdio run ./wdio.mobile.conf.ts" } } diff --git a/examples/wdio/wdio.mobile.conf.ts b/examples/wdio/wdio.mobile.conf.ts new file mode 100644 index 00000000..7f693ab4 --- /dev/null +++ b/examples/wdio/wdio.mobile.conf.ts @@ -0,0 +1,79 @@ +// Mobile-web (Android Chrome via Appium) variant of wdio.conf.ts. +// +// Prerequisites: +// 1. Appium 2.x running locally: `appium --address 127.0.0.1 --port 4723` +// 2. UiAutomator2 driver installed: `appium driver install uiautomator2` +// 3. An Android emulator running with Chrome installed +// (or a real device with USB debugging on and `adb devices` listing it). +// +// Run (from inside examples/wdio): +// pnpm mobile +// +// The DevTools service detects `platformName: Android|iOS` via shared +// capabilities and adjusts the action-snapshot probe (mobile XML element +// extraction) and the trace's context naming accordingly. + +import path from 'node:path' +import type { Options } from '@wdio/types' + +const __dirname = path.resolve(path.dirname(new URL(import.meta.url).pathname)) + +export const config: Options.Testrunner = { + runner: 'local', + + specs: ['./features/**/*.feature'], + exclude: [], + + hostname: '127.0.0.1', + port: 4723, + path: '/', + + maxInstances: 1, + // `wdio:enforceWebDriverClassic` isn't in @wdio/types yet but is honored + // at runtime — needed because Appium's BiDi shim for UiAutomator2 doesn't + // implement every BiDi command (e.g. script.addPreloadScript). + capabilities: [ + { + platformName: 'Android', + 'appium:automationName': 'UiAutomator2', + 'appium:deviceName': 'emulator-5554', + browserName: 'Chrome', + 'appium:chromedriverAutodownload': true, + 'wdio:enforceWebDriverClassic': true + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ] as any, + + logLevel: 'info', + bail: 0, + baseUrl: 'http://localhost', + waitforTimeout: 15000, + connectionRetryTimeout: 120000, + connectionRetryCount: 3, + services: [ + [ + 'devtools', + { + mode: 'trace' as const, + screencast: { enabled: true, pollIntervalMs: 250 } + } + ] + ], + framework: 'cucumber', + reporters: ['spec'], + cucumberOpts: { + require: [ + path.resolve(__dirname, 'features', 'step-definitions', 'steps.ts') + ], + backtrace: false, + requireModule: [], + dryRun: false, + failFast: false, + snippets: true, + source: true, + strict: false, + tagExpression: '', + timeout: 90000, + ignoreUndefinedDefinitions: false + } +} diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts index 194a8c31..27374a58 100644 --- a/packages/core/src/trace-exporter.ts +++ b/packages/core/src/trace-exporter.ts @@ -9,6 +9,7 @@ import type { ConsoleLog, Metadata, NetworkRequest, + TraceFormat, TraceLog, TraceMutation } from '@wdio/devtools-shared' @@ -279,10 +280,16 @@ function compareEvents(a: TraceEvent, b: TraceEvent): number { return dt !== 0 ? dt : eventOrder(a) - eventOrder(b) } -export async function exportTraceZip( +interface TraceBundle { + traceNdjson: string + networkNdjson: Buffer + resources: TraceZipResource[] +} + +function buildTraceBundle( trace: TraceLog, opts: { sessionId?: string; wallTimeOverride?: number } = {} -): Promise { +): TraceBundle { // wallTime anchors monotonic offsets at the first captured command so // subsequent actions render at positive deltas in the trace viewer. const firstCommandTs = trace.commands[0]?.timestamp @@ -297,10 +304,39 @@ export async function exportTraceZip( ...buildScreencastFrames(snapshots, pageId, wallTime, viewport), ...buildActionEvents(trace.commands, pageId, wallTime) ].sort(compareEvents) - const traceNdjson = events.map((e) => JSON.stringify(e)).join('\n') - const networkNdjson = buildNetworkNdjson(trace.networkRequests) - const resources = buildSnapshotResources(snapshots, pageId) - return buildTraceZip({ traceNdjson, networkNdjson, resources }) + return { + traceNdjson: events.map((e) => JSON.stringify(e)).join('\n'), + networkNdjson: buildNetworkNdjson(trace.networkRequests), + resources: buildSnapshotResources(snapshots, pageId) + } +} + +export async function exportTraceZip( + trace: TraceLog, + opts: { sessionId?: string; wallTimeOverride?: number } = {} +): Promise { + return buildTraceZip(buildTraceBundle(trace, opts)) +} + +async function exportTraceDirectory( + trace: TraceLog, + targetDir: string, + opts: { sessionId?: string; wallTimeOverride?: number } = {} +): Promise { + const bundle = buildTraceBundle(trace, opts) + await fs.mkdir(path.join(targetDir, 'resources'), { recursive: true }) + await Promise.all([ + fs.writeFile(path.join(targetDir, 'trace.trace'), bundle.traceNdjson), + bundle.networkNdjson.length + ? fs.writeFile( + path.join(targetDir, 'trace.network'), + bundle.networkNdjson + ) + : Promise.resolve(), + ...bundle.resources.map((r) => + fs.writeFile(path.join(targetDir, 'resources', r.resourceName), r.data) + ) + ]) } /** Minimum capturer surface needed to assemble a TraceLog. */ @@ -324,11 +360,14 @@ export interface WriteTraceZipOptions { * viewer still renders thumbnails for adapters without an action hook. */ actionSnapshots?: ActionSnapshot[] + /** Output layout — `zip` (default) writes a single archive, `directory` + * unpacks the same files into `trace-/`. */ + format?: TraceFormat } /** - * Build a TraceLog from a SessionCapturer-shaped source and write a - * Trace.zip. Returns the absolute path written. + * Build a TraceLog from a SessionCapturer-shaped source and write the trace + * artifact (zip file or directory). Returns the absolute path written. */ export async function writeTraceZip( capturer: TraceCapturer, @@ -353,8 +392,14 @@ export async function writeTraceZip( sources: Object.fromEntries(capturer.sources), ...(actionSnapshots.length ? { actionSnapshots } : {}) } - const zip = await exportTraceZip(traceLog, { sessionId: opts.sessionId }) await fs.mkdir(opts.outputDir, { recursive: true }) + if (opts.format === 'ndjson-directory') { + const dir = path.join(opts.outputDir, `trace-${opts.sessionId}`) + await fs.mkdir(dir, { recursive: true }) + await exportTraceDirectory(traceLog, dir, { sessionId: opts.sessionId }) + return dir + } + const zip = await exportTraceZip(traceLog, { sessionId: opts.sessionId }) const zipPath = path.join(opts.outputDir, `trace-${opts.sessionId}.zip`) await fs.writeFile(zipPath, zip) return zipPath diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index d8033aee..4c260678 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -114,7 +114,8 @@ class NightwatchDevToolsPlugin { hostname: options.hostname ?? 'localhost', screencast, bidi: options.bidi ?? false, - mode + mode, + traceFormat: options.traceFormat ?? 'zip' } this.#screencastOptions = { ...SCREENCAST_DEFAULTS, ...screencast } this.#bidiEnabled = options.bidi === true @@ -495,16 +496,17 @@ class NightwatchDevToolsPlugin { await Promise.allSettled(this.sessionCapturer.snapshotCaptures) } const snapshots = this.sessionCapturer.actionSnapshots - const zipPath = await writeTraceZip(this.sessionCapturer, { + const tracePath = await writeTraceZip(this.sessionCapturer, { outputDir: resolveAdapterOutputDir({ configPath: this.#configPath }), sessionId, - actionSnapshots: snapshots.length ? snapshots : undefined + actionSnapshots: snapshots.length ? snapshots : undefined, + format: this.options.traceFormat }) - log.info(`Trace.zip saved to ${zipPath}`) + log.info(`Trace saved to ${tracePath}`) } catch (err) { - log.warn(`trace.zip write failed: ${errorMessage(err)}`) + log.warn(`trace write failed: ${errorMessage(err)}`) } } diff --git a/packages/nightwatch-devtools/src/types.ts b/packages/nightwatch-devtools/src/types.ts index cb519808..c498b40d 100644 --- a/packages/nightwatch-devtools/src/types.ts +++ b/packages/nightwatch-devtools/src/types.ts @@ -16,10 +16,15 @@ export { type SuiteStats, type TestStats, type TestStatus, + type TraceFormat, type TraceLog } from '@wdio/devtools-shared' -import type { DevToolsMode, ScreencastOptions } from '@wdio/devtools-shared' +import type { + DevToolsMode, + ScreencastOptions, + TraceFormat +} from '@wdio/devtools-shared' export interface CommandStackFrame { command: string @@ -97,6 +102,9 @@ export interface DevToolsOptions { bidi?: boolean /** `live` (default) launches the DevTools UI; `trace` skips it. */ mode?: DevToolsMode + /** Trace output layout — `zip` (default) writes a single archive, + * `ndjson-directory` unpacks into `trace-/`. Only applies in trace mode. */ + traceFormat?: TraceFormat } export interface NightwatchBrowser { diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index 4a673549..127d69b4 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -123,7 +123,8 @@ class SeleniumDevToolsPlugin { captureScreenshots: options.captureScreenshots ?? true, rerunCommand: options.rerunCommand, headless: options.headless ?? false, - mode: options.mode ?? 'live' + mode: options.mode ?? 'live', + traceFormat: options.traceFormat ?? 'zip' } this.#rerunManager = new RerunManager(RUNNER) if (options.rerunCommand) { diff --git a/packages/selenium-devtools/src/plugin-internals.ts b/packages/selenium-devtools/src/plugin-internals.ts index 6bd73e99..3c742fdf 100644 --- a/packages/selenium-devtools/src/plugin-internals.ts +++ b/packages/selenium-devtools/src/plugin-internals.ts @@ -16,7 +16,8 @@ import type { ActionSnapshot, DevToolsMode, ScreencastOptions, - SeleniumDriverLike + SeleniumDriverLike, + TraceFormat } from './types.js' import type { RetryTracker } from '@wdio/devtools-core' import type { PendingTestAction, PendingScenario } from './test-management.js' @@ -30,6 +31,7 @@ export interface PluginInternals { captureScreenshots: boolean rerunCommand?: string mode?: DevToolsMode + traceFormat?: TraceFormat } readonly screencastOptions: ScreencastOptions readonly runner: string diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index 13934512..87059f54 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -28,7 +28,8 @@ import type { DevToolsMode, Metadata, ScreencastOptions, - SeleniumDriverLike + SeleniumDriverLike, + TraceFormat } from './types.js' import type { TestManager } from './helpers/testManager.js' @@ -42,6 +43,7 @@ export interface SessionLifecycleCtx { captureScreenshots: boolean rerunCommand?: string mode?: DevToolsMode + traceFormat?: TraceFormat } readonly screencastOptions: ScreencastOptions readonly runner: string @@ -230,26 +232,7 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { ctx.testManager?.finalizeSession() ctx.testReporter?.updateSuites() - const sessionId = capturerAtStart?.metadata?.sessionId - if (ctx.options.mode === 'trace' && capturerAtStart && sessionId) { - try { - if (ctx.snapshotCaptures.length) { - await Promise.allSettled(ctx.snapshotCaptures) - } - const zipPath = await writeTraceZip(capturerAtStart, { - outputDir: resolveAdapterOutputDir({ - testFilePath: testFilePathAtStart - }), - sessionId, - actionSnapshots: ctx.actionSnapshots.length - ? ctx.actionSnapshots - : undefined - }) - log.info(`Trace.zip saved to ${zipPath}`) - } catch (err) { - log.warn(`trace.zip write failed: ${errorMessage(err)}`) - } - } + await maybeWriteTrace(ctx, capturerAtStart, testFilePathAtStart) logSessionSummary(ctx) ctx.sessionCapturer?.cleanup() @@ -284,6 +267,33 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { } } +async function maybeWriteTrace( + ctx: SessionLifecycleCtx, + capturer: SessionCapturer | undefined, + testFilePath: string | undefined +): Promise { + const sessionId = capturer?.metadata?.sessionId + if (ctx.options.mode !== 'trace' || !capturer || !sessionId) { + return + } + try { + if (ctx.snapshotCaptures.length) { + await Promise.allSettled(ctx.snapshotCaptures) + } + const tracePath = await writeTraceZip(capturer, { + outputDir: resolveAdapterOutputDir({ testFilePath }), + sessionId, + actionSnapshots: ctx.actionSnapshots.length + ? ctx.actionSnapshots + : undefined, + format: ctx.options.traceFormat + }) + log.info(`Trace saved to ${tracePath}`) + } catch (err) { + log.warn(`trace write failed: ${errorMessage(err)}`) + } +} + function logSessionSummary(ctx: SessionLifecycleCtx): void { const cmdCount = ctx.sessionCapturer?.commandsLog.length ?? 0 const consoleCount = ctx.sessionCapturer?.consoleLogs.length ?? 0 diff --git a/packages/selenium-devtools/src/session.ts b/packages/selenium-devtools/src/session.ts index 2312770e..70cbe98c 100644 --- a/packages/selenium-devtools/src/session.ts +++ b/packages/selenium-devtools/src/session.ts @@ -256,16 +256,13 @@ export class SessionCapturer extends SessionCapturerBase { return } try { - const ready = await exec( - driver, - 'return typeof window.wdioTraceCollector !== "undefined";' - ) - if (ready !== true) { - return - } + // Atomic check+read in one script eval so the collector can't disappear + // (page navigation) between the existence check and the getTraceData + // call. Two round-trips left a TOCTOU race that logged spurious + // "Cannot read properties of undefined (reading 'getTraceData')" errors. const traceData = await exec( driver, - 'return window.wdioTraceCollector.getTraceData();' + 'return typeof window.wdioTraceCollector !== "undefined" ? window.wdioTraceCollector.getTraceData() : null;' ) if (!traceData) { return diff --git a/packages/selenium-devtools/src/types.ts b/packages/selenium-devtools/src/types.ts index dc451c45..dfcd4a0d 100644 --- a/packages/selenium-devtools/src/types.ts +++ b/packages/selenium-devtools/src/types.ts @@ -13,7 +13,8 @@ export { type PerformanceData, type SuiteStats, type TestStats, - type TestStatus + type TestStatus, + type TraceFormat } from '@wdio/devtools-shared' export interface DevToolsOptions { @@ -23,6 +24,9 @@ export interface DevToolsOptions { openUi?: boolean /** `live` (default) launches the DevTools UI; `trace` skips it. Overrides `openUi`. */ mode?: DevToolsMode + /** Trace output layout — `zip` (default) writes a single archive, + * `ndjson-directory` unpacks into `trace-/`. Only applies in trace mode. */ + traceFormat?: TraceFormat /** Capture screenshots after each command. Default true. */ captureScreenshots?: boolean /** Command template for per-test rerun. {{testName}} is substituted. */ @@ -39,7 +43,11 @@ export interface DevToolsOptions { // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing selenium-internal imports. -import type { DevToolsMode, ScreencastOptions } from '@wdio/devtools-shared' +import type { + DevToolsMode, + ScreencastOptions, + TraceFormat +} from '@wdio/devtools-shared' export type { ScreencastFrame, ScreencastOptions } from '@wdio/devtools-shared' /** diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 558f0755..2984500d 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -368,15 +368,16 @@ export default class DevToolsHookService implements Services.ServiceInstance { log.info(`DevTools trace saved to ${traceFilePath}`) if (this.#options.mode === 'trace') { - const zipPath = await writeTraceZip(this.#sessionCapturer, { + const tracePath = await writeTraceZip(this.#sessionCapturer, { outputDir, sessionId: this.#browser.sessionId, capabilities: this.#browser.capabilities, actionSnapshots: this.#actionSnapshots.length ? this.#actionSnapshots - : undefined + : undefined, + format: this.#options.traceFormat }) - log.info(`Trace.zip saved to ${zipPath}`) + log.info(`Trace saved to ${tracePath}`) } // Clean up console patching diff --git a/packages/service/src/types.ts b/packages/service/src/types.ts index b20de413..f10068da 100644 --- a/packages/service/src/types.ts +++ b/packages/service/src/types.ts @@ -23,11 +23,16 @@ export { // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing service-internal imports. -import type { DevToolsMode, ScreencastOptions } from '@wdio/devtools-shared' +import type { + DevToolsMode, + ScreencastOptions, + TraceFormat +} from '@wdio/devtools-shared' export type { DevToolsMode, ScreencastFrame, - ScreencastOptions + ScreencastOptions, + TraceFormat } from '@wdio/devtools-shared' export interface ExtendedCapabilities extends WebdriverIO.Capabilities { @@ -64,6 +69,9 @@ export interface ServiceOptions { screencast?: ScreencastOptions /** `live` (default) launches the DevTools UI; `trace` skips it. */ mode?: DevToolsMode + /** Trace output layout — `zip` (default) writes a single archive, + * `ndjson-directory` unpacks into `trace-/`. Only applies in trace mode. */ + traceFormat?: TraceFormat } declare namespace WebdriverIO { diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index ca7bc2d8..0a4bbe62 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -19,6 +19,12 @@ export type TestStatus = 'passed' | 'failed' | 'skipped' | 'pending' | 'running' /** `live` opens the DevTools UI window; `trace` skips it and lets a downstream exporter consume captured state. */ export type DevToolsMode = 'live' | 'trace' +/** `zip` (default) writes a single `trace-.zip`; `ndjson-directory` writes + * the same `trace.trace` + `trace.network` + `resources/` layout unpacked + * into `trace-/`. Both are consumable by `playwright show-trace` — the + * unpacked form skips the unzip step for agentic / scripted consumers. */ +export type TraceFormat = 'zip' | 'ndjson-directory' + /** * Enum-style accessor for the canonical TestStatus values. Adapter code uses * this for readable comparisons (`state === TEST_STATE.PASSED`). The app's From 47225b35589575457812dc96d6fa9ea934204523 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 10 Jun 2026 16:12:57 +0530 Subject: [PATCH 12/13] Service+Nightwatch withTimeout timer leak --- README.md | 4 ++-- examples/nightwatch/nightwatch.conf.cjs | 2 +- examples/wdio/wdio.conf.ts | 10 +++++++++- packages/core/src/with-timeout.ts | 17 ++++++++--------- packages/nightwatch-devtools/src/session.ts | 14 ++++---------- .../nightwatch-devtools/tests/session.test.ts | 9 +++------ .../selenium-devtools/tests/session.test.ts | 10 ++++------ packages/service/src/session.ts | 17 +++++++++-------- 8 files changed, 40 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 3a61908a..d88f2dfb 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ The zip contains: - `trace.trace` — NDJSON `context-options` + `before`/`after` action events - `trace.network` — HAR-style network entries derived from the existing capture - `resources/page@-.jpeg` — screenshot per user-facing action -- `resources/elements-page@-.json` — flat interactable element list from `@wdio/elements` +- `resources/elements-page@-.json` — flat interactable element list extracted by the page-injected scripts in `@wdio/devtools-core/element-scripts` - `resources/snapshot-page@-.txt` — depth-indented accessibility-tree snapshot (AI-friendly) What counts as a user-facing action is filtered through an allow-list in `@wdio/devtools-core/action-mapping.ts` (`url`, `click`, `setValue`, `sendKeys`, `get`, etc.). Internal commands like `findElement`/`waitUntil`/`executeScript` don't produce trace entries. @@ -95,7 +95,7 @@ Trace mode and live mode are **mutually exclusive** — `screencast` options are #### 📱 Mobile testing -Adapters detect mobile sessions via `platformName: 'android' | 'ios'` (and Appium `bstack:options`) and adjust the per-action snapshot to extract elements from the mobile XML tree instead of the DOM. The trace's `context-options` records `title: 'android'` / `'ios'` so the viewer labels frames correctly. +Adapters detect mobile sessions via `platformName: 'android' | 'ios'` (case-insensitive) and adjust the per-action snapshot to extract elements from the mobile XML tree instead of the DOM. The trace's `context-options` records `title: 'android' — ` / `'ios' — ` so the viewer labels frames correctly. A reference WDIO config is at [examples/wdio/wdio.mobile.conf.ts](examples/wdio/wdio.mobile.conf.ts). Prereqs to run it end-to-end with a local emulator: diff --git a/examples/nightwatch/nightwatch.conf.cjs b/examples/nightwatch/nightwatch.conf.cjs index 619d1a76..7f14af67 100644 --- a/examples/nightwatch/nightwatch.conf.cjs +++ b/examples/nightwatch/nightwatch.conf.cjs @@ -45,7 +45,7 @@ module.exports = { // off to avoid duplicate entries. globals: nightwatchDevtools({ port: 3000, - mode: 'trace', + mode: 'live', screencast: { enabled: true, pollIntervalMs: 200 }, bidi: true }) diff --git a/examples/wdio/wdio.conf.ts b/examples/wdio/wdio.conf.ts index 71830cf2..00abe28a 100644 --- a/examples/wdio/wdio.conf.ts +++ b/examples/wdio/wdio.conf.ts @@ -127,7 +127,15 @@ export const config: Options.Testrunner = { // Services take over a specific job you don't want to take care of. They enhance // your test setup with almost no effort. Unlike plugins, they don't add new // commands. Instead, they hook themselves up into the test process. - services: [['devtools', { mode: 'trace' as const }]], + services: [ + [ + 'devtools', + { + mode: 'live' as const, + screencast: { enabled: true, pollIntervalMs: 200 } + } + ] + ], // // Framework you want to run your specs with. // The following are supported: Mocha, Jasmine, and Cucumber diff --git a/packages/core/src/with-timeout.ts b/packages/core/src/with-timeout.ts index bdd37469..feb840e2 100644 --- a/packages/core/src/with-timeout.ts +++ b/packages/core/src/with-timeout.ts @@ -12,15 +12,14 @@ export function withTimeout( const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve(fallback), ms) }) - return Promise.race([ - promise.then((v) => { - if (timer) { - clearTimeout(timer) - } - return v - }), - timeout - ]) + // .finally on the race result clears the timer in BOTH the resolved AND + // rejected branch — earlier code only cleared on fulfillment, leaving a + // dangling timer per failed probe that delayed process exit by up to `ms`. + return Promise.race([promise, timeout]).finally(() => { + if (timer) { + clearTimeout(timer) + } + }) } /** Default ceiling for a single in-page snapshot probe. */ diff --git a/packages/nightwatch-devtools/src/session.ts b/packages/nightwatch-devtools/src/session.ts index 87b3cde5..7fc70e42 100644 --- a/packages/nightwatch-devtools/src/session.ts +++ b/packages/nightwatch-devtools/src/session.ts @@ -456,17 +456,11 @@ export class SessionCapturer extends SessionCapturerBase { // Capture network requests from Chrome performance logs await this.captureNetworkFromPerformanceLogs(browser) - // Also try the injected wdioTraceCollector script for XHR/fetch and mutations + // Also try the injected wdioTraceCollector script for XHR/fetch and mutations. + // Atomic check+read — the inline `typeof === 'undefined' → null` guard is + // the only safe form; a separate existence check would race page navigation + // (the collector can disappear between the two round-trips). try { - const checkResult = await browser.execute( - 'return typeof window.wdioTraceCollector !== "undefined"' - ) - const collectorExists = unwrapDriverValue(checkResult) === true - - if (!collectorExists) { - return - } - const result = await browser.execute(` if (typeof window.wdioTraceCollector === 'undefined') { return null; diff --git a/packages/nightwatch-devtools/tests/session.test.ts b/packages/nightwatch-devtools/tests/session.test.ts index 4f00c948..1347ee57 100644 --- a/packages/nightwatch-devtools/tests/session.test.ts +++ b/packages/nightwatch-devtools/tests/session.test.ts @@ -314,11 +314,8 @@ describe('SessionCapturer.captureTrace', () => { getLog: vi.fn(async () => []), execute: vi.fn(async () => { call++ - if (call === 1) { - // collector check - return { value: true } - } - // getTraceData + // Single atomic check+read — the inline `typeof === 'undefined' → null` + // guard in the script body avoids the navigation TOCTOU race. return { value: { mutations: [ @@ -332,6 +329,6 @@ describe('SessionCapturer.captureTrace', () => { }) const cap = makeCapturer(browser) await cap.captureTrace(browser) - expect(call).toBe(2) + expect(call).toBe(1) }) }) diff --git a/packages/selenium-devtools/tests/session.test.ts b/packages/selenium-devtools/tests/session.test.ts index a780adf4..785b5d3b 100644 --- a/packages/selenium-devtools/tests/session.test.ts +++ b/packages/selenium-devtools/tests/session.test.ts @@ -221,20 +221,18 @@ describe('selenium SessionCapturer (with stashed executeScript)', () => { let call = 0 stubExec(async () => { call++ - return call === 1 ? true : null + return null }) const cap = makeCapturer({ id: 'd' }) await expect(cap.captureTrace()).resolves.toBeUndefined() - expect(call).toBe(2) + // Single atomic check+read in one executeScript — see session.ts comment. + expect(call).toBe(1) }) it('captureTrace processes payload when collector returns data', async () => { let call = 0 stubExec(async () => { call++ - if (call === 1) { - return true - } return { mutations: [], networkRequests: [], @@ -243,7 +241,7 @@ describe('selenium SessionCapturer (with stashed executeScript)', () => { }) const cap = makeCapturer({ id: 'd' }) await cap.captureTrace() - expect(call).toBe(2) + expect(call).toBe(1) }) it('captureTrace swallows ECONNREFUSED / no-such-session errors silently', async () => { diff --git a/packages/service/src/session.ts b/packages/service/src/session.ts index fcd6b508..97026437 100644 --- a/packages/service/src/session.ts +++ b/packages/service/src/session.ts @@ -207,20 +207,21 @@ export class SessionCapturer extends SessionCapturerBase { } try { - const collectorExists = await browser.execute( - () => typeof window.wdioTraceCollector !== 'undefined' + // Atomic check+read in a single browser.execute so the collector can't + // disappear (page navigation) between the existence check and the + // getTraceData call. Two round-trips left a TOCTOU race that surfaced + // spurious "Cannot read properties of undefined" errors. + const payload = await browser.execute(() => + typeof window.wdioTraceCollector !== 'undefined' + ? window.wdioTraceCollector.getTraceData() + : null ) - - if (!collectorExists) { + if (!payload) { log.warn( 'wdioTraceCollector not loaded yet - page loaded before preload script took effect' ) return } - - const payload = await browser.execute(() => - window.wdioTraceCollector.getTraceData() - ) this.processTracePayload(payload as Record) } catch (err) { log.error(`Failed to capture trace: ${errorMessage(err)}`) From 40a910c296f81af601946fc9bbb96287c649c1e6 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 10 Jun 2026 16:23:50 +0530 Subject: [PATCH 13/13] Bound polynomial regex in checkUniqueness xpath parser --- packages/core/src/locators/locator-generation.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/core/src/locators/locator-generation.ts b/packages/core/src/locators/locator-generation.ts index cba05cc8..a66bdf49 100644 --- a/packages/core/src/locators/locator-generation.ts +++ b/packages/core/src/locators/locator-generation.ts @@ -94,7 +94,11 @@ function checkUniqueness( return checkXPathUniqueness(ctx.parsedDOM, xpath, targetNode) } - const match = xpath.match(/\/\/\*\[@([^=]+)="([^"]+)"\]/) + // Bounded quantifiers ({1,200} for attr names, {1,2000} for values) prevent + // polynomial-time backtracking on near-misses like `//[@<` repeated — CodeQL's + // js/polynomial-redos query flagged the unbounded form. The bounds are well + // above any realistic XPath attribute name/value length. + const match = xpath.match(/\/\/\*\[@([^=]{1,200})="([^"]{1,2000})"\]/) if (match) { const [, attr, value] = match return { isUnique: isAttributeUnique(ctx.sourceXML, attr, value) }