diff --git a/packages/contracts/src/dispatch-disclosure.fixtures.ts b/packages/contracts/src/dispatch-disclosure.fixtures.ts index 89ed8507ce..9bee01c77d 100644 --- a/packages/contracts/src/dispatch-disclosure.fixtures.ts +++ b/packages/contracts/src/dispatch-disclosure.fixtures.ts @@ -56,7 +56,7 @@ export const DISPATCH_DISCLOSURE_DRIVER_OWNERS: Readonly> 'web-agent-browser.': 'packages/platform-web/src/__tests__/agent-browser-provider-dispatch-disclosure.test.ts', 'maestro-direct.': - 'src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios.test.ts', + 'packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.test.ts', }; /** The per-row driving loop every driver file carries. */ diff --git a/packages/daemon/package.json b/packages/daemon/package.json index d606016ff5..99b5ac44d0 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -56,10 +56,6 @@ "types": "./src/execution/__tests__/interaction-get-runtime-fixture.ts", "default": "./src/execution/__tests__/interaction-get-runtime-fixture.ts" }, - "./testing/__tests__/ios-comparison-key-fixture": { - "types": "./src/core/__tests__/ios-comparison-key-fixture.ts", - "default": "./src/core/__tests__/ios-comparison-key-fixture.ts" - }, "./testing/__tests__/legacy-snapshot-capture-fixture": { "types": "./src/__tests__/support/legacy-snapshot-capture-fixture.ts", "default": "./src/__tests__/support/legacy-snapshot-capture-fixture.ts" @@ -68,10 +64,6 @@ "types": "./src/__tests__/support/platform-services-fixture.ts", "default": "./src/__tests__/support/platform-services-fixture.ts" }, - "./testing/__tests__/replay-runtime/replay-command-fixture": { - "types": "./src/sessions/__tests__/replay-runtime/replay-command-fixture.ts", - "default": "./src/sessions/__tests__/replay-runtime/replay-command-fixture.ts" - }, "./testing/__tests__/request-router-dispatch-mocks": { "types": "./src/__tests__/support/request-router-dispatch-mocks.ts", "default": "./src/__tests__/support/request-router-dispatch-mocks.ts" @@ -196,14 +188,6 @@ "types": "./src/execution/interaction/internal/interaction-android-escape.ts", "default": "./src/execution/interaction/internal/interaction-android-escape.ts" }, - "./testing/interaction/internal/interaction-ios-tap-outcome": { - "types": "./src/execution/interaction/internal/interaction-ios-tap-outcome.ts", - "default": "./src/execution/interaction/internal/interaction-ios-tap-outcome.ts" - }, - "./testing/interaction/internal/interaction-touch-android-readiness": { - "types": "./src/execution/interaction/internal/interaction-touch-android-readiness.ts", - "default": "./src/execution/interaction/internal/interaction-touch-android-readiness.ts" - }, "./testing/interaction/internal/interaction-touch-response": { "types": "./src/execution/interaction/internal/interaction-touch-response.ts", "default": "./src/execution/interaction/internal/interaction-touch-response.ts" diff --git a/packages/daemon/src/__tests__/support/interactor-resolution-fixture.ts b/packages/daemon/src/__tests__/support/interactor-resolution-fixture.ts new file mode 100644 index 0000000000..b234198a5a --- /dev/null +++ b/packages/daemon/src/__tests__/support/interactor-resolution-fixture.ts @@ -0,0 +1,44 @@ +import { vi } from 'vitest'; +import type { + Interactor, + RunnerContext, + SnapshotOptions, + SnapshotResult, +} from '@agent-device/contracts/interactor-types'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { installInteractorResolution } from '../../core/interactor-resolution.ts'; + +type InteractorSnapshotCall = Readonly<{ + device: DeviceInfo; + runnerContext: RunnerContext; + options: SnapshotOptions; +}>; + +/** + * The snapshot the resolved interactor answers with. Every capture that falls back to + * `captureSnapshotWithInteractor` lands here with the `{ device, runnerContext, options }` the + * production seam hands the interactor, so a suite configures captures and reads their call + * arguments from one spy. + */ +export const mockInteractorSnapshot = vi.fn< + (call: InteractorSnapshotCall) => Promise +>(async () => ({ + nodes: [], + backend: 'xctest', + producer: 'apple-runner', +})); + +/** + * Installs an interactor resolution whose interactor snapshots through + * {@link mockInteractorSnapshot}. Call from `beforeEach`; the spy keeps its configuration until the + * suite resets it. + */ +export function installInteractorSnapshotResolution(): void { + installInteractorResolution({ + resolve: async (device, runnerContext) => + ({ + snapshot: async (options: SnapshotOptions) => + await mockInteractorSnapshot({ device, runnerContext, options }), + }) as unknown as Interactor, + }); +} diff --git a/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts new file mode 100644 index 0000000000..1791d8760a --- /dev/null +++ b/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts @@ -0,0 +1,478 @@ +import type { + CreateDaemonCommandSurface, + DaemonCommandSurface, +} from '@agent-device/contracts/daemon-command-surface'; +import type { + AgentDeviceBackend, + BackendActionResult, + BackendCommandContext, + BackendSnapshotOptions, +} from '@agent-device/contracts/backend'; +import type { + InteractionEvidence, + InteractionTarget, + PreresolvedInteractionTarget, + ResolvedInteractionTarget, + SettleObservation, + SettleParams, +} from '@agent-device/contracts/interaction'; +import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; +import { attachResolvedInteractionTarget } from '@agent-device/host-kit/interaction-outcome'; +import { + buildDragGesturePlan, + singlePointerPlanEndpoints, +} from '@agent-device/contracts/gesture-plan'; +import { AppError } from '@agent-device/kernel/errors'; +import type { Point, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; +import { extractReadableText } from '@agent-device/capture-kit/text-surface'; +import { buildSelectorChainForNode } from '@agent-device/selectors'; +import { fakeCreateCommandSurface } from './command-surface-fake.ts'; + +/** + * A command surface that answers the touch actions the way the real command family does at the two + * seams the daemon owns: it dispatches a resolved target through the runtime config's backend, and + * it reads the session through the config's session store. Suites that pin what the daemon does + * around a touch — admission, bound-operation wiring, response shaping, recording, settle capture + * wiring — need both, because the daemon's decisions land on those seams. + * + * Target resolution is deliberately the smallest rule that lets a dispatch happen: a point is + * itself, a `@ref` is the centre of the node the session's authorized frame (or latest + * observation) names, and a selector is the first node of one fresh capture that matches one of its `||` alternatives + * (each a conjunction of `key=value` terms). Promotion, occlusion, off-screen and bounds guards, label recovery, and + * `--verify` evidence are command-family behavior and are asserted beside that code. + * + * Values the fake supplies itself rather than reading from a backend: an `is` read always passes + * (`pass: true`, `matches: 1`); selector reads (`get`, `is`) capture once with + * `READ_CAPTURE_OPTIONS`, while a point or `@ref` read captures nothing; `--verify` evidence is a + * placeholder digest (`fake-digest-`). Daemon suites assert how the daemon passes, records and + * discloses these values, never their content; the real predicate, capture and evidence values are + * asserted against the real command surface in `src/commands/interaction/runtime/`. + */ + +/** The runtime configs this surface was assembled with, oldest first. */ +export const touchSurfaceConfigs: AgentDeviceRuntimeConfig[] = []; + +export function resetTouchSurfaceFake(): void { + touchSurfaceConfigs.length = 0; +} + +type CommandOptions = Readonly<{ + session?: string; + requestId?: string; + signal?: AbortSignal; + settle?: SettleParams; + verify?: boolean; + preresolvedTarget?: PreresolvedInteractionTarget; +}>; + +export const touchCreateCommandSurface: CreateDaemonCommandSurface = async (config) => { + touchSurfaceConfigs.push(config); + const inert = await fakeCreateCommandSurface(config); + return { + ...inert, + signal: config.signal, + selectors: touchSelectors(config, inert), + interactions: touchInteractions(config, inert), + }; +}; + +function touchSelectors( + config: AgentDeviceRuntimeConfig, + inert: DaemonCommandSurface, +): DaemonCommandSurface['selectors'] { + return { + ...inert.selectors, + get: async (options) => { + const resolved = await resolveTarget(config, options.target, options, READ_CAPTURE_OPTIONS); + if (resolved.kind === 'point' || !resolved.node) { + throw new AppError('COMMAND_FAILED', 'get target did not resolve to a node'); + } + return { + kind: options.property, + target: resolved.target, + node: resolved.node, + selectorChain: resolved.selectorChain, + preActionNodes: resolved.preAction?.nodes ?? [], + ...(options.property === 'text' + ? { text: await readNodeText(config, resolved.node, options) } + : {}), + } as never; + }, + is: async (options) => { + const resolved = await resolveTarget( + config, + { kind: 'selector', selector: options.selector }, + options, + READ_CAPTURE_OPTIONS, + ); + return { + predicate: options.predicate, + pass: true, + selector: options.selector, + matches: 1, + selectorChain: resolved.kind === 'point' ? undefined : resolved.selectorChain, + node: resolved.kind === 'point' ? undefined : resolved.node, + preActionNodes: resolved.kind === 'point' ? undefined : resolved.preAction?.nodes, + }; + }, + }; +} + +async function readNodeText( + config: AgentDeviceRuntimeConfig, + node: SnapshotNode, + options: CommandOptions, +): Promise { + const read = await config.backend.readText?.(context(config, options), node); + return read?.text.trim() ? read.text : extractReadableText(node); +} + +function touchInteractions( + config: AgentDeviceRuntimeConfig, + inert: DaemonCommandSurface, +): DaemonCommandSurface['interactions'] { + const press = async ( + target: InteractionTarget, + options: CommandOptions & { button?: unknown; [key: string]: unknown } = {}, + ) => { + const resolved = await resolveTarget(config, target, options, undefined, 'tapTarget'); + const backendResult = await dispatch(resolved, () => dispatchTap(config, resolved, options)); + return await finish(config, resolved, options, { backendResult }); + }; + return { + ...inert.interactions, + click: press, + press, + gesture: async (options) => await dragOrInert(config, inert, options), + fill: async (target, text, options = {}) => { + const resolved = await resolveTarget(config, target, options, undefined, 'fillTarget'); + const backendResult = await dispatch(resolved, () => + dispatchFill(config, resolved, text, options), + ); + return await finish(config, resolved, options, { backendResult, text }); + }, + longPress: async (target, options = {}) => { + const resolved = await resolveTarget(config, target, options); + const backendResult = await dispatch(resolved, () => + backendOperation(config.backend.longPress?.bind(config.backend))( + context(config, options), + requirePointOf(resolved), + { durationMs: options.durationMs }, + ), + ); + return await finish(config, resolved, options, { + backendResult, + durationMs: options.durationMs, + }); + }, + hover: async (target, options = {}) => { + const resolved = await resolveTarget(config, target, options, undefined, 'hoverTarget'); + const backendResult = await dispatch(resolved, () => + dispatchHover(config, resolved, options), + ); + return await finish(config, resolved, options, { backendResult }); + }, + }; +} + +/** + * A target-authored drag resolves both endpoints like a press target, performs ONE plan through + * the backend, and reports the endpoints and the recording side channel the daemon stores. Every + * other gesture keeps the inert answer. + */ +async function dragOrInert( + config: AgentDeviceRuntimeConfig, + inert: DaemonCommandSurface, + options: Parameters[0], +) { + const gesture = options.gesture; + if (!('intent' in gesture) || gesture.intent !== 'drag') + return await inert.interactions.gesture(options); + const commandOptions = { session: options.session, requestId: options.requestId }; + const [source, destination] = await Promise.all( + [gesture.source, gesture.destination].map( + async (expression) => + await resolveTarget( + config, + expression.startsWith('@') + ? { kind: 'ref', ref: expression } + : { kind: 'selector', selector: expression }, + commandOptions, + ), + ), + ); + const viewport = + (await config.backend.resolveGestureViewport?.(context(config, commandOptions))) ?? + source!.preAction?.nodes.find((node) => node.rect)?.rect; + if (!viewport) throw new AppError('COMMAND_FAILED', 'gesture viewport is unavailable'); + const plan = buildDragGesturePlan( + { + from: requirePointOf(source!), + to: requirePointOf(destination!), + sourceHoldMs: gesture.sourceHoldMs, + moveMs: gesture.moveMs, + destinationHoldMs: gesture.destinationHoldMs, + }, + viewport, + ); + const backendResult = await backendOperation(config.backend.performGesture?.bind(config.backend))( + context(config, commandOptions), + plan, + ); + const { start: from, end: to } = singlePointerPlanEndpoints(plan); + const durationMs = plan.durationMs; + const endpoint = (resolved: ResolvedInteractionTarget) => { + if (resolved.kind === 'point' || !resolved.node || !resolved.selectorChain) { + throw new AppError('COMMAND_FAILED', 'drag endpoint did not resolve to a node'); + } + return { + selectorChain: resolved.selectorChain, + node: resolved.node, + preActionNodes: resolved.preAction?.nodes ?? [], + }; + }; + const sourceTarget = endpoint(source!); + const destinationTarget = endpoint(destination!); + const exact = { source: 'ref', phase: 'pre-action', kind: 'exact' } as const; + return { + kind: 'drag', + durationMs, + pointerCount: 1, + from, + to, + backendResult: backendResult ?? undefined, + targets: { + source: { selectorChain: sourceTarget.selectorChain, resolution: exact }, + destination: { selectorChain: destinationTarget.selectorChain, resolution: exact }, + }, + recording: { + sourceSelector: sourceTarget.selectorChain.join(' || '), + destinationSelector: destinationTarget.selectorChain.join(' || '), + sourceTarget, + destinationTarget, + }, + } as const; +} + +/** What a read (`get`, `is`) asks its capture for: the whole tree, with rects. */ +const READ_CAPTURE_OPTIONS = { interactiveOnly: false, includeRects: true } as const; + +async function resolveTarget( + config: AgentDeviceRuntimeConfig, + target: InteractionTarget, + options: CommandOptions, + captureOptions: BackendSnapshotOptions = { interactiveOnly: true }, + nativeRefRoute?: 'tapTarget' | 'fillTarget' | 'hoverTarget', +): Promise { + if (target.kind === 'point') return { kind: 'point', point: { x: target.x, y: target.y } }; + const pre = options.preresolvedTarget; + if (pre && target.kind === 'ref') return resolvedFromNode(config, target, pre.node, pre.tree); + if (target.kind === 'ref') { + if (nativeRefRoute && config.backend[nativeRefRoute]) return { kind: 'ref', target }; + const record = await config.sessions?.get(options.session ?? 'default'); + const tree = record?.refFrameSnapshot ?? record?.snapshot; + const node = tree?.nodes.find((candidate) => `@${candidate.ref}` === refBody(target.ref)); + if (!tree || !node) throw new AppError('COMMAND_FAILED', `Ref ${target.ref} not found`); + return resolvedFromNode(config, target, node, tree); + } + const captured = await config.backend.captureSnapshot?.(context(config, options), captureOptions); + const tree = captured?.snapshot; + const node = tree?.nodes.find((candidate) => matchesSelector(candidate, target.selector)); + if (!tree || !node) { + throw new AppError('COMMAND_FAILED', `Selector ${target.selector} did not match`); + } + return { + kind: 'selector', + target: { kind: 'selector', selector: target.selector }, + point: centerOf(node) as Point, + node, + selectorChain: buildSelectorChainForNode(node, config.backend.platform), + preAction: { nodes: tree.nodes }, + }; +} + +function resolvedFromNode( + config: AgentDeviceRuntimeConfig, + target: Extract, + node: SnapshotNode, + tree: Pick, +): ResolvedInteractionTarget { + return { + kind: 'ref', + target: { kind: 'ref', ref: target.ref }, + point: centerOf(node), + node, + selectorChain: buildSelectorChainForNode(node, config.backend.platform), + preAction: { nodes: tree.nodes }, + }; +} + +function refBody(ref: string): string { + return ref.startsWith('@') ? ref : `@${ref}`; +} + +function centerOf(node: SnapshotNode): Point | undefined { + const rect = node.rect; + if (!rect) return undefined; + return { x: Math.round(rect.x + rect.width / 2), y: Math.round(rect.y + rect.height / 2) }; +} + +/** A selector chain: `||` alternatives, each a conjunction of `key=value` terms. */ +function matchesSelector(node: SnapshotNode, selector: string): boolean { + const actual: Record = { + id: node.identifier, + label: node.label, + text: node.label ?? node.value, + value: node.value, + role: node.type?.replace(/^XCUIElementType/i, '').toLowerCase(), + }; + return selector.split(' || ').some((alternative) => { + const terms = [...alternative.matchAll(/(\w+)=(?:"((?:[^"\\]|\\.)*)"|(\S+))/g)]; + return ( + terms.length > 0 && terms.every(([, key, quoted, bare]) => actual[key!] === (quoted ?? bare)) + ); + }); +} + +function context(config: AgentDeviceRuntimeConfig, options: CommandOptions): BackendCommandContext { + return { + session: options.session, + requestId: options.requestId, + signal: options.signal ?? config.signal, + }; +} + +async function dispatch( + resolved: ResolvedInteractionTarget, + run: () => Promise, +): Promise { + try { + return await run(); + } catch (error) { + attachResolvedInteractionTarget(error, resolved); + throw error; + } +} + +function backendOperation(operation: TOperation | undefined): TOperation { + if (!operation) { + throw new AppError('UNSUPPORTED_OPERATION', 'backend does not support the action'); + } + return operation; +} + +function requirePointOf(resolved: ResolvedInteractionTarget): Point { + if (resolved.kind === 'point') return resolved.point; + if (!resolved.point) throw new AppError('COMMAND_FAILED', 'resolved target has no point'); + return resolved.point; +} + +async function dispatchTap( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + options: CommandOptions & { button?: unknown }, +): Promise { + const tapOptions = options as Parameters>[2]; + if (resolved.kind === 'ref' && !resolved.point && config.backend.tapTarget) { + return await config.backend.tapTarget(context(config, options), resolved.target, tapOptions); + } + return await backendOperation(config.backend.tap?.bind(config.backend))( + context(config, options), + requirePointOf(resolved), + tapOptions, + ); +} + +async function dispatchFill( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + text: string, + options: CommandOptions & { delayMs?: number }, +): Promise { + const fillOptions = { + delayMs: options.delayMs, + allowNonHittableCoordinateFallback: ( + options as { allowNonHittableCoordinateFallback?: boolean } + ).allowNonHittableCoordinateFallback, + }; + if (resolved.kind === 'ref' && !resolved.point && config.backend.fillTarget) { + return await config.backend.fillTarget( + context(config, options), + resolved.target, + text, + fillOptions, + ); + } + return await backendOperation(config.backend.fill?.bind(config.backend))( + context(config, options), + requirePointOf(resolved), + text, + fillOptions, + ); +} + +async function dispatchHover( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + options: CommandOptions, +): Promise { + if (resolved.kind === 'ref' && !resolved.point && config.backend.hoverTarget) { + return await config.backend.hoverTarget(context(config, options), resolved.target); + } + return await backendOperation(config.backend.hover?.bind(config.backend))( + context(config, options), + requirePointOf(resolved), + ); +} + +async function finish>( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + options: CommandOptions, + extra: TExtra, +) { + const evidence = options.verify ? await verifyEvidence(config, resolved, options) : undefined; + const settle = options.settle ? await settleObservation(config, resolved, options) : undefined; + return { + ...resolved, + ...extra, + ...(evidence ? { evidence } : {}), + ...(settle ? { settle } : {}), + } as never; +} + +async function verifyEvidence( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + options: CommandOptions, +): Promise { + const after = await config.backend.captureSnapshot?.(context(config, options), { + interactiveOnly: true, + }); + const nodes = after?.snapshot?.nodes ?? []; + const before = resolved.kind === 'point' ? undefined : resolved.preAction?.nodes; + return { + nodeCount: nodes.length, + interactiveNodeCount: nodes.length, + digest: `fake-digest-${nodes.length}`, + changedFromBefore: before === undefined || before.length !== nodes.length, + }; +} + +async function settleObservation( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + options: CommandOptions, +): Promise { + const captures = resolved.kind === 'point' ? 2 : 1; + for (let capture = 0; capture < captures; capture += 1) { + await config.backend.captureSnapshot?.(context(config, options), { interactiveOnly: true }); + } + return { + settled: true, + waitedMs: 0, + captures, + quietMs: options.settle?.quietMs ?? 0, + timeoutMs: options.settle?.timeoutMs ?? 0, + }; +} diff --git a/packages/daemon/src/__tests__/support/touch-interaction-runner.ts b/packages/daemon/src/__tests__/support/touch-interaction-runner.ts new file mode 100644 index 0000000000..70136e479f --- /dev/null +++ b/packages/daemon/src/__tests__/support/touch-interaction-runner.ts @@ -0,0 +1,49 @@ +import { getRuntimeBindings } from '../../execution/__tests__/interaction-get-runtime-fixture.ts'; +import { handleInteractionCommands } from '../../execution/interaction/index.ts'; +import { contextFromFlags } from '../../execution/interaction/internal/__tests__/interaction-touch-fixtures.ts'; +import type { SessionStore } from '../../core/session-store.ts'; +import { touchCreateCommandSurface } from './touch-command-surface-fake.ts'; + +/** + * One interaction request through `handleInteractionCommands` on the touch command surface fake + * and the shared runtime-binding fixture, so suites state only the request they are pinning. + */ +export async function runInteraction( + sessionStore: SessionStore, + sessionName: string, + command: string, + positionals: string[], + flags: Record = {}, +) { + return await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { token: 't', session: sessionName, command, positionals, flags }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); +} + +/** The re-entry a mutating `find` makes: a click carrying the node `find` already resolved. */ +export async function runFindInternalClick( + sessionStore: SessionStore, + sessionName: string, + internal: Record, +) { + return await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['@e2'], + flags: { noRecord: true }, + internal, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); +} diff --git a/src/__tests__/daemon-composition/interaction/interaction-gesture-drag.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-gesture-drag.test.ts similarity index 85% rename from src/__tests__/daemon-composition/interaction/interaction-gesture-drag.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-gesture-drag.test.ts index e9b4d7a1be..b8315a8f12 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-gesture-drag.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-gesture-drag.test.ts @@ -1,21 +1,21 @@ -import { composedPlatformServicesFixture } from '../platform-services-fixture.ts'; import { beforeEach, expect, test } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { makeIosSession, authoringPublication, -} from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { activateCompleteRefFrame, refFrameState } from '@agent-device/daemon/testing/ref-frame'; +} from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { activateCompleteRefFrame, refFrameState } from '../../../core/ref-frame.ts'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { gestureRuntimeBindingsFixture } from '@agent-device/daemon/testing/interaction/internal/__tests__/gesture-runtime-bindings.fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +import { handleInteractionCommands } from '../index.ts'; +import { gestureRuntimeBindingsFixture } from './__tests__/gesture-runtime-bindings.fixtures.ts'; const contextFromFlags = () => ({}); const captureSnapshotForSession = async ({ session, -}: import('@agent-device/daemon/testing/session-state').SessionRef) => session.snapshot!; +}: import('../../../core/session-state.ts').SessionRef) => session.snapshot!; let gestures = gestureRuntimeBindingsFixture(); beforeEach(() => { @@ -63,8 +63,8 @@ function makeDragSession(sessionName: string) { async function runDrag(sessionStore: ReturnType, sessionName: string) { return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - platformServices: composedPlatformServicesFixture(), + createCommandSurface: touchCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), req: { token: 't', session: sessionName, diff --git a/src/__tests__/daemon-composition/interaction/interaction-ios-tap-outcome.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.test.ts similarity index 94% rename from src/__tests__/daemon-composition/interaction/interaction-ios-tap-outcome.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.test.ts index c960608bf2..361270a012 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-ios-tap-outcome.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.test.ts @@ -1,5 +1,8 @@ import type { CommandFlags } from '@agent-device/contracts/command'; -import { legacyDispatchCapture } from '@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'; +import { + legacyDispatchCapture, + resetLegacySnapshotCapture, +} from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; import { beforeEach, expect, test, vi } from 'vitest'; import fs from 'node:fs'; @@ -7,47 +10,40 @@ import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot'; import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { handleInteractionCommands } from '../index.ts'; import { makeIosSession, authoringPublication, -} from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { SessionScriptWriter } from '@agent-device/daemon/testing/session-script-writer'; -import { runReplayForTest } from '@agent-device/daemon/testing/__tests__/replay-runtime/replay-command-fixture'; +} from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { SessionScriptWriter } from '../../../core/session-script-writer.ts'; +import { runReplayForTest } from '../../../sessions/__tests__/replay-runtime/replay-command-fixture.ts'; import { replayScriptSourceBundleFor } from '@agent-device/replay-port/replay-script-source-fixtures'; import { imageViewerNodes, profileNodes, snapshot, snapshotPayload, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures'; +} from './__tests__/interaction-ios-tap-outcome-fixtures.ts'; import { appCaptureComparisonIdentity, appCaptureComparisonKey, systemSurfaceCaptureComparisonKey, -} from '@agent-device/daemon/testing/__tests__/ios-comparison-key-fixture'; +} from '../../../core/__tests__/ios-comparison-key-fixture.ts'; import { IOS_SIMULATOR } from '@agent-device/test-kit/device-fixtures'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -import { corroborateIosTapFailure } from '@agent-device/daemon/testing/interaction/internal/interaction-ios-tap-outcome'; -import { movingTargetBaseline } from '@agent-device/daemon/testing/deferred-interaction-outcome'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { corroborateIosTapFailure } from './interaction-ios-tap-outcome.ts'; +import { movingTargetBaseline } from '../../../core/deferred-interaction-outcome.ts'; import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', async () => { - const fixture = - await import('@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'); - return { - captureSnapshotWithInteractor: vi.fn(fixture.captureSnapshotThroughLegacyDispatchFixture), - }; -}); - -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; const contextFromFlags = (flags: CommandFlags | undefined) => ({ count: flags?.count, @@ -70,7 +66,7 @@ async function runClick( const session = sessionStore.get(sessionName); if (session?.snapshot && !session.scriptPublication && options.positionals?.length !== 2) { const baseline = session.snapshot; - mockCaptureSnapshotForSession.mockResolvedValueOnce({ + mockInteractorSnapshot.mockResolvedValueOnce({ nodes: baseline.nodes, backend: 'xctest', producer: 'apple-runner', @@ -78,7 +74,7 @@ async function runClick( }); } return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 'test', session: sessionName, @@ -95,7 +91,8 @@ async function runClick( beforeEach(() => { resetGetRuntimeFixture(); - mockCaptureSnapshotForSession.mockClear(); + installInteractorSnapshotResolution(); + resetLegacySnapshotCapture(mockInteractorSnapshot); legacyDispatchCapture.mockReset(); mockTapPoint.mockImplementation(async (input) => { return await legacyDispatchCapture( @@ -580,7 +577,7 @@ test('corroborates a tap when the request carries no flags and the baseline used // mirrors the raw daemon/JSON-RPC production boundary, which can omit the // key entirely. CLI and batch paths always materialize a flags object. const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 'test', session: sessionName, @@ -839,7 +836,7 @@ test('corroborated runtime taps retain target evidence through save and replay', sessionStore: replayStore, invoke: async (req) => { const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req, sessionName: replaySessionName, sessionStore: replayStore, @@ -873,7 +870,7 @@ test('a tap corroborates against its admitted snapshot after a same-lifetime rec return {}; }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 'test', session: name, command: 'click', positionals: ['104', '222'], flags: {} }, sessionName: name, sessionRef: ref, diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-android-freshness.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-freshness.test.ts similarity index 80% rename from src/__tests__/daemon-composition/interaction/interaction-touch-android-freshness.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-android-freshness.test.ts index c9329f67b1..af6788fa6c 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-android-freshness.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-freshness.test.ts @@ -1,26 +1,24 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { activateCompleteRefFrame } from '../../../core/ref-frame.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - makeAndroidSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { contextFromFlags, makeAndroidSession } from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // The Android ref-refresh capture a @ref mutation takes before dispatch: when // it runs, what it may not retarget, and the comparison-safe baseline it hands // finalization when the refresh fails. -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { const actual = await importOriginal(); return { @@ -31,27 +29,14 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - import { getAndroidAppState, getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -import { activateCompleteRefFrame } from '@agent-device/daemon/testing/ref-frame'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); beforeEach(() => { resetGetRuntimeFixture(); @@ -61,9 +46,8 @@ beforeEach(() => { mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); mockGetAndroidScreenSize.mockReset(); mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test('press @ref refreshes Android snapshot when freshness tracking is active', async () => { @@ -93,7 +77,7 @@ test('press @ref refreshes Android snapshot when freshness tracking is active', }; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: [ { index: 0, @@ -109,7 +93,7 @@ test('press @ref refreshes Android snapshot when freshness tracking is active', }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -124,8 +108,8 @@ test('press @ref refreshes Android snapshot when freshness tracking is active', }); expect(response?.ok).toBe(true); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); - expect(mockCaptureSnapshotForSession.mock.calls[0]?.[0].options).toMatchObject({ + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); + expect(mockInteractorSnapshot.mock.calls[0]?.[0].options).toMatchObject({ interactiveOnly: true, }); expect(mockTapPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 140, y: 220 } })); @@ -168,7 +152,7 @@ test('ADR 0014: Android freshness cannot retarget an admitted ref by positional // The freshness refresh returns a DIFFERENT element at @e1's index — after // navigation the button at that position is now "Cancel", not "Continue". - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: [ { index: 0, @@ -184,7 +168,7 @@ test('ADR 0014: Android freshness cannot retarget an admitted ref by positional }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -232,10 +216,10 @@ test('press @ref falls back to cached Android ref when freshness refresh fails', }; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockRejectedValueOnce(new Error('uiautomator timeout')); + mockInteractorSnapshot.mockRejectedValueOnce(new Error('uiautomator timeout')); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -250,7 +234,7 @@ test('press @ref falls back to cached Android ref when freshness refresh fails', }); expect(response?.ok).toBe(true); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); expect(mockTapPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 60, y: 40 } })); expect(sessionStore.get(sessionName)?.androidSnapshotFreshness).toMatchObject({ action: 'press', @@ -303,7 +287,7 @@ test('coordinate press preserves Android route freshness from last comparable sn sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts new file mode 100644 index 0000000000..ab006e0055 --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts @@ -0,0 +1,47 @@ +import { expect, test, vi } from 'vitest'; +import type { AndroidObservationHost } from '@agent-device/contracts/android-observation'; +import { createAndroidObservationAdapter } from '@agent-device/platform-android'; +import { makeAndroidSession } from '../../../__tests__/support/session-factories.ts'; +import { expireRefFrame } from '../../../core/ref-frame.ts'; +import { runWithAndroidDialogReadinessCheck } from './interaction-touch-android-readiness.ts'; + +// What blocking-dialog readiness costs a run of taps, end to end. Every one of these `dumpsys` +// spawns is a device round trip on the critical path of an interaction, so the budget is pinned +// here rather than left to whatever the composition of the readiness seams happens to produce. + +const NORMAL_FOCUS_DUMP = + ' mCurrentFocus=Window{a1b2c3 u0 com.agentdevice.tester/com.agentdevice.tester.MainActivity}'; + +test('a run of taps on a clear screen costs two window dumps per tap', async () => { + const session = makeAndroidSession('readiness-spawn-budget'); + const runAdb = vi.fn(async (_device, args) => ({ + exitCode: 0, + stdout: args.includes('dumpsys') ? NORMAL_FOCUS_DUMP : '', + stderr: '', + })); + const observation = createAndroidObservationAdapter({ + runAdb, + readSnapshotNodes: async () => [], + openApp: async () => undefined, + }); + + for (let tap = 0; tap < 3; tap += 1) { + const outcome = await runWithAndroidDialogReadinessCheck( + session, + 'press', + { refContext: undefined }, + async () => { + // The dispatch seam every interaction crosses (ADR 0014). + expireRefFrame(session); + return { pressed: true }; + }, + observation, + ); + expect(outcome.aborted).toBe(false); + } + + // Two checks per tap, one dump each: the focused-window line answers the question, so the + // second `dumpsys` variant never runs on a clear screen. Neither check may be skipped — a dialog + // surfaces with no adb traffic at all, so the previous check is evidence about nothing. + expect(runAdb.mock.calls.filter(([, args]) => args.includes('dumpsys'))).toHaveLength(6); +}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-android-readiness.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness.test.ts similarity index 82% rename from src/__tests__/daemon-composition/interaction/interaction-touch-android-readiness.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness.test.ts index e6a5fba15b..05482a34b8 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-android-readiness.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness.test.ts @@ -1,25 +1,26 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { expireRefFrame } from '@agent-device/daemon/testing/ref-frame'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { expireRefFrame } from '../../../core/ref-frame.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - makeAndroidSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { contextFromFlags, makeAndroidSession } from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // The Android device state a dispatch has to survive: an escape to launcher or // Settings fails the press, a permission prompt warns instead, and // blocking-dialog recovery aborts an admitted @ref rather than retargeting it. type EnsureAndroidBlockingSystemDialogReady = - typeof import('@agent-device/daemon/testing/android-system-dialog').ensureAndroidBlockingSystemDialogReady; + typeof import('../../android-system-dialog.ts').ensureAndroidBlockingSystemDialogReady; const { androidDialogReadiness } = vi.hoisted(() => ({ androidDialogReadiness: { @@ -28,10 +29,6 @@ const { androidDialogReadiness } = vi.hoisted(() => ({ }, })); -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { const actual = await importOriginal(); return { @@ -42,21 +39,10 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - // The blocking-dialog readiness check runs for real (its focus probe is mocked // above); the recovery regression below swaps in a recovering implementation. -vi.mock('@agent-device/daemon/testing/android-system-dialog', async (importOriginal) => { - const actual = - await importOriginal(); +vi.mock('../../android-system-dialog.ts', async (importOriginal) => { + const actual = await importOriginal(); androidDialogReadiness.actual = actual.ensureAndroidBlockingSystemDialogReady; return { ...actual, ensureAndroidBlockingSystemDialogReady: androidDialogReadiness.spy }; }); @@ -66,11 +52,9 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); beforeEach(() => { resetGetRuntimeFixture(); @@ -80,9 +64,8 @@ beforeEach(() => { mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); mockGetAndroidScreenSize.mockReset(); mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); androidDialogReadiness.spy.mockReset(); androidDialogReadiness.spy.mockImplementation((params) => androidDialogReadiness.actual(params)); }); @@ -115,7 +98,7 @@ test('press @ref fails when Android tap escapes to launcher', async () => { await expect( handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -164,7 +147,7 @@ test('press @ref fails when Android tap escapes to Settings', async () => { await expect( handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -219,7 +202,7 @@ test.each(ANDROID_PERMISSION_PROMPT_PACKAGES)( }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -279,7 +262,7 @@ test('a ref action aborts with the shared ref_frame_expired rejection after Andr sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, command: 'press', positionals: ['@e1'], flags: {} }, sessionName, sessionStore, diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios-eligibility.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios-eligibility.test.ts similarity index 66% rename from src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios-eligibility.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios-eligibility.test.ts index 37c14bb245..7c27f2f5e3 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios-eligibility.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios-eligibility.test.ts @@ -1,67 +1,29 @@ -import { test, expect, vi, beforeEach } from 'vitest'; +import { test, expect, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockFillPoint, mockTapElementSelector, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { contextFromFlags } from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { contextFromFlags } from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // Ordinary selectors stay capture-backed. Only the explicit Maestro // non-hittable fallback may use the direct iOS selector route. -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - beforeEach(() => { resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test('ordinary click uses canonical capture even when the owner binds selector tap', async () => { @@ -71,7 +33,7 @@ test('ordinary click uses canonical capture even when the owner binds selector t sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' }), ); - mockCaptureSnapshotForSession.mockResolvedValueOnce({ + mockInteractorSnapshot.mockResolvedValueOnce({ nodes: attachRefs([ { index: 0, @@ -93,7 +55,7 @@ test('ordinary click uses canonical capture even when the owner binds selector t }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -109,7 +71,7 @@ test('ordinary click uses canonical capture even when the owner binds selector t expect(response?.ok).toBe(true); expect(mockTapElementSelector).not.toHaveBeenCalled(); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); expect(mockTapPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 90, y: 100 } })); }); @@ -121,7 +83,7 @@ test('fill simple iOS id selector resolves runtime text input evidence before co makeIosSession(sessionName, { appBundleId: 'com.example.app' }), ); - mockCaptureSnapshotForSession.mockResolvedValueOnce({ + mockInteractorSnapshot.mockResolvedValueOnce({ nodes: attachRefs([ { index: 0, @@ -151,7 +113,7 @@ test('fill simple iOS id selector resolves runtime text input evidence before co }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -166,7 +128,7 @@ test('fill simple iOS id selector resolves runtime text input evidence before co }); expect(response?.ok).toBe(true); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); expect(mockFillPoint).toHaveBeenCalledWith( expect.objectContaining({ point: { x: 190, y: 102 }, @@ -199,7 +161,7 @@ test('click simple iOS selector forwards Maestro non-hittable coordinate fallbac }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -243,7 +205,7 @@ test('click simple iOS id selector waits for snapshot path after pending gesture }; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: attachRefs([ { index: 0, @@ -265,7 +227,7 @@ test('click simple iOS id selector waits for snapshot path after pending gesture }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.test.ts similarity index 73% rename from src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.test.ts index 49f4c5f381..659329ae34 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.test.ts @@ -6,43 +6,33 @@ import { DISPATCH_DISCLOSURE_TABLE_PATH, dispatchDisclosureRowsOwnedBy, } from '@agent-device/contracts/dispatch-disclosure-fixtures'; -import { beforeEach, expect, test, vi } from 'vitest'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { beforeEach, expect, test } from 'vitest'; +import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { runInteraction } from '../../../__tests__/support/touch-interaction-runner.ts'; +import { refFrameState } from '../../../core/ref-frame.ts'; import { getRuntimeBindings, mockTapElementSelector, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeStaleRefSession, makeTwoButtonNodes, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { runInteraction } from './interaction-touch-runner.ts'; -import { refFrameState } from '@agent-device/daemon/testing/ref-frame'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; beforeEach(() => { resetGetRuntimeFixture(); - vi.mocked(captureSnapshotWithInteractor).mockReset(); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test.each([ @@ -60,7 +50,7 @@ test.each([ mockTapElementSelector.mockRejectedValue(new AppError(code, message)); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -128,7 +118,7 @@ for (const row of ROWS) { const sessionName = `maestro-direct-${row.id}`; sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); mockTapElementSelector.mockRejectedValueOnce(failure()); - vi.mocked(captureSnapshotWithInteractor).mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: makeTwoButtonNodes(), backend: 'xctest', producer: 'apple-runner', diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts new file mode 100644 index 0000000000..eb7d1b9080 --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts @@ -0,0 +1,223 @@ +import { test, expect, beforeEach } from 'vitest'; +import { attachRefs } from '@agent-device/kernel/snapshot'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { activateCompleteRefFrame, expireRefFrame } from '../../../core/ref-frame.ts'; +import { STALE_SNAPSHOT_REFS_WARNING } from '../../session-snapshot.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { runInteraction } from '../../../__tests__/support/touch-interaction-runner.ts'; +import { + getRuntimeBindings, + mockFillPoint, + resetGetRuntimeFixture, +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { + contextFromFlags, + makeSession, + makeStaleRefSession, +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; + +// How fill is admitted and projected: its @ref admission and pins, and the bound fill the resolved +// point reaches. + +beforeEach(() => { + resetGetRuntimeFixture(); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); +}); + +test('fill @ref dispatches the resolved point with the typing delay and records the ref', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'fill-ref-record'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + identifier: 'auth_email', + rect: { x: 44, y: 120, width: 200, height: 32 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.publish(sessionName, session); + + const response = await runInteraction( + sessionStore, + sessionName, + 'fill', + ['@e1', 'hello@example.com'], + { delayMs: 25 }, + ); + + expect(response?.ok).toBe(true); + expect(mockFillPoint).toHaveBeenCalledWith( + expect.objectContaining({ + point: { x: 144, y: 136 }, + text: 'hello@example.com', + delayMs: 25, + }), + ); + const result = (sessionStore.get(sessionName)?.actions[0]?.result ?? {}) as Record< + string, + unknown + >; + expect(result.ref).toBe('e1'); +}); + +test('fill @ref rejects after a device action expired the frame', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'stale-ref-fill'; + const session = makeStaleRefSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 200, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + // ADR 0014: an unobserved device action expired the frame. + expireRefFrame(session); + sessionStore.publish(sessionName, session); + mockFillPoint.mockRejectedValue( + new Error('dispatch should not be called for an expired-frame ref'), + ); + + const response = await runInteraction(sessionStore, sessionName, 'fill', [ + '@e1', + 'hello@example.com', + ]); + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.code).toBe('COMMAND_FAILED'); + expect(response.error.message).toMatch(/expired ref frame/); + expect(response.error.details?.reason).toBe('ref_frame_expired'); + expect(response.error.details?.hint).toBe(STALE_SNAPSHOT_REFS_WARNING); + } + expect(mockFillPoint).not.toHaveBeenCalled(); +}); + +test('fill with a pinned stale ref rejects; pinned current is clean', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'pinned-fill'; + const session = makeStaleRefSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 200, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + session.snapshotGeneration = 3; + // Re-issue the frame over the overridden snapshot so its source tree matches. + activateCompleteRefFrame(session); + sessionStore.publish(sessionName, session); + + const stale = await runInteraction(sessionStore, sessionName, 'fill', ['@e1~s2', 'hello']); + expect(stale?.ok).toBe(false); + if (stale && !stale.ok) { + expect(stale.error.code).toBe('COMMAND_FAILED'); + expect(stale.error.details?.reason).toBe('ref_generation_mismatch'); + expect(stale.error.details?.dispatched).toBe('no'); + expect(stale.error.details?.hint).toBe( + "Ref @e1 was minted from snapshot s2 but the session's ref frame is now s3 — re-run snapshot -i.", + ); + } + expect(mockFillPoint).not.toHaveBeenCalled(); + + const current = await runInteraction(sessionStore, sessionName, 'fill', ['@e1~s3', 'hello']); + expect(current?.ok).toBe(true); + if (current?.ok) { + expect(current.data?.warning).toBeUndefined(); + } +}); + +test("ADR 0014 blocker-2: a mutating find's internal fill from an expired frame carries no stale-ref warning", async () => { + // Removing the coarse marker (step 8) means an expired frame now derives read + // staleness. A mutating `find fill` re-resolves the locator itself and re-enters + // the fill leaf with a LOCATOR-minted ref (`internal.findResolvedTarget`) — the + // caller never consumed a `@ref`, so the public find response must not claim + // stale refs even though the frame is expired. + const sessionStore = makeSessionStore(); + const sessionName = 'find-internal-fill-expired'; + const session = makeStaleRefSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 200, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + // Re-issue the frame over the overridden snapshot, then expire it as if a prior + // device side effect changed the screen. + activateCompleteRefFrame(session); + expireRefFrame(session); + sessionStore.publish(sessionName, session); + mockFillPoint.mockResolvedValue({ filled: true }); + + // Contrast: a user-supplied `@ref` against the expired frame rejects before + // dispatch (it consumed a stale ref). + const userRef = await runInteraction(sessionStore, sessionName, 'fill', ['@e1', 'hello']); + expect(userRef?.ok).toBe(false); + if (userRef && !userRef.ok) { + expect(userRef.error.details?.reason).toBe('ref_frame_expired'); + expect(userRef.error.details?.dispatched).toBe('no'); + } + expect(mockFillPoint).not.toHaveBeenCalled(); + + // The mutating find's internal dispatch (the exact request find.ts hands the + // leaf) bypasses admission AND attaches no stale-ref warning. + const findNodes = session.snapshot.nodes; + const findNode = findNodes[0]!; + const internal = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['@e1', 'hello'], + flags: {}, + internal: { + findResolvedTarget: { ref: '@e1', node: findNode, tree: { nodes: findNodes } }, + }, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + expect(internal?.ok).toBe(true); + if (internal?.ok) { + expect(internal.data?.warning).toBeUndefined(); + } +}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-payload.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-payload.test.ts similarity index 86% rename from src/__tests__/daemon-composition/interaction/interaction-touch-payload.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-payload.test.ts index ffbd8b9a8f..00cf236c95 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-payload.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-payload.test.ts @@ -1,28 +1,28 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, installTestScreenRecording, makeAndroidSession, makeSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // What a built touch payload carries: the recorded action entry and touch // visualization, the coordinates and reference frame each platform resolves, // and the native timing the recorded result preserves. -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { const actual = await importOriginal(); return { @@ -33,26 +33,14 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - import { getAndroidAppState, getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); beforeEach(() => { resetGetRuntimeFixture(); @@ -62,9 +50,8 @@ beforeEach(() => { mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); mockGetAndroidScreenSize.mockReset(); mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); // fallow-ignore-next-line complexity @@ -98,7 +85,7 @@ test('press coordinates appends touch-visualization events while recording', asy }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -146,7 +133,7 @@ test('press coordinates on iOS recording captures a full snapshot for the touch mockTapPoint.mockResolvedValue({ x: 220, y: 600 }); // Regression: a filtered snapshot has no Application/Window node, so viewport inference would // return a leaf-element bounding box and the recording overlay would misplace tap markers. - mockCaptureSnapshotForSession.mockResolvedValueOnce({ + mockInteractorSnapshot.mockResolvedValueOnce({ nodes: attachRefs([ { index: 0, @@ -165,7 +152,7 @@ test('press coordinates on iOS recording captures a full snapshot for the touch }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -180,8 +167,8 @@ test('press coordinates on iOS recording captures a full snapshot for the touch }); expect(response?.ok).toBe(true); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); - expect(mockCaptureSnapshotForSession.mock.calls[0]?.[0].options).toMatchObject({ + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); + expect(mockInteractorSnapshot.mock.calls[0]?.[0].options).toMatchObject({ interactiveOnly: true, }); const event = sessionStore.get(sessionName)?.screenRecording?.handle.inspect().gestureEvents[0]; @@ -206,7 +193,7 @@ test('press coordinates on Android recording uses physical screen size when no s mockTapPoint.mockResolvedValue({ x: 300, y: 2300 }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -243,7 +230,7 @@ test('press coordinates on Android recording caches physical screen size across mockTapPoint.mockResolvedValue({ x: 300, y: 2300 }); await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -260,7 +247,7 @@ test('press coordinates on Android recording caches physical screen size across mockTapPoint.mockResolvedValue({ x: 320, y: 2200 }); await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -292,7 +279,7 @@ test('press coordinates without recording skips Android screen-size lookup', asy mockTapPoint.mockResolvedValue({ x: 300, y: 2300 }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -327,7 +314,7 @@ test('press coordinates during recording still dispatches when Android screen-si mockGetAndroidScreenSize.mockRejectedValue(new Error('adb unavailable')); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -392,7 +379,7 @@ test('press @ref preserves native timing in recorded result and touch visualizat }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-prepare.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-prepare.test.ts similarity index 88% rename from src/__tests__/daemon-composition/interaction/interaction-touch-prepare.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-prepare.test.ts index 7c0fb0f86a..531de7cb23 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-prepare.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-prepare.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, expect, test, vi } from 'vitest'; +import { beforeEach, expect, test } from 'vitest'; import type { CommandFlags } from '@agent-device/contracts/command'; import { type DeviceBinding, @@ -16,16 +16,20 @@ import { import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { createUnavailableRuntimeFactsForTest } from '@agent-device/test-kit/runtime-operation-facts'; -import { makeIosAppSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; +import { makeIosAppSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; import type { BindDeviceRuntime, InspectDeviceRuntimeFacts, -} from '@agent-device/daemon/testing/request-runtime-binding'; -import { composedPlatformServicesFixture } from '../platform-services-fixture.ts'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import type { SessionStore } from '@agent-device/daemon/testing/session-store'; +} from '../../../core/request-runtime-binding.ts'; +import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import type { SessionStore } from '../../../core/session-store.ts'; +import { handleInteractionCommands } from '../index.ts'; // #3510: the capture an interaction's post-action observation polls. A coordinate-targeted press or // fill acts through a use that declares no capture, so before this its settle loop read the screen @@ -55,11 +59,6 @@ const BRIDGE_PRODUCED = Object.freeze({ producer: 'simulator-ax-bridge' as const, }); -const mockInteractorCapture = vi.hoisted(() => vi.fn()); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: mockInteractorCapture, -})); - const contextFromFlags = (_flags: CommandFlags | undefined) => ({}); /** An owner that captures, taps points, fills points, and does nothing else. */ @@ -141,12 +140,12 @@ async function runCommand( inspectFacts: InspectDeviceRuntimeFacts = bindings.inspectFacts, ) { return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, command, positionals, flags: { ...flags } }, sessionName, sessionStore, contextFromFlags, - platformServices: composedPlatformServicesFixture(), + platformServices: daemonPlatformServicesFixture(), inspectFacts, bindDevice: bindings.bindDevice, }); @@ -157,7 +156,8 @@ function settled(response: unknown) { } beforeEach(() => { - mockInteractorCapture.mockReset(); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); // The observation's bind is a second, capture-only use beside the action's exact one — taken once @@ -190,7 +190,7 @@ test.each([ // issued resolve against the tree the caller's next `snapshot` reads. expect(session.snapshot?.producer).toBe('simulator-ax-bridge'); expect(bindings.captureCount()).toBeGreaterThan(1); - expect(mockInteractorCapture).not.toHaveBeenCalled(); + expect(mockInteractorSnapshot).not.toHaveBeenCalled(); }, ); @@ -214,7 +214,7 @@ test('an observation capture the owner cannot admit still lets the coordinate pr const sessionName = 'point-press-probe-failure'; publishSession(sessionStore, sessionName); const bindings = settleFixture(); - mockInteractorCapture.mockResolvedValue(BRIDGE_PRODUCED); + mockInteractorSnapshot.mockResolvedValue(BRIDGE_PRODUCED); let inspections = 0; const response = await runCommand( @@ -235,7 +235,7 @@ test('an observation capture the owner cannot admit still lets the coordinate pr expect(response?.ok).toBe(true); expect(settled(response)).toMatchObject({ settled: true }); - expect(mockInteractorCapture).toHaveBeenCalled(); + expect(mockInteractorSnapshot).toHaveBeenCalled(); }); test('a tree-resolved press reuses the capture its action binding already carries', async () => { diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-press-admission.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts similarity index 76% rename from src/__tests__/daemon-composition/interaction/interaction-touch-press-admission.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts index db81737659..3c579684c2 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-press-admission.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts @@ -1,79 +1,35 @@ -import { test, expect, vi, beforeEach } from 'vitest'; +import { test, expect, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { activateCompleteRefFrame, refFrameState } from '@agent-device/daemon/testing/ref-frame'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { activateCompleteRefFrame, refFrameState } from '../../../core/ref-frame.ts'; +import { setSessionSnapshot, STALE_SNAPSHOT_REFS_WARNING } from '../../session-snapshot.ts'; import { - setSessionSnapshot, - STALE_SNAPSHOT_REFS_WARNING, -} from '@agent-device/daemon/testing/session-snapshot'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { runInteraction } from '../../../__tests__/support/touch-interaction-runner.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, - findResolvedTarget, - makeFindPreresolvedTree, makeSession, makeStaleRefSession, makeTwoButtonNodes, - readPressPoint, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { runFindInternalClick, runInteraction } from './interaction-touch-runner.ts'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // Whether a targeted touch may act, and on what: click-option and positional -// validation, and the ADR 0014 @ref rules (staleness, pins, frame expiry, -// locator provenance) that admit or reject before any device op. - -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); +// validation, and the ADR 0014 @ref rules (staleness, pins, frame expiry) that +// admit or reject before any device op. beforeEach(() => { resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test('click --button middle on macOS fails with an explicit unsupported-operation error', async () => { @@ -95,7 +51,7 @@ test('click --button middle on macOS fails with an explicit unsupported-operatio ); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -123,7 +79,7 @@ test('press coordinates does not treat extra trailing args as selector', async ( sessionStore.publish(sessionName, makeSession(sessionName)); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -143,30 +99,12 @@ test('press coordinates does not treat extra trailing args as selector', async ( expect(sessionStore.get(sessionName)?.actions.length).toBe(1); }); -test('#1654: the resolved-target payload fails closed when its ref provenance disagrees', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'find-preresolved-mismatched-ref'; - sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); - const preresolved = makeFindPreresolvedTree(); - - const response = await runFindInternalClick(sessionStore, sessionName, { - findResolvedTarget: findResolvedTarget(preresolved, '@e9'), - }); - - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toContain('provenance does not match'); - } - expect(readPressPoint(mockTapPoint)).toBeUndefined(); -}); - test('press selector then press @ref rejects refs that outlived the stored snapshot before dispatch', async () => { const sessionStore = makeSessionStore(); const sessionName = 'stale-ref-warns'; const session = makeStaleRefSession(sessionName); sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: makeTwoButtonNodes(), backend: 'xctest', producer: 'apple-runner', @@ -202,7 +140,7 @@ test('a ref press crosses the ADR 0014 side-effect seam and expires the ref fram const sessionName = 'seam-expiry'; const session = makeStaleRefSession(sessionName); sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: makeTwoButtonNodes(), backend: 'xctest', producer: 'apple-runner', @@ -225,7 +163,7 @@ test('ADR 0014 evidence #1: a second ref mutation rejects (bare and pinned) unti // A complete snapshot issued the frame at generation 500. activateCompleteRefFrame(session); sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: makeTwoButtonNodes(), backend: 'xctest', producer: 'apple-runner', @@ -270,7 +208,7 @@ test('re-issuing a complete frame lets press @ref succeed again without warning' const sessionName = 'reissued-refs-no-warning'; const session = makeStaleRefSession(sessionName); sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: makeTwoButtonNodes(), backend: 'xctest', producer: 'apple-runner', diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.test.ts new file mode 100644 index 0000000000..fa0e023f0b --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.test.ts @@ -0,0 +1,201 @@ +import { test, expect, beforeEach } from 'vitest'; +import { attachRefs } from '@agent-device/kernel/snapshot'; +import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { runFindInternalClick } from '../../../__tests__/support/touch-interaction-runner.ts'; +import { + getRuntimeBindings, + mockTapPoint, + resetGetRuntimeFixture, +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { + contextFromFlags, + findResolvedTarget, + makeFindPreresolvedTree, + makeSession, + makeStaleRefSession, + readPressPoint, +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; + +// How an admitted press/click reaches the device: the button projection onto the bound tap, a +// mutating find's pre-resolved node (#1654) handed to the command surface, and the capture a read +// takes after a press. + +beforeEach(() => { + resetGetRuntimeFixture(); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); +}); + +test('click --button secondary on @ref dispatches a secondary press on macOS and records click', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'default'; + const session = makeSession(sessionName); + session.device = { + platform: 'apple', + appleOs: 'macos', + id: 'macos-desktop', + name: 'My Mac', + kind: 'device', + booted: true, + }; + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeCell', + label: 'failed-step.json', + rect: { x: 400, y: 500, width: 200, height: 20 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.publish(sessionName, session); + + mockTapPoint.mockResolvedValue({ button: 'secondary' }); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['@e1'], + flags: { clickButton: 'secondary' }, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toBeTruthy(); + expect(response?.ok).toBe(true); + expect(readPressPoint(mockTapPoint)).toEqual(['500', '510']); + expect(mockTapPoint.mock.calls[0]?.[0].options.button).toBe('secondary'); + if (response?.ok) { + expect(response.data?.button).toBe('secondary'); + expect(response.data?.ref).toBe('e1'); + } + + const stored = sessionStore.get(sessionName); + expect(stored).toBeTruthy(); + expect(stored?.actions[0]?.command).toBe('click'); + expect(stored?.actions[0]?.flags.clickButton).toBe('secondary'); +}); + +test('#1654: a mutating find click acts on the node find resolved, not a re-resolved @ref', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'find-preresolved-click'; + sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); + const preresolved = makeFindPreresolvedTree(); + + const response = await runFindInternalClick(sessionStore, sessionName, { + findResolvedTarget: findResolvedTarget(preresolved), + }); + + expect(response?.ok).toBe(true); + // find's node — (300,500,20,20) → (310, 510). A second resolution would read + // @e2 out of the SESSION frame tree instead and tap (60, 40). + expect(readPressPoint(mockTapPoint)).toEqual(['310', '510']); +}); + +test('#1654 control: without the resolved-target payload the same click still resolves @ref from the session tree', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'find-preresolved-control'; + sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); + + const response = await runFindInternalClick(sessionStore, sessionName, {}); + + // The ordinary ref path is untouched: @e1 is "Continue" at (10,20,100,40). + expect(response?.ok).toBe(true); + expect(readPressPoint(mockTapPoint)).toEqual(['60', '40']); +}); + +test('#1654: the leaf performs no ref lookup at all — a session that could not resolve @e2 still acts', async () => { + // Defensive proof of the structural contract: this synthetic session has no + // stored snapshot, so any second resolution necessarily fails ("Ref @e2 not + // found or has no bounds"). Production find normally stores its capture; + // this test proves lookup absence without claiming that missing state occurs. + const sessionStore = makeSessionStore(); + const sessionName = 'find-preresolved-no-session-tree'; + const session = makeSession(sessionName); + session.snapshot = undefined; + sessionStore.publish(sessionName, session); + const preresolved = makeFindPreresolvedTree(); + + const withoutPreresolution = await runFindInternalClick(sessionStore, sessionName, {}); + expect(withoutPreresolution?.ok).toBe(false); + + const response = await runFindInternalClick(sessionStore, sessionName, { + findResolvedTarget: findResolvedTarget(preresolved), + }); + + expect(response?.ok).toBe(true); + expect(readPressPoint(mockTapPoint)).toEqual(['310', '510']); +}); + +test('a read right after a press captures the post-tap screen instead of reusing the pre-tap tree', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'read-after-press'; + sessionStore.publish( + sessionName, + makeIosSession(sessionName, { appBundleId: 'com.example.app' }), + ); + const screen = (step: string) => ({ + backend: 'xctest' as const, + producer: 'apple-runner' as const, + nodes: [ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 393, height: 852 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Next', + rect: { x: 24, y: 120, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'StaticText', + label: step, + identifier: 'step', + rect: { x: 24, y: 220, width: 320, height: 24 }, + }, + ], + }); + mockInteractorSnapshot + .mockResolvedValueOnce(screen('Step 1')) + .mockResolvedValue(screen('Step 2')); + const run = async (command: string, positionals: string[]) => + await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { token: 't', session: sessionName, command, positionals, flags: {} }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(await run('is', ['visible', 'label=Next'])).toMatchObject({ ok: true }); + expect(await run('press', ['84', '142'])).toMatchObject({ ok: true }); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); + + const read = await run('get', ['text', 'id="step"']); + + expect(read).toMatchObject({ ok: true, data: { text: 'Step 2' } }); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(2); +}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-response.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-response-evidence.test.ts similarity index 76% rename from src/__tests__/daemon-composition/interaction/interaction-touch-response.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-response-evidence.test.ts index 55b01f4330..a19eecc5f0 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-response.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-response-evidence.test.ts @@ -1,70 +1,32 @@ -import { test, expect, vi, beforeEach } from 'vitest'; +import { test, expect, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockFillPoint, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, installTestScreenRecording, makeSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // The identity extras the one response site composes: --verify evidence rides // the interactionResultExtra allowlist on every branch, and no branch invents // an evidence field without it. -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - beforeEach(() => { resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test('press @ref --verify surfaces evidence through the interactionResultExtra allowlist', async () => { @@ -89,7 +51,7 @@ test('press @ref --verify surfaces evidence through the interactionResultExtra a // Post-action capture reports an extra node, so changedFromBefore should // read true against the pre-action (stored) snapshot's single node. - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: [ { index: 0, @@ -113,7 +75,7 @@ test('press @ref --verify surfaces evidence through the interactionResultExtra a }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -147,7 +109,7 @@ test('press @ref --verify surfaces evidence through the interactionResultExtra a // without a fresh pre-action capture (zero extra cost, per #1047's design) — // only the post-action verify capture issues a 'snapshot' dispatch, after 'press'. expect(mockTapPoint).toHaveBeenCalledOnce(); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledOnce(); + expect(mockInteractorSnapshot).toHaveBeenCalledOnce(); }); test('press @ref without --verify never includes an evidence field', async () => { @@ -172,7 +134,7 @@ test('press @ref without --verify never includes an evidence field', async () => mockTapPoint.mockResolvedValue({ pressed: true }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -192,7 +154,7 @@ test('press @ref without --verify never includes an evidence field', async () => } // No verify flag means no post-action snapshot capture at all. expect(mockTapPoint).toHaveBeenCalledOnce(); - expect(mockCaptureSnapshotForSession).not.toHaveBeenCalled(); + expect(mockInteractorSnapshot).not.toHaveBeenCalled(); }); test('fill selector --verify surfaces evidence through the interactionResultExtra allowlist', async () => { @@ -215,7 +177,7 @@ test('fill selector --verify surfaces evidence through the interactionResultExtr }; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: [ { index: 0, @@ -231,7 +193,7 @@ test('fill selector --verify surfaces evidence through the interactionResultExtr }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -277,7 +239,7 @@ test('fill @ref --verify surfaces evidence in the ref response branch', async () }; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: [ { index: 0, @@ -301,7 +263,7 @@ test('fill @ref --verify surfaces evidence in the ref response branch', async () }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -348,7 +310,7 @@ test('fill @ref without --verify never includes an evidence field', async () => sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -367,7 +329,7 @@ test('fill @ref without --verify never includes an evidence field', async () => expect(response.data?.evidence).toBeUndefined(); } // No verify flag means no post-action snapshot capture at all. - expect(mockCaptureSnapshotForSession).not.toHaveBeenCalled(); + expect(mockInteractorSnapshot).not.toHaveBeenCalled(); }); test('fill @ref preserves fallback coordinates for recording when platform result is sparse', async () => { @@ -399,7 +361,7 @@ test('fill @ref preserves fallback coordinates for recording when platform resul mockFillPoint.mockResolvedValue({ filled: true }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts similarity index 75% rename from src/__tests__/daemon-composition/interaction/interaction-touch.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts index 8139f29792..6690831aee 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts @@ -1,8 +1,12 @@ -import { test, expect, vi, beforeEach } from 'vitest'; +import { test, expect, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; import { WEB_DESKTOP_DEVICE } from '@agent-device/test-kit/device-fixtures'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockHoverPoint, @@ -11,64 +15,22 @@ import { mockTapPoint, resetGetRuntimeFixture, runtimeBindingSpies, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeMacOsDesktopSession, makeMacOsMenubarSession, makeSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // Router ownership: one representative per touch command proves // handleInteractionCommands claims press/click/longpress/hover/fill. -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - beforeEach(() => { resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test('press coordinates dispatches press and records as press', async () => { @@ -78,7 +40,7 @@ test('press coordinates dispatches press and records as press', async () => { sessionStore.publish(sessionName, storedSession); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -121,7 +83,7 @@ test.each([ sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, command, positionals: [...positionals], flags: {} }, sessionName, sessionStore, @@ -140,7 +102,7 @@ test('click rejects macOS desktop surface interactions until helper routing exis sessionStore.publish(sessionName, makeMacOsDesktopSession(sessionName)); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -167,7 +129,7 @@ test('fill rejects macOS menubar surface interactions until helper routing exist sessionStore.publish(sessionName, makeMacOsMenubarSession(sessionName)); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -209,7 +171,7 @@ test('longpress @ref resolves the target and dispatches coordinate longpress', a sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -255,7 +217,7 @@ test('hover @ref on web dispatches through the provider hoverRef route, not coor }; sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -274,6 +236,7 @@ test('hover @ref on web dispatches through the provider hoverRef route, not coor data: { ref: 'e1', gesture: 'hover', message: expect.stringMatching(/Hovered @e1/) }, }); expect(mockHoverRef).toHaveBeenCalledWith(expect.objectContaining({ ref: '@e1' })); + expect(mockHoverPoint).not.toHaveBeenCalled(); expect(sessionStore.get(sessionName)?.actions[0]?.command).toBe('hover'); }); @@ -283,7 +246,7 @@ test('hover selector on web resolves the target and dispatches coordinate hover' const session = makeSession(sessionName); session.device = WEB_DESKTOP_DEVICE; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: attachRefs([ { index: 0, @@ -299,7 +262,7 @@ test('hover selector on web resolves the target and dispatches coordinate hover' }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -326,7 +289,7 @@ test('hover is refused by capability on touch platforms before any dispatch', as sessionStore.publish(sessionName, makeSession(sessionName)); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.test.ts new file mode 100644 index 0000000000..47da4cab06 --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.test.ts @@ -0,0 +1,129 @@ +import { test, expect, beforeEach } from 'vitest'; +import { attachRefs } from '@agent-device/kernel/snapshot'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + resetTouchSurfaceFake, + touchSurfaceConfigs, +} from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { runInteraction } from '../../../__tests__/support/touch-interaction-runner.ts'; +import { + elementReadFixtureState, + mockQuerySelector, + mockTapPoint, + resetGetRuntimeFixture, +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { makeSession } from './__tests__/interaction-touch-fixtures.ts'; + +// What the shared runtime dispatch hands the command surface: the tapped point and the recorded +// `@ref` the caller wrote, and the off-screen rescue hook that rides the request-bound binding. + +beforeEach(() => { + resetGetRuntimeFixture(); + resetTouchSurfaceFake(); +}); + +test('press @ref taps the resolved point once and records the ref', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'press-ref'; + const session = makeSession(sessionName); + const nodes = attachRefs([ + { + index: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + identifier: 'auth_continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ]); + session.snapshot = { nodes, createdAt: Date.now(), backend: 'xctest' }; + sessionStore.publish(sessionName, session); + + const response = await runInteraction(sessionStore, sessionName, 'press', ['@e1']); + + expect(response?.ok).toBe(true); + expect(mockTapPoint).toHaveBeenCalledTimes(1); + expect(mockTapPoint.mock.calls[0]?.[0]?.point).toEqual({ x: 60, y: 40 }); + const stored = sessionStore.get(sessionName); + expect(stored?.actions[0]?.positionals).toEqual(['@e1']); + expect(stored?.actions[0]?.flags).toEqual({}); +}); + +// #1542 wiring, daemon route: the off-screen rescue's live read rides the SAME request-bound +// binding the touch command was admitted for. These two tests pin both declared sides of the +// CONDITIONAL `querySelector` fact: an owner that advertises it hands the command surface a +// rescue hook that answers with the live rect; an owner that does not hands it none, so the guard +// refuses on the bulk-tree verdict with ZERO queries. Deleting `conditional: ['querySelector']` +// from the captured touch uses, or making the rescue hook return undefined, breaks one of these +// two; the Apple binding's own spread is pinned in platform-apple's runtime.test.ts. + +const RESCUE_ROUTE_SESSION = 'rescue-route'; + +function offscreenRefSessionStore() { + const sessionStore = makeSessionStore(); + const session = makeSession(RESCUE_ROUTE_SESSION); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + depth: 0, + type: 'Window', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Far item', + identifier: 'far_item', + rect: { x: 20, y: 1200, width: 120, height: 44 }, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.publish(RESCUE_ROUTE_SESSION, session); + return sessionStore; +} + +const FAR_ITEM = { identifier: 'far_item', label: 'Far item' }; +const ROOT_VIEWPORT = { x: 0, y: 0, width: 390, height: 844 }; + +test('press @ref hands the command surface a rescue hook that reads the live rect when the owner advertises querySelector', async () => { + const sessionStore = offscreenRefSessionStore(); + elementReadFixtureState.querySelectorAvailable = true; + mockQuerySelector.mockResolvedValue({ + found: true, + node: { rect: { x: 20, y: 400, width: 120, height: 44 }, hittable: true }, + }); + + const response = await runInteraction(sessionStore, RESCUE_ROUTE_SESSION, 'press', ['@e2']); + + expect(response?.ok).toBe(true); + const confirm = touchSurfaceConfigs.at(-1)?.backend.confirmOffscreenTargetVisible; + expect(confirm).toBeDefined(); + // The rescue answers with the LIVE rect, not the stale bulk-tree row's off-screen geometry, and + // the live read carried the derived selector. + const rescued = await confirm!({}, FAR_ITEM, ROOT_VIEWPORT); + expect(rescued).toEqual({ x: 20, y: 400, width: 120, height: 44 }); + expect(mockQuerySelector).toHaveBeenCalledTimes(1); + expect(mockQuerySelector.mock.calls[0]?.[0].selector).toEqual({ + key: 'id', + value: 'far_item', + }); +}); + +test('press @ref hands the command surface no rescue hook when the owner does not advertise querySelector', async () => { + // The fixture default: the owner's facts declare the CONDITIONAL read unavailable, so the + // route holds no hook and the guard's refusal is built from the bulk tree alone. + const sessionStore = offscreenRefSessionStore(); + + const response = await runInteraction(sessionStore, RESCUE_ROUTE_SESSION, 'press', ['@e2']); + + expect(response?.ok).toBe(true); + expect(touchSurfaceConfigs.at(-1)?.backend.confirmOffscreenTargetVisible).toBeUndefined(); + expect(mockQuerySelector).not.toHaveBeenCalled(); +}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-get.test.ts b/packages/daemon/src/execution/selector-runtime-get-reads.test.ts similarity index 52% rename from src/__tests__/daemon-composition/interaction/interaction-get.test.ts rename to packages/daemon/src/execution/selector-runtime-get-reads.test.ts index 0694700b81..5d6d13a490 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-get.test.ts +++ b/packages/daemon/src/execution/selector-runtime-get-reads.test.ts @@ -1,50 +1,33 @@ import { test, expect, vi, beforeEach } from 'vitest'; -import { legacyDispatchCapture } from '@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'; -import { attachRefs } from '@agent-device/kernel/snapshot'; -import { WEB_DESKTOP_DEVICE } from '@agent-device/test-kit/device-fixtures'; import { - makeAndroidSession as makeBaseAndroidSession, - makeIosSession, -} from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { expireRefFrame } from '@agent-device/daemon/testing/ref-frame'; + legacyDispatchCapture, + resetLegacySnapshotCapture, +} from '../__tests__/support/legacy-snapshot-capture-fixture.ts'; +import { attachRefs } from '@agent-device/kernel/snapshot'; +import { makeIosSession } from '../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../__tests__/support/store-factory.ts'; +import { expireRefFrame } from '../core/ref-frame.ts'; +import { setSessionSnapshot, STALE_SNAPSHOT_REFS_WARNING } from './session-snapshot.ts'; import { - setSessionSnapshot, - STALE_SNAPSHOT_REFS_WARNING, -} from '@agent-device/daemon/testing/session-snapshot'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../__tests__/support/touch-command-surface-fake.ts'; +import { runInteraction } from '../__tests__/support/touch-interaction-runner.ts'; +import { handleInteractionCommands } from './interaction/index.ts'; import { contextFromFlags, makeSession, makeStaleRefSession, - makeVisibleButtonSnapshot, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { runInteraction } from './interaction-touch-runner.ts'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './interaction/internal/__tests__/interaction-touch-fixtures.ts'; -// Non-touch interaction routing: the `get` and `is` reads the public handler -// owns. Touch commands live in the interaction-touch* test files. +// The `get` reads the public interaction handler owns: which capture or bound operation answers a +// selector or `@ref` read, and the frame-derived warnings a `@ref` read carries. const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn(), })); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', async () => { - const fixture = - await import('@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'); - return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; -}); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { const actual = await importOriginal(); @@ -60,20 +43,11 @@ import { getRuntimeBindings, mockReadTextAtPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, -} from '@agent-device/platform-android/mechanics'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); +} from './__tests__/interaction-get-runtime-fixture.ts'; + beforeEach(() => { - legacyDispatchCapture.mockReset(); - legacyDispatchCapture.mockResolvedValue({}); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); + installInteractorSnapshotResolution(); + resetLegacySnapshotCapture(mockInteractorSnapshot); mockRunAppleRunnerCommand.mockReset(); mockRunAppleRunnerCommand.mockResolvedValue({}); resetGetRuntimeFixture(); @@ -104,7 +78,7 @@ test('get text prefers underlying value for text surfaces and avoids recording g ); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -156,7 +130,7 @@ test('get text uses backend read expansion when the resolved node has a rect', a ); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -203,7 +177,7 @@ test('get text answers from the captured tree when the bound owner advertises no elementReadFixtureState.readTextAtPointAvailable = false; const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -246,7 +220,7 @@ test('an eligible direct iOS selector cannot operate before admission', async () }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -306,7 +280,7 @@ test('get text simple iOS id selector resolves through the bound capture, not a }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -386,7 +360,7 @@ test('get text iOS label selector uses snapshot disambiguation instead of runner }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -410,335 +384,6 @@ test('get text iOS label selector uses snapshot disambiguation instead of runner } }); -test('is visible preserves CLI snapshot flags during runtime snapshot capture', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'snapshot-flags'; - sessionStore.publish(sessionName, makeSession(sessionName)); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeWindow', - label: 'Login', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeButton', - label: 'Continue', - identifier: 'auth_continue', - rect: { x: 10, y: 20, width: 100, height: 40 }, - enabled: true, - hittable: true, - visible: true, - }, - ], - backend: 'xctest', - producer: 'apple-runner', - }; - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'id=auth_continue'], - flags: { snapshotDepth: 2, snapshotScope: 'Login', snapshotRaw: true }, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ - snapshotDepth: 2, - snapshotScope: 'Login', - snapshotRaw: true, - snapshotInteractiveOnly: false, - snapshotIncludeRects: true, - }); -}); - -test('is visible reuses fresh cached iOS snapshots with rects', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'ios-visible-cached'; - const session = makeSession(sessionName); - session.snapshot = makeVisibleButtonSnapshot('Cached action', { - backend: 'xctest', - producer: 'apple-runner', - }); - sessionStore.publish(sessionName, session); - legacyDispatchCapture.mockRejectedValue(new Error('unexpected fresh snapshot')); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Cached action"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(legacyDispatchCapture).not.toHaveBeenCalled(); -}); - -test('is visible recaptures web snapshots when cached nodes may lack rects', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'web-visible-refreshes-rects'; - const session = makeSession(sessionName); - session.device = WEB_DESKTOP_DEVICE; - session.snapshot = buildSnapshotState( - { - nodes: [{ index: 0, type: 'button', label: 'Submit order' }], - backend: 'web', - producer: 'agent-browser', - }, - { snapshotInteractiveOnly: false }, - ); - sessionStore.publish(sessionName, session); - legacyDispatchCapture.mockResolvedValue( - makeVisibleButtonSnapshot('Submit order', { - backend: 'web', - producer: 'agent-browser', - }), - ); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Submit order"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ - snapshotIncludeRects: true, - }); -}); - -// PIN CHANGED TWICE (#1739, R37). #557 asserted `ok: true` with `pass: false` and zero snapshots -// here, from the direct-iOS shortcut. That broke `is`'s documented contract — it "exits non-zero -// on failure" (website/docs/docs/commands.md) — and on device printed `Passed: is text` with exit -// 0 for a failed assertion. The reversal made the shortcut answer only when the predicate held; -// the shortcut is now retired outright, so the bound capture answers every predicate and this is -// simply what `is` does. The assertion below is unchanged across both edits because it was always -// about the OUTCOME, not about which path produced it. -test('a failing is predicate is COMMAND_FAILED, never a zero-exit pass', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'is-selected-ios-direct-selector-false'; - sessionStore.publish( - sessionName, - makeIosSession(sessionName, { appBundleId: 'com.example.app' }), - ); - mockRunAppleRunnerCommand.mockResolvedValue({ - found: true, - nodes: [ - { - index: 0, - depth: 0, - type: 'Button', - label: 'Submit', - identifier: 'submit', - selected: false, - rect: { x: 126, y: 555, width: 75, height: 38 }, - }, - ], - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['selected', 'id="submit"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - // The session snapshot has no `id=submit`, so the bound capture reports the typed selector - // failure. Nothing can report a failed assertion as a completed command. - expect(response?.ok).toBe(false); - expect(legacyDispatchCapture.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(1); - if (response?.ok === false) { - expect(response.error?.code).toBe('COMMAND_FAILED'); - } -}); - -test('is visible passes for list text that inherits viewport visibility from an ancestor', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'visible-list-item'; - sessionStore.publish(sessionName, makeSession(sessionName)); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeCell', - rect: { x: 0, y: 160, width: 390, height: 44 }, - hittable: false, - }, - { - index: 2, - parentIndex: 1, - type: 'XCUIElementTypeStaticText', - label: 'Trip ideas', - hittable: false, - }, - ], - backend: 'xctest', - producer: 'apple-runner', - }; - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Trip ideas"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - if (response?.ok) { - expect(response.data?.predicate).toBe('visible'); - expect(response.data?.pass).toBe(true); - expect(response.data?.selector).toBe('label="Trip ideas"'); - } -}); - -test('is visible fails for nodes outside the current viewport', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'visible-offscreen'; - sessionStore.publish(sessionName, makeSession(sessionName)); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeStaticText', - label: 'Far item', - rect: { x: 20, y: 2600, width: 120, height: 40 }, - hittable: false, - }, - ], - backend: 'xctest', - producer: 'apple-runner', - }; - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Far item"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/actual=\{"visible":false/); - } -}); - -test('is reports Android permission dialog blocker when app content assertion fails', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'android-permission-blocked'; - sessionStore.publish( - sessionName, - makeBaseAndroidSession(sessionName, { appBundleId: 'com.example.demo' }), - ); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { nodes: [], backend: 'uiautomator' }; - }); - mockGetAndroidAppState.mockResolvedValue({ - package: 'com.google.android.permissioncontroller', - activity: 'com.android.permissioncontroller.permission.ui.GrantPermissionsActivity', - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Metro Ready"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.message).toMatch(/permission dialog is blocking/); - expect(response.error.details).toMatchObject({ - blockedBy: 'android_foreground_surface', - expectedPackage: 'com.example.demo', - foregroundPackage: 'com.google.android.permissioncontroller', - }); - } -}); - // fallow-ignore-next-line complexity test('ADR 0014 evidence #17: get text @ref reads the retained frame tree, not a newer observation', async () => { const sessionStore = makeSessionStore(); diff --git a/packages/daemon/src/execution/selector-runtime-is-reads.test.ts b/packages/daemon/src/execution/selector-runtime-is-reads.test.ts new file mode 100644 index 0000000000..96c90db1d7 --- /dev/null +++ b/packages/daemon/src/execution/selector-runtime-is-reads.test.ts @@ -0,0 +1,270 @@ +import { test, expect, vi, beforeEach } from 'vitest'; +import { + legacyDispatchCapture, + resetLegacySnapshotCapture, +} from '../__tests__/support/legacy-snapshot-capture-fixture.ts'; +import { WEB_DESKTOP_DEVICE } from '@agent-device/test-kit/device-fixtures'; +import { + makeAndroidSession as makeBaseAndroidSession, + makeIosSession, +} from '../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../__tests__/support/store-factory.ts'; +import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../__tests__/support/touch-command-surface-fake.ts'; +import { handleInteractionCommands } from './interaction/index.ts'; +import { + contextFromFlags, + makeSession, + makeVisibleButtonSnapshot, +} from './interaction/internal/__tests__/interaction-touch-fixtures.ts'; + +// The `is` reads the public interaction handler owns: the capture each predicate takes, and the +// Android foreground blocker a failed assertion discloses. + +vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getAndroidAppState: vi.fn(async () => ({})), + getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), + }; +}); + +import { + getRuntimeBindings, + resetGetRuntimeFixture, +} from './__tests__/interaction-get-runtime-fixture.ts'; +import { + getAndroidAppState, + getAndroidBlockingDialogObservation, +} from '@agent-device/platform-android/mechanics'; +const mockGetAndroidAppState = vi.mocked(getAndroidAppState); +const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); + +beforeEach(() => { + installInteractorSnapshotResolution(); + resetLegacySnapshotCapture(mockInteractorSnapshot); + mockGetAndroidAppState.mockReset(); + mockGetAndroidAppState.mockResolvedValue({}); + mockGetAndroidBlockingDialogObservation.mockReset(); + mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); + resetGetRuntimeFixture(); +}); + +test('is visible preserves CLI snapshot flags during runtime snapshot capture', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'snapshot-flags'; + sessionStore.publish(sessionName, makeSession(sessionName)); + + legacyDispatchCapture.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); + return { + nodes: [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeWindow', + label: 'Login', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + identifier: 'auth_continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + visible: true, + }, + ], + backend: 'xctest', + producer: 'apple-runner', + }; + }); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'id=auth_continue'], + flags: { snapshotDepth: 2, snapshotScope: 'Login', snapshotRaw: true }, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ + snapshotDepth: 2, + snapshotScope: 'Login', + snapshotRaw: true, + snapshotInteractiveOnly: false, + snapshotIncludeRects: true, + }); +}); + +test('is visible reuses fresh cached iOS snapshots with rects', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-visible-cached'; + const session = makeSession(sessionName); + session.snapshot = makeVisibleButtonSnapshot('Cached action', { + backend: 'xctest', + producer: 'apple-runner', + }); + sessionStore.publish(sessionName, session); + legacyDispatchCapture.mockRejectedValue(new Error('unexpected fresh snapshot')); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Cached action"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(legacyDispatchCapture).not.toHaveBeenCalled(); +}); + +test('is visible recaptures web snapshots when cached nodes may lack rects', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'web-visible-refreshes-rects'; + const session = makeSession(sessionName); + session.device = WEB_DESKTOP_DEVICE; + session.snapshot = buildSnapshotState( + { + nodes: [{ index: 0, type: 'button', label: 'Submit order' }], + backend: 'web', + producer: 'agent-browser', + }, + { snapshotInteractiveOnly: false }, + ); + sessionStore.publish(sessionName, session); + legacyDispatchCapture.mockResolvedValue( + makeVisibleButtonSnapshot('Submit order', { + backend: 'web', + producer: 'agent-browser', + }), + ); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Submit order"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ + snapshotIncludeRects: true, + }); +}); + +// PIN CHANGED TWICE (#1739, R37). #557 asserted `ok: true` with `pass: false` and zero snapshots +// here, from the direct-iOS shortcut. That broke `is`'s documented contract — it "exits non-zero +// on failure" (website/docs/docs/commands.md) — and on device printed `Passed: is text` with exit +// 0 for a failed assertion. The reversal made the shortcut answer only when the predicate held; +// the shortcut is now retired outright, so the bound capture answers every predicate and this is +// simply what `is` does. The assertion below is unchanged across both edits because it was always +// about the OUTCOME, not about which path produced it. +test('a failing is predicate is COMMAND_FAILED, never a zero-exit pass', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'is-selected-ios-direct-selector-false'; + sessionStore.publish( + sessionName, + makeIosSession(sessionName, { appBundleId: 'com.example.app' }), + ); + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['selected', 'id="submit"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + // The session snapshot has no `id=submit`, so the bound capture reports the typed selector + // failure. Nothing can report a failed assertion as a completed command. + expect(response?.ok).toBe(false); + expect(legacyDispatchCapture.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(1); + if (response?.ok === false) { + expect(response.error?.code).toBe('COMMAND_FAILED'); + } +}); + +test('is reports Android permission dialog blocker when app content assertion fails', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'android-permission-blocked'; + sessionStore.publish( + sessionName, + makeBaseAndroidSession(sessionName, { appBundleId: 'com.example.demo' }), + ); + + legacyDispatchCapture.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); + return { nodes: [], backend: 'uiautomator' }; + }); + mockGetAndroidAppState.mockResolvedValue({ + package: 'com.google.android.permissioncontroller', + activity: 'com.android.permissioncontroller.permission.ui.GrantPermissionsActivity', + }); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Metro Ready"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toBeTruthy(); + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.message).toMatch(/permission dialog is blocking/); + expect(response.error.details).toMatchObject({ + blockedBy: 'android_foreground_surface', + expectedPackage: 'com.example.demo', + foregroundPackage: 'com.google.android.permissioncontroller', + permissionDialog: true, + }); + } +}); diff --git a/packages/test-kit/src/boundary-fault-matrix.ts b/packages/test-kit/src/boundary-fault-matrix.ts index 7b55945c2a..4a8a7c0055 100644 --- a/packages/test-kit/src/boundary-fault-matrix.ts +++ b/packages/test-kit/src/boundary-fault-matrix.ts @@ -196,7 +196,7 @@ export const BOUNDARY_FAULT_MATRIX = { mutation: { kind: 'covered', evidence: [ - 'src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios.test.ts', + 'packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.test.ts', ], invariants: ['best-effort-degradation'], }, diff --git a/src/__tests__/daemon-composition/interaction-touch-android-readiness-spawns.test.ts b/src/__tests__/daemon-composition/interaction-touch-android-readiness-spawns.test.ts deleted file mode 100644 index 4a01cdd4ca..0000000000 --- a/src/__tests__/daemon-composition/interaction-touch-android-readiness-spawns.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { expect, test } from 'vitest'; -import { makeAndroidSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { withFakeAdb } from '../test-utils/fake-adb.ts'; -import { expireRefFrame } from '@agent-device/daemon/testing/ref-frame'; -import { runWithAndroidDialogReadinessCheck } from '@agent-device/daemon/testing/interaction/internal/interaction-touch-android-readiness'; -import { androidObservationPolicyFixture as androidObservation } from '@agent-device/daemon/testing/__tests__/android-observation-fixture'; - -// What blocking-dialog readiness costs a run of taps, end to end. Every one of these `dumpsys` -// spawns is a device round trip on the critical path of an interaction, so the budget is pinned -// here rather than left to whatever the composition of the readiness seams happens to produce. - -const NORMAL_FOCUS_DUMP = - ' mCurrentFocus=Window{a1b2c3 u0 com.agentdevice.tester/com.agentdevice.tester.MainActivity}'; - -test('a run of taps on a clear screen costs one window dump per tap', async () => { - const session = makeAndroidSession('readiness-spawn-budget'); - - const dumps = await withFakeAdb( - (args) => (args[1] === 'dumpsys' ? NORMAL_FOCUS_DUMP : ''), - async ({ calls }) => { - for (let tap = 0; tap < 3; tap += 1) { - const outcome = await runWithAndroidDialogReadinessCheck( - session, - 'press', - { refContext: undefined }, - async () => { - // The dispatch seam every interaction crosses (ADR 0014). - expireRefFrame(session); - return { pressed: true }; - }, - androidObservation, - ); - expect(outcome.aborted).toBe(false); - } - return calls.filter((args) => args[1] === 'dumpsys').length; - }, - ); - - // Two checks per tap, one dump each: the focused-window line answers the question, so the - // second `dumpsys` variant never runs on a clear screen. Neither check may be skipped — a dialog - // surfaces with no adb traffic at all, so the previous check is evidence about nothing. - expect(dumps).toBe(6); -}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-fill.test.ts b/src/__tests__/daemon-composition/interaction/interaction-touch-fill.test.ts deleted file mode 100644 index 35087c1e0b..0000000000 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-fill.test.ts +++ /dev/null @@ -1,393 +0,0 @@ -import { test, expect, vi, beforeEach } from 'vitest'; -import { attachRefs } from '@agent-device/kernel/snapshot'; -import { withAppleRunnerProvider } from '@agent-device/platform-apple/runner'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { activateCompleteRefFrame, expireRefFrame } from '@agent-device/daemon/testing/ref-frame'; -import { STALE_SNAPSHOT_REFS_WARNING } from '@agent-device/daemon/testing/session-snapshot'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { - getRuntimeBindings, - mockFillPoint, - resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - makeSession, - makeStaleRefSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { runInteraction } from './interaction-touch-runner.ts'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -// How fill is admitted and projected: its @ref admission and pins, the editable -// node it keeps, and the guards that refuse before typing. - -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -async function withRunner(operation: () => Promise): Promise { - return await withAppleRunnerProvider(mockRunAppleRunnerCommand, { deviceId: 'sim-1' }, operation); -} - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - -beforeEach(() => { - resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); -}); - -test('fill @ref fails fast when the target is off-screen', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'fill-offscreen-ref'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - depth: 0, - type: 'Window', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - rect: { x: 20, y: 1180, width: 180, height: 44 }, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - const response = await withRunner(() => - handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'fill', - positionals: ['@e2', 'hello@example.com'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }), - ); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(false); - expect(mockFillPoint).not.toHaveBeenCalled(); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/off-screen/i); - // #1366: direction-named, selector-first recovery hint (see press test above). - expect(response.error.hint).toMatch(/scroll down/i); - expect(response.error.hint).toMatch(/selector/i); - expect(response.error.details?.reason).toBe('offscreen_ref'); - expect(response.error.details?.scrollDirection).toBe('down'); - } -}); - -test('fill @ref fails closed when stored ref bounds are invalid (ADR 0014)', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.device = { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel 8 Pro', - kind: 'emulator', - booted: true, - }; - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'android.widget.EditText', - label: 'Email', - rect: { x: 20, y: 40, width: Number.NaN, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'android', - }; - sessionStore.publish(sessionName, session); - - mockFillPoint.mockRejectedValue( - new Error('dispatch must not run: no positional recapture on unusable frame evidence'), - ); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'fill', - positionals: ['@e1', 'hello@example.com'], - flags: { delayMs: 25 }, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - // ADR 0014: the authorized frame's @e1 has an unusable rect, so the fill FAILS - // rather than recapturing a fresh tree and filling the same index. - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/Ref @e1 has no usable bounds/); - expect(response.error.details).toMatchObject({ reason: 'target_bounds_invalid', ref: 'e1' }); - } - expect(mockFillPoint).not.toHaveBeenCalled(); -}); - -test('fill @ref rejects after a device action expired the frame', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'stale-ref-fill'; - const session = makeStaleRefSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - rect: { x: 10, y: 20, width: 200, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - // ADR 0014: an unobserved device action expired the frame. - expireRefFrame(session); - sessionStore.publish(sessionName, session); - mockFillPoint.mockRejectedValue( - new Error('dispatch should not be called for an expired-frame ref'), - ); - - const response = await runInteraction(sessionStore, sessionName, 'fill', [ - '@e1', - 'hello@example.com', - ]); - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/expired ref frame/); - expect(response.error.details?.reason).toBe('ref_frame_expired'); - expect(response.error.details?.hint).toBe(STALE_SNAPSHOT_REFS_WARNING); - } - expect(mockFillPoint).not.toHaveBeenCalled(); -}); - -test('fill with a pinned stale ref rejects; pinned current is clean', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'pinned-fill'; - const session = makeStaleRefSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - rect: { x: 10, y: 20, width: 200, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - session.snapshotGeneration = 3; - // Re-issue the frame over the overridden snapshot so its source tree matches. - activateCompleteRefFrame(session); - sessionStore.publish(sessionName, session); - - const stale = await runInteraction(sessionStore, sessionName, 'fill', ['@e1~s2', 'hello']); - expect(stale?.ok).toBe(false); - if (stale && !stale.ok) { - expect(stale.error.code).toBe('COMMAND_FAILED'); - expect(stale.error.details?.hint).toBe( - "Ref @e1 was minted from snapshot s2 but the session's ref frame is now s3 — re-run snapshot -i.", - ); - } - expect(mockFillPoint).not.toHaveBeenCalled(); - - const current = await runInteraction(sessionStore, sessionName, 'fill', ['@e1~s3', 'hello']); - expect(current?.ok).toBe(true); - if (current?.ok) { - expect(current.data?.warning).toBeUndefined(); - } -}); - -test("ADR 0014 blocker-2: a mutating find's internal fill from an expired frame carries no stale-ref warning", async () => { - // Removing the coarse marker (step 8) means an expired frame now derives read - // staleness. A mutating `find fill` re-resolves the locator itself and re-enters - // the fill leaf with a LOCATOR-minted ref (`internal.findResolvedTarget`) — the - // caller never consumed a `@ref`, so the public find response must not claim - // stale refs even though the frame is expired. - const sessionStore = makeSessionStore(); - const sessionName = 'find-internal-fill-expired'; - const session = makeStaleRefSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - rect: { x: 10, y: 20, width: 200, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - // Re-issue the frame over the overridden snapshot, then expire it as if a prior - // device side effect changed the screen. - activateCompleteRefFrame(session); - expireRefFrame(session); - sessionStore.publish(sessionName, session); - mockFillPoint.mockResolvedValue({ filled: true }); - - // Contrast: a user-supplied `@ref` against the expired frame rejects before - // dispatch (it consumed a stale ref). - const userRef = await runInteraction(sessionStore, sessionName, 'fill', ['@e1', 'hello']); - expect(userRef?.ok).toBe(false); - if (userRef && !userRef.ok) { - expect(userRef.error.details?.reason).toBe('ref_frame_expired'); - } - - // The mutating find's internal dispatch (the exact request find.ts hands the - // leaf) bypasses admission AND attaches no stale-ref warning. - const findNodes = session.snapshot.nodes; - const findNode = findNodes[0]!; - const internal = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'fill', - positionals: ['@e1', 'hello'], - flags: {}, - internal: { - findResolvedTarget: { ref: '@e1', node: findNode, tree: { nodes: findNodes } }, - }, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - expect(internal?.ok).toBe(true); - if (internal?.ok) { - expect(internal.data?.warning).toBeUndefined(); - } -}); - -test('fill @ref keeps the original editable node when its parent is the hittable ancestor', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeCell', - label: 'Email row', - rect: { x: 20, y: 100, width: 320, height: 72 }, - enabled: true, - hittable: true, - }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - identifier: 'auth_email', - rect: { x: 44, y: 120, width: 200, height: 32 }, - enabled: true, - hittable: false, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - mockFillPoint.mockResolvedValue({ filled: true }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'fill', - positionals: ['@e2', 'hello@example.com'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - expect(mockFillPoint).toHaveBeenCalledWith( - expect.objectContaining({ - point: { x: 144, y: 136 }, - text: 'hello@example.com', - }), - ); - - const stored = sessionStore.get(sessionName); - const result = (stored?.actions[0]?.result ?? {}) as Record; - expect(result.ref).toBe('e2'); -}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-press.test.ts b/src/__tests__/daemon-composition/interaction/interaction-touch-press.test.ts deleted file mode 100644 index 3cb67024c7..0000000000 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-press.test.ts +++ /dev/null @@ -1,459 +0,0 @@ -import { test, expect, vi, beforeEach } from 'vitest'; -import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { - getRuntimeBindings, - mockTapPoint, - resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - findResolvedTarget, - makeFindPreresolvedTree, - makeMacOsMenubarSession, - makeSession, - makeStaleRefSession, - readPressPoint, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { runFindInternalClick } from './interaction-touch-runner.ts'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -// Which node an admitted press/click acts on: hittable-ancestor promotion, the -// macOS menubar wrapper, button projection, and a mutating find's pre-resolved -// node (#1654). - -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - -beforeEach(() => { - resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); -}); - -test('click on a macOS menubar wrapper ref promotes to the same-rect menu bar item', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'macos-menubar-wrapper-ref-click'; - const session = makeMacOsMenubarSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - depth: 0, - type: 'MenuBarSurface', - label: 'Menu Bar', - surface: 'menubar', - rect: { x: 0, y: 0, width: 1512, height: 982 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'MenuBar', - rect: { x: 989, y: 4.5, width: 29, height: 24 }, - hittable: true, - surface: 'menubar', - }, - { - index: 2, - depth: 2, - parentIndex: 1, - type: 'MenuBarItem', - rect: { x: 989, y: 4.5, width: 29, height: 24 }, - hittable: true, - surface: 'menubar', - }, - ]), - createdAt: Date.now(), - backend: 'macos-helper', - }; - sessionStore.publish(sessionName, session); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'click', - positionals: ['@e2'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(readPressPoint(mockTapPoint)).toEqual(['1004', '17']); - if (response?.ok) { - expect(response.data?.selectorChain).toEqual(['role="menubaritem"']); - } -}); - -test('press @ref promotes a non-hittable node to its hittable ancestor before tapping', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - parentIndex: 2, - type: 'XCUIElementTypeCell', - label: 'Settings row', - rect: { x: 20, y: 100, width: 320, height: 72 }, - enabled: true, - hittable: true, - }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeStaticText', - label: 'Settings', - rect: { x: 44, y: 124, width: 84, height: 20 }, - enabled: false, - hittable: false, - }, - { - index: 2, - type: 'XCUIElementTypeApplication', - rect: { x: 0, y: 0, width: 390, height: 844 }, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - mockTapPoint.mockResolvedValue({ pressed: true }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e2'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - if (response?.ok) { - expect(response.data?.ref).toBe('e2'); - expect(response.data?.x).toBe(180); - expect(response.data?.y).toBe(136); - // Promotion landed on a hittable ancestor, so there is nothing to flag. - expect(response.data?.targetHittable).toBeUndefined(); - expect(response.data?.hint).toBeUndefined(); - } - expect(readPressPoint(mockTapPoint)).toEqual(['180', '136']); - - const stored = sessionStore.get(sessionName); - const result = (stored?.actions[0]?.result ?? {}) as Record; - expect(result.ref).toBe('e2'); - expect(Array.isArray(result.selectorChain)).toBe(true); -}); - -test('press @ref does not promote to a full-screen hittable ancestor', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeWindow', - rect: { x: 0, y: 0, width: 402, height: 874 }, - enabled: true, - hittable: true, - }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeCell', - label: 'General', - rect: { x: 16, y: 293, width: 370, height: 52 }, - enabled: true, - hittable: false, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - mockTapPoint.mockResolvedValue({ pressed: true }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e2'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - if (response?.ok) { - expect(response.data?.x).toBe(201); - expect(response.data?.y).toBe(319); - // #1037: the press still proceeds against the non-hittable node (no stricter - // resolution), but the daemon response flags it so agents can react instead - // of assuming the tap had a visible effect. - expect(response.data?.targetHittable).toBe(false); - expect(typeof response.data?.hint).toBe('string'); - expect(response.data?.hint as string).toMatch(/hittable: false/); - } - expect(readPressPoint(mockTapPoint)).toEqual(['201', '319']); -}); - -test('click --button secondary on @ref dispatches a secondary press on macOS and records click', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.device = { - platform: 'apple', - appleOs: 'macos', - id: 'macos-desktop', - name: 'My Mac', - kind: 'device', - booted: true, - }; - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeCell', - label: 'failed-step.json', - rect: { x: 400, y: 500, width: 200, height: 20 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - mockTapPoint.mockResolvedValue({ button: 'secondary' }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'click', - positionals: ['@e1'], - flags: { clickButton: 'secondary' }, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - expect(readPressPoint(mockTapPoint)).toEqual(['500', '510']); - expect(mockTapPoint.mock.calls[0]?.[0].options.button).toBe('secondary'); - if (response?.ok) { - expect(response.data?.button).toBe('secondary'); - expect(response.data?.ref).toBe('e1'); - } - - const stored = sessionStore.get(sessionName); - expect(stored).toBeTruthy(); - expect(stored?.actions[0]?.command).toBe('click'); - expect(stored?.actions[0]?.flags.clickButton).toBe('secondary'); -}); - -test('#1654: a mutating find click acts on the node find resolved, not a re-resolved @ref', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'find-preresolved-click'; - sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); - const preresolved = makeFindPreresolvedTree(); - - const response = await runFindInternalClick(sessionStore, sessionName, { - findResolvedTarget: findResolvedTarget(preresolved), - }); - - expect(response?.ok).toBe(true); - // find's node — (300,500,20,20) → (310, 510). A second resolution would read - // @e2 out of the SESSION frame tree instead and tap (60, 40). - expect(readPressPoint(mockTapPoint)).toEqual(['310', '510']); -}); - -test('#1654 control: without the resolved-target payload the same click still resolves @ref from the session tree', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'find-preresolved-control'; - sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); - - const response = await runFindInternalClick(sessionStore, sessionName, {}); - - // The ordinary ref path is untouched: @e1 is "Continue" at (10,20,100,40). - expect(response?.ok).toBe(true); - expect(readPressPoint(mockTapPoint)).toEqual(['60', '40']); -}); - -test('#1654: the leaf performs no ref lookup at all — a session that could not resolve @e2 still acts', async () => { - // Defensive proof of the structural contract: this synthetic session has no - // stored snapshot, so any second resolution necessarily fails ("Ref @e2 not - // found or has no bounds"). Production find normally stores its capture; - // this test proves lookup absence without claiming that missing state occurs. - const sessionStore = makeSessionStore(); - const sessionName = 'find-preresolved-no-session-tree'; - const session = makeSession(sessionName); - session.snapshot = undefined; - sessionStore.publish(sessionName, session); - const preresolved = makeFindPreresolvedTree(); - - const withoutPreresolution = await runFindInternalClick(sessionStore, sessionName, {}); - expect(withoutPreresolution?.ok).toBe(false); - - const response = await runFindInternalClick(sessionStore, sessionName, { - findResolvedTarget: findResolvedTarget(preresolved), - }); - - expect(response?.ok).toBe(true); - expect(readPressPoint(mockTapPoint)).toEqual(['310', '510']); -}); - -test('#1654: the shared guards still run on the pre-resolved node', async () => { - // The pre-resolution replaces the lookup, not the guarantees: an occluded - // node is still refused, at the same ADR 0011 `runtime-ref` occlusion cell. - const sessionStore = makeSessionStore(); - const sessionName = 'find-preresolved-occluded'; - sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); - const preresolved = makeFindPreresolvedTree(); - const blocked = { - ...preresolved.node, - interactionBlocked: { by: 'overlay', reason: 'covered' as const }, - }; - - const response = await runFindInternalClick(sessionStore, sessionName, { - findResolvedTarget: { - ref: '@e2', - node: blocked, - tree: { nodes: [preresolved.nodes[0]!, blocked] }, - }, - }); - - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toContain('covered by another visible element'); - } - expect(readPressPoint(mockTapPoint)).toBeUndefined(); -}); - -test('a read right after a press captures the post-tap screen instead of reusing the pre-tap tree', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'read-after-press'; - sessionStore.publish( - sessionName, - makeIosSession(sessionName, { appBundleId: 'com.example.app' }), - ); - const screen = (step: string) => ({ - backend: 'xctest' as const, - producer: 'apple-runner' as const, - nodes: [ - { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 393, height: 852 } }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Next', - rect: { x: 24, y: 120, width: 120, height: 44 }, - enabled: true, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'StaticText', - label: step, - identifier: 'step', - rect: { x: 24, y: 220, width: 320, height: 24 }, - }, - ], - }); - mockCaptureSnapshotForSession - .mockResolvedValueOnce(screen('Step 1')) - .mockResolvedValue(screen('Step 2')); - const run = async (command: string, positionals: string[]) => - await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { token: 't', session: sessionName, command, positionals, flags: {} }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(await run('is', ['visible', 'label=Next'])).toMatchObject({ ok: true }); - expect(await run('press', ['84', '142'])).toMatchObject({ ok: true }); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); - - const read = await run('get', ['text', 'id="step"']); - - expect(read).toMatchObject({ ok: true, data: { text: 'Step 2' } }); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(2); -}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-runner.ts b/src/__tests__/daemon-composition/interaction/interaction-touch-runner.ts deleted file mode 100644 index c540cc58ba..0000000000 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-runner.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { getRuntimeBindings } from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { contextFromFlags } from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; -import type { SessionStore } from '@agent-device/daemon/testing/session-store'; - -/** - * The real command-surface legs the relocated interaction suites drive through the handler. - * The daemon-policy suites keep their fake surface and the shared factories in - * `interaction-touch-fixtures.ts`; running the real command surface is the composition these - * suites moved here to pin, so the runners live beside them. - */ -export async function runInteraction( - sessionStore: SessionStore, - sessionName: string, - command: string, - positionals: string[], - flags: Record = {}, -) { - return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { token: 't', session: sessionName, command, positionals, flags }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); -} - -export async function runFindInternalClick( - sessionStore: SessionStore, - sessionName: string, - internal: Record, -) { - return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'click', - positionals: ['@e2'], - flags: { noRecord: true }, - internal, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); -} diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-runtime.test.ts b/src/__tests__/daemon-composition/interaction/interaction-touch-runtime.test.ts deleted file mode 100644 index 839ada4962..0000000000 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-runtime.test.ts +++ /dev/null @@ -1,447 +0,0 @@ -import { test, expect, vi, beforeEach } from 'vitest'; -import { attachRefs } from '@agent-device/kernel/snapshot'; -import { withAppleRunnerProvider } from '@agent-device/platform-apple/runner'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { - elementReadFixtureState, - getRuntimeBindings, - mockQuerySelector, - mockTapPoint, - resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - makeSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -// What the shared runtime dispatch does with the resolved target: refuse -// unusable frame evidence rather than recapture positionally (ADR 0014), refuse -// off-screen targets, and record the `@ref` the caller wrote. - -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -async function withRunner(operation: () => Promise): Promise { - return await withAppleRunnerProvider(mockRunAppleRunnerCommand, { deviceId: 'sim-1' }, operation); -} - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - -beforeEach(() => { - resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); -}); - -test('press @ref taps the resolved point once and records the ref', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'press-ref'; - const session = makeSession(sessionName); - const nodes = attachRefs([ - { - index: 0, - type: 'XCUIElementTypeButton', - label: 'Continue', - identifier: 'auth_continue', - rect: { x: 10, y: 20, width: 100, height: 40 }, - enabled: true, - hittable: true, - }, - ]); - session.snapshot = { nodes, createdAt: Date.now(), backend: 'xctest' }; - sessionStore.publish(sessionName, session); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e1'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(mockTapPoint).toHaveBeenCalledTimes(1); - expect(mockTapPoint.mock.calls[0]?.[0]?.point).toEqual({ x: 60, y: 40 }); - const stored = sessionStore.get(sessionName); - expect(stored?.actions[0]?.positionals).toEqual(['@e1']); - expect(stored?.actions[0]?.flags).toEqual({}); -}); - -test('press @ref fails closed when the authorized ref has no usable bounds (ADR 0014)', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'stale-ref-refresh'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeButton', - label: 'Continue', - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - mockTapPoint.mockRejectedValue( - new Error('dispatch must not run: no positional recapture on missing frame evidence'), - ); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e1'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - // ADR 0014: the authorized frame's @e1 has no usable rect, so the ref FAILS - // rather than recapturing and accepting the same index from a newer tree by - // positional coincidence. No fresh capture, no dispatch. - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/Ref @e1 has no usable bounds/); - expect(response.error.details).toMatchObject({ reason: 'target_bounds_invalid', ref: 'e1' }); - } - expect(mockTapPoint).not.toHaveBeenCalled(); -}); - -test('press @ref fails closed when stored ref bounds are invalid (ADR 0014)', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.device = { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel 8 Pro', - kind: 'emulator', - booted: true, - }; - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'android.widget.TextView', - label: 'My App', - rect: { x: 20, y: 40, width: Number.NaN, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'android', - }; - sessionStore.publish(sessionName, session); - - mockTapPoint.mockRejectedValue( - new Error('dispatch must not run: no positional recapture on unusable frame evidence'), - ); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e1'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - // ADR 0014: the authorized frame's @e1 has an unusable rect (NaN), so it FAILS - // rather than recapturing and accepting the same index from a newer tree. - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/Ref @e1 has no usable bounds/); - expect(response.error.details).toMatchObject({ reason: 'target_bounds_invalid', ref: 'e1' }); - } - expect(mockTapPoint).not.toHaveBeenCalled(); -}); - -// #1542 wiring, daemon route: the off-screen rescue's live read rides the SAME request-bound -// binding the touch command was admitted for. These two tests pin both declared sides of the -// CONDITIONAL `querySelector` fact: an owner that advertises it answers the guard with a live -// rect the tap lands on; an owner that does not refuses on the bulk-tree verdict with ZERO -// queries. Deleting `conditional: ['querySelector']` from the captured touch uses, or making the -// rescue hook return undefined, breaks one of these two; the Apple binding's own spread is pinned -// in platform-apple's runtime.test.ts. - -const RESCUE_ROUTE_SESSION = 'rescue-route'; - -function offscreenRefSessionStore() { - const sessionStore = makeSessionStore(); - const session = makeSession(RESCUE_ROUTE_SESSION); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - depth: 0, - type: 'Window', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeButton', - label: 'Far item', - identifier: 'far_item', - rect: { x: 20, y: 1200, width: 120, height: 44 }, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(RESCUE_ROUTE_SESSION, session); - return sessionStore; -} - -async function pressOffscreenRef(sessionStore: ReturnType) { - return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: RESCUE_ROUTE_SESSION, - command: 'press', - positionals: ['@e2'], - flags: {}, - }, - sessionName: RESCUE_ROUTE_SESSION, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); -} - -test('press @ref an off-screen node is rescued to the live rect when the owner advertises querySelector', async () => { - const sessionStore = offscreenRefSessionStore(); - elementReadFixtureState.querySelectorAvailable = true; - mockQuerySelector.mockResolvedValue({ - found: true, - node: { rect: { x: 20, y: 400, width: 120, height: 44 }, hittable: true }, - }); - - const response = await pressOffscreenRef(sessionStore); - - // The rescue wins: the press lands at the CENTER OF THE LIVE RECT, not the stale - // bulk-tree row's off-screen geometry, and the live read carried the derived selector. - expect(response?.ok).toBe(true); - expect(mockQuerySelector).toHaveBeenCalledTimes(1); - expect(mockQuerySelector.mock.calls[0]?.[0].selector).toEqual({ - key: 'id', - value: 'far_item', - }); - expect(mockTapPoint).toHaveBeenCalledTimes(1); - expect(mockTapPoint.mock.calls[0]?.[0]?.point).toEqual({ x: 80, y: 422 }); -}); - -test('press @ref an off-screen node refuses with zero queries when the owner does not advertise querySelector', async () => { - // The fixture default: the owner's facts declare the CONDITIONAL read unavailable, so the - // route holds no hook and the guard's refusal is built from the bulk tree alone. - const sessionStore = offscreenRefSessionStore(); - - const response = await pressOffscreenRef(sessionStore); - - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.details?.reason).toBe('offscreen_ref'); - } - expect(mockQuerySelector).not.toHaveBeenCalled(); - expect(mockTapPoint).not.toHaveBeenCalled(); -}); - -test('press @ref fails fast when the target is off-screen', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'press-offscreen-ref'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - depth: 0, - type: 'Window', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeButton', - label: 'Far item', - rect: { x: 20, y: 1200, width: 120, height: 44 }, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - const response = await withRunner(() => - handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e2'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }), - ); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(false); - expect(mockTapPoint).not.toHaveBeenCalled(); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/off-screen/i); - // #1366: the hint names the concrete scroll direction and steers to a selector-based retry - // (a @ref would be rejected as expired after the scroll). A ref refusal has no selector to - // check between passes, so this hint names the move and leaves the stop condition to the - // selector retry it prescribes. - expect(response.error.hint).toMatch(/scroll down/i); - expect(response.error.hint).toMatch(/selector/i); - expect(response.error.details?.reason).toBe('offscreen_ref'); - expect(response.error.details?.scrollDirection).toBe('down'); - } -}); - -test('press @ref with a trailing label recovers within the authorized frame (no positional recapture)', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.device = { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel 8 Pro', - kind: 'emulator', - booted: true, - }; - // @e1's rect is unusable, but the SAME frame tree carries another node with - // the trailing label at usable bounds. ADR 0014 label recovery resolves within - // the authorized frame — never by recapturing a fresh tree. - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'android.widget.TextView', - label: 'Different', - rect: { x: 20, y: 40, width: Number.NaN, height: 40 }, - enabled: true, - hittable: true, - }, - { - index: 1, - type: 'android.widget.TextView', - label: 'My App', - rect: { x: 100, y: 200, width: 80, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'android', - }; - sessionStore.publish(sessionName, session); - - mockCaptureSnapshotForSession.mockRejectedValue( - new Error('no positional recapture: recovery stays in-frame'), - ); - mockTapPoint.mockResolvedValue({ pressed: true }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e1', 'My App'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - expect(mockTapPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 140, y: 220 } })); - if (response?.ok) { - expect(response.data?.x).toBe(140); - expect(response.data?.y).toBe(220); - expect(response.data?.resolution).toEqual({ - source: 'ref', - phase: 'pre-action', - kind: 'label-fallback', - }); - } -}); diff --git a/src/commands/interaction/runtime/interactions-preresolved-target.test.ts b/src/commands/interaction/runtime/interactions-preresolved-target.test.ts new file mode 100644 index 0000000000..9bd4677dca --- /dev/null +++ b/src/commands/interaction/runtime/interactions-preresolved-target.test.ts @@ -0,0 +1,164 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Point, RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { PreresolvedInteractionTarget } from '@agent-device/contracts/interaction'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { ref } from './selector-read-utils.ts'; +import { createLocalArtifactAdapter } from '../../../io.ts'; +import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +import { createAgentDevice } from '../../../runtime.ts'; + +// #1654: a mutating `find` hands the interaction leaf the node it already resolved, so the leaf +// adopts that node instead of resolving the same `@ref` a second time, while the shared guards +// keep running on it. + +const SESSION_TREE: RawSnapshotNode[] = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Cancel', + rect: { x: 10, y: 80, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, +]; + +// The tree a mutating `find` matched against, deliberately NOT the tree stored on the session. +// `@e2` names the "Continue" button in both trees, but at a distinctive point here and at (60, 40) +// in the session tree, so the tap coordinates say which tree the leaf actually resolved against. +function findResolved(): PreresolvedInteractionTarget & { node: SnapshotNode } { + const nodes = makeSnapshotState([ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + rect: { x: 300, y: 500, width: 20, height: 20 }, + enabled: true, + hittable: true, + }, + ]).nodes; + return { ref: '@e2', node: nodes[1]!, tree: { nodes } }; +} + +function deviceWithSessionTree(snapshot: ReturnType | undefined) { + const taps: Point[] = []; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: snapshot ?? makeSnapshotState([]) }), + tap: async (_context, point) => { + taps.push(point); + }, + }, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + }); + return { device, taps }; +} + +test('runtime click acts on the node find resolved, not a re-resolved @ref', async () => { + const { device, taps } = deviceWithSessionTree(makeSnapshotState(SESSION_TREE)); + + const result = await device.interactions.click(ref('@e2'), { + session: 'default', + preresolvedTarget: findResolved(), + }); + + assert.equal(result.kind, 'ref'); + // find's node (300,500,20,20) -> (310, 510). A second resolution would read @e2 out of the + // SESSION tree instead and tap (60, 40). + assert.deepEqual(taps, [{ x: 310, y: 510 }]); +}); + +test('runtime click without the resolved-target payload still resolves @ref from the session tree', async () => { + const { device, taps } = deviceWithSessionTree(makeSnapshotState(SESSION_TREE)); + + await device.interactions.click(ref('@e2'), { session: 'default' }); + + // The ordinary ref path is untouched: @e2 is "Continue" at (10,20,100,40). + assert.deepEqual(taps, [{ x: 60, y: 40 }]); +}); + +test('runtime click with a resolved-target payload performs no ref lookup at all', async () => { + // This synthetic session has no stored snapshot, so any second resolution necessarily fails + // ("Ref @e2 not found or has no bounds"). Production find normally stores its capture; this + // proves lookup absence without claiming that missing state occurs. + const { device, taps } = deviceWithSessionTree(undefined); + + await assert.rejects(() => device.interactions.click(ref('@e2'), { session: 'default' })); + assert.deepEqual(taps, []); + + await device.interactions.click(ref('@e2'), { + session: 'default', + preresolvedTarget: findResolved(), + }); + + assert.deepEqual(taps, [{ x: 310, y: 510 }]); +}); + +test('runtime click refuses a resolved-target payload whose ref provenance disagrees', async () => { + const { device, taps } = deviceWithSessionTree(makeSnapshotState(SESSION_TREE)); + + await assert.rejects( + () => + device.interactions.click(ref('@e2'), { + session: 'default', + preresolvedTarget: { ...findResolved(), ref: '@e9' }, + }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /provenance does not match/); + return true; + }, + ); + assert.deepEqual(taps, []); +}); + +test('runtime click keeps the shared guards running on a pre-resolved node', async () => { + // The pre-resolution replaces the lookup, not the guarantees: an occluded node is still refused, + // at the same ADR 0011 `runtime-ref` occlusion cell. + const { device, taps } = deviceWithSessionTree(makeSnapshotState(SESSION_TREE)); + const resolved = findResolved(); + const blocked: SnapshotNode = { + ...resolved.node, + interactionBlocked: 'covered', + }; + + await assert.rejects( + () => + device.interactions.click(ref('@e2'), { + session: 'default', + preresolvedTarget: { + ref: '@e2', + node: blocked, + tree: { nodes: [resolved.tree.nodes[0]!, blocked] }, + }, + }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /covered by another visible element/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_covered'); + assert.equal(details?.dispatched, 'no'); + return true; + }, + ); + assert.deepEqual(taps, []); +}); diff --git a/src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts b/src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts new file mode 100644 index 0000000000..0ee355f146 --- /dev/null +++ b/src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts @@ -0,0 +1,202 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Point } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { ref } from './selector-read-utils.ts'; +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; + +// What a `@ref` interaction does with the frame evidence it is handed (ADR 0014): refuse unusable +// bounds rather than recapture positionally, refuse off-screen targets with a direction, and +// recover a label only inside the authorized frame. + +function androidTextViewSnapshot(type: string, label: string) { + return makeSnapshotState( + [ + { + index: 0, + type, + label, + rect: { x: 20, y: 40, width: Number.NaN, height: 40 }, + enabled: true, + hittable: true, + }, + ], + { backend: 'android' }, + ); +} + +function offscreenSnapshot(type: string, label: string, y: number) { + return makeSnapshotState([ + { index: 0, depth: 0, type: 'Window', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type, + label, + rect: { x: 20, y, width: 180, height: 44 }, + hittable: true, + }, + ]); +} + +function assertBoundsInvalid(error: unknown): boolean { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /Ref @e1 has no usable bounds/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.ref, 'e1'); + return true; +} + +function assertOffscreenRefusal(error: unknown): boolean { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /off-screen/i); + // #1366: the hint names the concrete scroll direction and steers to a selector-based retry (a + // @ref would be rejected as expired after the scroll). + const details = (error as { details?: Record }).details; + const hint = String(details?.hint ?? ''); + assert.match(hint, /scroll down/i); + assert.match(hint, /selector/i); + assert.equal(details?.reason, 'offscreen_ref'); + assert.equal(details?.scrollDirection, 'down'); + return true; +} + +test('runtime press @ref fails closed when stored ref bounds are invalid (ADR 0014)', async () => { + const taps: Point[] = []; + let captures = 0; + const snapshot = androidTextViewSnapshot('android.widget.TextView', 'My App'); + const device = createInteractionDevice(snapshot, { + platform: 'android', + captureSnapshot: async () => { + captures += 1; + return { snapshot }; + }, + tap: async (_context, point) => { + taps.push(point); + }, + }); + + await assert.rejects( + () => device.interactions.press(ref('@e1'), { session: 'default' }), + assertBoundsInvalid, + ); + assert.deepEqual(taps, []); + assert.equal(captures, 0, 'a ref with invalid bounds is refused without a positional recapture'); +}); + +test('runtime fill @ref fails closed when stored ref bounds are invalid (ADR 0014)', async () => { + const fills: Point[] = []; + let captures = 0; + const snapshot = androidTextViewSnapshot('android.widget.EditText', 'Email'); + const device = createInteractionDevice(snapshot, { + platform: 'android', + captureSnapshot: async () => { + captures += 1; + return { snapshot }; + }, + fill: async (_context, point) => { + fills.push(point); + }, + }); + + await assert.rejects( + () => + device.interactions.fill(ref('@e1'), 'hello@example.com', { + session: 'default', + delayMs: 25, + }), + assertBoundsInvalid, + ); + assert.deepEqual(fills, []); + assert.equal(captures, 0, 'a ref with invalid bounds is refused without a positional recapture'); +}); + +test('runtime press @ref fails fast when the target is off-screen', async () => { + const taps: Point[] = []; + const device = createInteractionDevice( + offscreenSnapshot('XCUIElementTypeButton', 'Far item', 1200), + { + tap: async (_context, point) => { + taps.push(point); + }, + }, + ); + + await assert.rejects( + () => device.interactions.press(ref('@e2'), { session: 'default' }), + assertOffscreenRefusal, + ); + assert.deepEqual(taps, []); +}); + +test('runtime fill @ref fails fast when the target is off-screen', async () => { + const fills: Point[] = []; + const device = createInteractionDevice( + offscreenSnapshot('XCUIElementTypeTextField', 'Email', 1180), + { + fill: async (_context, point) => { + fills.push(point); + }, + }, + ); + + await assert.rejects( + () => device.interactions.fill(ref('@e2'), 'hello@example.com', { session: 'default' }), + assertOffscreenRefusal, + ); + assert.deepEqual(fills, []); +}); + +test('runtime press @ref with a trailing label recovers within the authorized frame (no positional recapture)', async () => { + // @e1's rect is unusable, but the SAME frame tree carries another node with the trailing label at + // usable bounds. ADR 0014 label recovery resolves within the authorized frame, never by + // recapturing a fresh tree. + const snapshot = makeSnapshotState( + [ + { + index: 0, + type: 'android.widget.TextView', + label: 'Different', + rect: { x: 20, y: 40, width: Number.NaN, height: 40 }, + enabled: true, + hittable: true, + }, + { + index: 1, + type: 'android.widget.TextView', + label: 'My App', + rect: { x: 100, y: 200, width: 80, height: 40 }, + enabled: true, + hittable: true, + }, + ], + { backend: 'android' }, + ); + const taps: Point[] = []; + const device = createInteractionDevice(snapshot, { + platform: 'android', + captureSnapshot: async () => { + throw new Error('no positional recapture: recovery stays in-frame'); + }, + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.press(ref('@e1', { fallbackLabel: 'My App' }), { + session: 'default', + }); + + assert.deepEqual(taps, [{ x: 140, y: 220 }]); + assert.equal(result.kind, 'ref'); + assert.deepEqual(result.point, { x: 140, y: 220 }); + assert.deepEqual(result.resolution, { + source: 'ref', + phase: 'pre-action', + kind: 'label-fallback', + }); +}); diff --git a/src/commands/interaction/runtime/interactions-target-promotion.test.ts b/src/commands/interaction/runtime/interactions-target-promotion.test.ts new file mode 100644 index 0000000000..ffaa657559 --- /dev/null +++ b/src/commands/interaction/runtime/interactions-target-promotion.test.ts @@ -0,0 +1,176 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Point } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { ref } from './selector-read-utils.ts'; +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; + +// Which node an admitted press/click acts on: hittable-ancestor promotion, the macOS menubar +// wrapper, and the point the action lands on. + +test('runtime click on a macOS menubar wrapper ref promotes to the same-rect menu bar item', async () => { + const snapshot = makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'MenuBarSurface', + label: 'Menu Bar', + surface: 'menubar', + rect: { x: 0, y: 0, width: 1512, height: 982 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'MenuBar', + rect: { x: 989, y: 4.5, width: 29, height: 24 }, + hittable: true, + surface: 'menubar', + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'MenuBarItem', + rect: { x: 989, y: 4.5, width: 29, height: 24 }, + hittable: true, + surface: 'menubar', + }, + ], + { backend: 'macos-helper' }, + ); + const taps: Point[] = []; + const device = createInteractionDevice(snapshot, { + platform: 'macos', + sessionMetadata: { surface: 'menubar' }, + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.click(ref('@e2'), { session: 'default' }); + + assert.deepEqual(taps, [{ x: 1004, y: 17 }]); + assert.equal(result.kind, 'ref'); + assert.deepEqual(result.selectorChain, ['role="menubaritem"']); +}); + +test('runtime press @ref promotes a non-hittable node to its hittable ancestor before tapping', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + parentIndex: 2, + type: 'XCUIElementTypeCell', + label: 'Settings row', + rect: { x: 20, y: 100, width: 320, height: 72 }, + enabled: true, + hittable: true, + }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Settings', + rect: { x: 44, y: 124, width: 84, height: 20 }, + enabled: false, + hittable: false, + }, + { + index: 2, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, + ]); + const taps: Point[] = []; + const device = createInteractionDevice(snapshot, { + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.press(ref('@e2'), { session: 'default' }); + + assert.deepEqual(taps, [{ x: 180, y: 136 }]); + assert.equal(result.kind, 'ref'); + assert.deepEqual(result.target, { kind: 'ref', ref: '@e2' }); + // Promotion landed on a hittable ancestor, so there is nothing to flag. + assert.equal(result.targetHittable, undefined); + assert.equal(result.hint, undefined); + assert.ok(Array.isArray(result.selectorChain)); +}); + +test('runtime press @ref does not promote to a full-screen hittable ancestor', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + type: 'XCUIElementTypeWindow', + rect: { x: 0, y: 0, width: 402, height: 874 }, + enabled: true, + hittable: true, + }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeCell', + label: 'General', + rect: { x: 16, y: 293, width: 370, height: 52 }, + enabled: true, + hittable: false, + }, + ]); + const taps: Point[] = []; + const device = createInteractionDevice(snapshot, { + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.press(ref('@e2'), { session: 'default' }); + + assert.deepEqual(taps, [{ x: 201, y: 319 }]); + // #1037: the press still proceeds against the non-hittable node (no stricter resolution), but the + // result flags it so agents can react instead of assuming the tap had a visible effect. + assert.equal(result.kind, 'ref'); + assert.equal(result.targetHittable, false); + assert.match(result.hint ?? '', /hittable: false/); +}); + +test('runtime fill @ref keeps the original editable node when its parent is the hittable ancestor', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + type: 'XCUIElementTypeCell', + label: 'Email row', + rect: { x: 20, y: 100, width: 320, height: 72 }, + enabled: true, + hittable: true, + }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + identifier: 'auth_email', + rect: { x: 44, y: 120, width: 200, height: 32 }, + enabled: true, + hittable: false, + }, + ]); + const fills: Array<{ point: Point; text: string }> = []; + const device = createInteractionDevice(snapshot, { + fill: async (_context, point, text) => { + fills.push({ point, text }); + }, + }); + + const result = await device.interactions.fill(ref('@e2'), 'hello@example.com', { + session: 'default', + }); + + assert.deepEqual(fills, [{ point: { x: 144, y: 136 }, text: 'hello@example.com' }]); + assert.equal(result.kind, 'ref'); + assert.deepEqual(result.target, { kind: 'ref', ref: '@e2' }); + assert.equal(result.node?.identifier, 'auth_email'); +}); diff --git a/src/commands/interaction/runtime/selector-is-visibility.test.ts b/src/commands/interaction/runtime/selector-is-visibility.test.ts new file mode 100644 index 0000000000..2a6dfbe1e7 --- /dev/null +++ b/src/commands/interaction/runtime/selector-is-visibility.test.ts @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { selector } from './selector-read-utils.ts'; +import { createSelectorDevice } from './__tests__/test-utils/index.ts'; + +// What `is visible` decides about a matched node: it inherits the viewport visibility of an +// ancestor that has a rect, and a node outside the viewport fails the assertion. + +test('runtime is visible passes for list text that inherits viewport visibility from an ancestor', async () => { + const snapshot = makeSnapshotState( + [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeCell', + rect: { x: 0, y: 160, width: 390, height: 44 }, + hittable: false, + }, + { + index: 2, + parentIndex: 1, + type: 'XCUIElementTypeStaticText', + label: 'Trip ideas', + hittable: false, + }, + ], + { backend: 'xctest', producer: 'apple-runner' }, + ); + const device = createSelectorDevice(snapshot); + + const result = await device.selectors.isVisible(selector('label="Trip ideas"'), { + session: 'default', + }); + + assert.equal(result.predicate, 'visible'); + assert.equal(result.pass, true); + assert.equal(result.selector, 'label="Trip ideas"'); +}); + +test('runtime is visible fails for nodes outside the current viewport', async () => { + const snapshot = makeSnapshotState( + [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Far item', + rect: { x: 20, y: 2600, width: 120, height: 40 }, + hittable: false, + }, + ], + { backend: 'xctest', producer: 'apple-runner' }, + ); + const device = createSelectorDevice(snapshot); + + await assert.rejects( + () => device.selectors.isVisible(selector('label="Far item"'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /actual=\{"visible":false/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'predicate_failed'); + assert.equal(details?.predicate, 'visible'); + return true; + }, + ); +}); diff --git a/src/commands/interaction/runtime/selector-read-disambiguation.test.ts b/src/commands/interaction/runtime/selector-read-disambiguation.test.ts new file mode 100644 index 0000000000..0b4f38a53e --- /dev/null +++ b/src/commands/interaction/runtime/selector-read-disambiguation.test.ts @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { selector } from './selector-read-utils.ts'; +import { createSelectorDevice } from './__tests__/test-utils/index.ts'; + +// A label selector that matches several nodes of one row resolves through the captured tree's +// disambiguation, never through a runner query. + +test('runtime get text on an iOS label selector disambiguates among same-label nodes in one capture', async () => { + const snapshot = makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 393, height: 852 }, + enabled: true, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Cell', + label: 'General', + rect: { x: 0, y: 100, width: 393, height: 48 }, + enabled: true, + hittable: true, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'General', + rect: { x: 0, y: 100, width: 393, height: 48 }, + enabled: true, + hittable: true, + }, + { + index: 3, + depth: 2, + parentIndex: 1, + type: 'StaticText', + label: 'General', + rect: { x: 18, y: 112, width: 80, height: 24 }, + enabled: true, + hittable: false, + }, + ], + { backend: 'xctest', producer: 'apple-runner' }, + ); + let captures = 0; + const device = createSelectorDevice(snapshot, { + captureSnapshot: () => { + captures += 1; + return { snapshot }; + }, + }); + + const result = await device.selectors.get({ + session: 'default', + property: 'text', + target: selector('label="General"'), + }); + + assert.equal(result.kind, 'text'); + assert.equal(result.text, 'General'); + assert.notEqual(result.node.type, 'Cell'); + assert.deepEqual(result.target, { kind: 'selector', selector: 'label="General"' }); + assert.equal(captures, 1); +});