From 32231668a9f41cfd237faad1d808ceb50188542a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:33:13 +0200 Subject: [PATCH 01/13] test(daemon): add the interactor-resolution fixture and the touch command surface fake The interactor-resolution fixture replaces the snapshot-interactor-capture module mock: captures reach one spy through the installed InteractorResolution. The touch surface fake dispatches resolved targets through the runtime config's backend and reads the session through its store, so daemon suites pin admission, bound-operation wiring, response shaping, and capture wiring without the real command family. Co-Authored-By: Claude Opus 5.5 --- .../support/interactor-resolution-fixture.ts | 44 ++ .../support/touch-command-surface-fake.ts | 458 ++++++++++++++++++ .../support/touch-interaction-runner.ts | 60 +++ 3 files changed, 562 insertions(+) create mode 100644 packages/daemon/src/__tests__/support/interactor-resolution-fixture.ts create mode 100644 packages/daemon/src/__tests__/support/touch-command-surface-fake.ts create mode 100644 packages/daemon/src/__tests__/support/touch-interaction-runner.ts diff --git a/packages/daemon/src/__tests__/support/interactor-resolution-fixture.ts b/packages/daemon/src/__tests__/support/interactor-resolution-fixture.ts new file mode 100644 index 0000000000..b234198a5a --- /dev/null +++ b/packages/daemon/src/__tests__/support/interactor-resolution-fixture.ts @@ -0,0 +1,44 @@ +import { vi } from 'vitest'; +import type { + Interactor, + RunnerContext, + SnapshotOptions, + SnapshotResult, +} from '@agent-device/contracts/interactor-types'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { installInteractorResolution } from '../../core/interactor-resolution.ts'; + +type InteractorSnapshotCall = Readonly<{ + device: DeviceInfo; + runnerContext: RunnerContext; + options: SnapshotOptions; +}>; + +/** + * The snapshot the resolved interactor answers with. Every capture that falls back to + * `captureSnapshotWithInteractor` lands here with the `{ device, runnerContext, options }` the + * production seam hands the interactor, so a suite configures captures and reads their call + * arguments from one spy. + */ +export const mockInteractorSnapshot = vi.fn< + (call: InteractorSnapshotCall) => Promise +>(async () => ({ + nodes: [], + backend: 'xctest', + producer: 'apple-runner', +})); + +/** + * Installs an interactor resolution whose interactor snapshots through + * {@link mockInteractorSnapshot}. Call from `beforeEach`; the spy keeps its configuration until the + * suite resets it. + */ +export function installInteractorSnapshotResolution(): void { + installInteractorResolution({ + resolve: async (device, runnerContext) => + ({ + snapshot: async (options: SnapshotOptions) => + await mockInteractorSnapshot({ device, runnerContext, options }), + }) as unknown as Interactor, + }); +} diff --git a/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts new file mode 100644 index 0000000000..a652e26df2 --- /dev/null +++ b/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts @@ -0,0 +1,458 @@ +import type { + CreateDaemonCommandSurface, + DaemonCommandSurface, +} from '@agent-device/contracts/daemon-command-surface'; +import type { + AgentDeviceBackend, + BackendActionResult, + BackendCommandContext, + BackendSnapshotOptions, +} from '@agent-device/contracts/backend'; +import type { + InteractionEvidence, + InteractionTarget, + PreresolvedInteractionTarget, + ResolvedInteractionTarget, + SettleObservation, + SettleParams, +} from '@agent-device/contracts/interaction'; +import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; +import { attachResolvedInteractionTarget } from '@agent-device/host-kit/interaction-outcome'; +import { AppError } from '@agent-device/kernel/errors'; +import type { Point, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; +import { extractReadableText } from '@agent-device/capture-kit/text-surface'; +import { buildSelectorChainForNode } from '@agent-device/selectors'; +import { fakeCreateCommandSurface } from './command-surface-fake.ts'; + +/** + * A command surface that answers the touch actions the way the real command family does at the two + * seams the daemon owns: it dispatches a resolved target through the runtime config's backend, and + * it reads the session through the config's session store. Suites that pin what the daemon does + * around a touch — admission, bound-operation wiring, response shaping, recording, settle capture + * wiring — need both, because the daemon's decisions land on those seams. + * + * Target resolution is deliberately the smallest rule that lets a dispatch happen: a point is + * itself, a `@ref` is the centre of the node the session's authorized frame (or latest + * observation) names, and a selector is the first node of one fresh capture that matches one of its `||` alternatives + * (each a conjunction of `key=value` terms). Promotion, occlusion, off-screen and bounds guards, label recovery, and + * `--verify` evidence are command-family behavior and are asserted beside that code. + */ + +/** The runtime configs this surface was assembled with, oldest first. */ +export const touchSurfaceConfigs: AgentDeviceRuntimeConfig[] = []; + +export function resetTouchSurfaceFake(): void { + touchSurfaceConfigs.length = 0; +} + +type CommandOptions = Readonly<{ + session?: string; + requestId?: string; + signal?: AbortSignal; + settle?: SettleParams; + verify?: boolean; + preresolvedTarget?: PreresolvedInteractionTarget; +}>; + +export const touchCreateCommandSurface: CreateDaemonCommandSurface = async (config) => { + touchSurfaceConfigs.push(config); + const inert = await fakeCreateCommandSurface(config); + return { + ...inert, + signal: config.signal, + selectors: touchSelectors(config, inert), + interactions: touchInteractions(config, inert), + }; +}; + +function touchSelectors( + config: AgentDeviceRuntimeConfig, + inert: DaemonCommandSurface, +): DaemonCommandSurface['selectors'] { + return { + ...inert.selectors, + get: async (options) => { + const resolved = await resolveTarget(config, options.target, options, READ_CAPTURE_OPTIONS); + if (resolved.kind === 'point' || !resolved.node) { + throw new AppError('COMMAND_FAILED', 'get target did not resolve to a node'); + } + return { + kind: options.property, + target: resolved.target, + node: resolved.node, + selectorChain: resolved.selectorChain, + preActionNodes: resolved.preAction?.nodes ?? [], + ...(options.property === 'text' + ? { text: await readNodeText(config, resolved.node, options) } + : {}), + } as never; + }, + is: async (options) => { + const resolved = await resolveTarget( + config, + { kind: 'selector', selector: options.selector }, + options, + READ_CAPTURE_OPTIONS, + ); + return { + predicate: options.predicate, + pass: true, + selector: options.selector, + matches: 1, + selectorChain: resolved.kind === 'point' ? undefined : resolved.selectorChain, + node: resolved.kind === 'point' ? undefined : resolved.node, + preActionNodes: resolved.kind === 'point' ? undefined : resolved.preAction?.nodes, + }; + }, + }; +} + +async function readNodeText( + config: AgentDeviceRuntimeConfig, + node: SnapshotNode, + options: CommandOptions, +): Promise { + const read = await config.backend.readText?.(context(config, options), node); + return read?.text.trim() ? read.text : extractReadableText(node); +} + +function touchInteractions( + config: AgentDeviceRuntimeConfig, + inert: DaemonCommandSurface, +): DaemonCommandSurface['interactions'] { + const press = async ( + target: InteractionTarget, + options: CommandOptions & { button?: unknown; [key: string]: unknown } = {}, + ) => { + const resolved = await resolveTarget(config, target, options); + const backendResult = await dispatch(resolved, () => dispatchTap(config, resolved, options)); + return await finish(config, resolved, options, { backendResult }); + }; + return { + ...inert.interactions, + click: press, + press, + gesture: async (options) => await dragOrInert(config, inert, options), + fill: async (target, text, options = {}) => { + const resolved = await resolveTarget(config, target, options); + const backendResult = await dispatch(resolved, () => + dispatchFill(config, resolved, text, options), + ); + return await finish(config, resolved, options, { backendResult, text }); + }, + longPress: async (target, options = {}) => { + const resolved = await resolveTarget(config, target, options); + const backendResult = await dispatch(resolved, () => + backendOperation(config.backend.longPress?.bind(config.backend))( + context(config, options), + requirePointOf(resolved), + { durationMs: options.durationMs }, + ), + ); + return await finish(config, resolved, options, { + backendResult, + durationMs: options.durationMs, + }); + }, + hover: async (target, options = {}) => { + const resolved = await resolveTarget(config, target, options); + const backendResult = await dispatch(resolved, () => + dispatchHover(config, resolved, options), + ); + return await finish(config, resolved, options, { backendResult }); + }, + }; +} + +/** + * A target-authored drag resolves both endpoints like a press target, performs ONE plan through + * the backend, and reports the endpoints and the recording side channel the daemon stores. Every + * other gesture keeps the inert answer. + */ +async function dragOrInert( + config: AgentDeviceRuntimeConfig, + inert: DaemonCommandSurface, + options: Parameters[0], +) { + const gesture = options.gesture; + if (!('intent' in gesture) || gesture.intent !== 'drag') + return await inert.interactions.gesture(options); + const commandOptions = { session: options.session, requestId: options.requestId }; + const [source, destination] = await Promise.all( + [gesture.source, gesture.destination].map( + async (expression) => + await resolveTarget( + config, + expression.startsWith('@') + ? { kind: 'ref', ref: expression } + : { kind: 'selector', selector: expression }, + commandOptions, + ), + ), + ); + const from = requirePointOf(source!); + const to = requirePointOf(destination!); + const durationMs = + (gesture.sourceHoldMs ?? 0) + (gesture.moveMs ?? 0) + (gesture.destinationHoldMs ?? 0); + const backendResult = await backendOperation(config.backend.performGesture?.bind(config.backend))( + context(config, commandOptions), + { topology: 'single', intent: 'drag', durationMs } as never, + ); + const endpoint = (resolved: ResolvedInteractionTarget) => { + if (resolved.kind === 'point' || !resolved.node || !resolved.selectorChain) { + throw new AppError('COMMAND_FAILED', 'drag endpoint did not resolve to a node'); + } + return { + selectorChain: resolved.selectorChain, + node: resolved.node, + preActionNodes: resolved.preAction?.nodes ?? [], + }; + }; + const sourceTarget = endpoint(source!); + const destinationTarget = endpoint(destination!); + const exact = { source: 'ref', phase: 'pre-action', kind: 'exact' } as const; + return { + kind: 'drag', + durationMs, + pointerCount: 1, + from, + to, + backendResult: backendResult ?? undefined, + targets: { + source: { selectorChain: sourceTarget.selectorChain, resolution: exact }, + destination: { selectorChain: destinationTarget.selectorChain, resolution: exact }, + }, + recording: { + sourceSelector: sourceTarget.selectorChain.join(' || '), + destinationSelector: destinationTarget.selectorChain.join(' || '), + sourceTarget, + destinationTarget, + }, + } as const; +} + +/** What a read (`get`, `is`) asks its capture for: the whole tree, with rects. */ +const READ_CAPTURE_OPTIONS = { interactiveOnly: false, includeRects: true } as const; + +async function resolveTarget( + config: AgentDeviceRuntimeConfig, + target: InteractionTarget, + options: CommandOptions, + captureOptions: BackendSnapshotOptions = { interactiveOnly: true }, +): Promise { + if (target.kind === 'point') return { kind: 'point', point: { x: target.x, y: target.y } }; + const pre = options.preresolvedTarget; + if (pre && target.kind === 'ref') return resolvedFromNode(config, target, pre.node, pre.tree); + if (target.kind === 'ref') { + if (hasNativeRefRoute(config.backend)) return { kind: 'ref', target }; + const record = await config.sessions?.get(options.session ?? 'default'); + const tree = record?.refFrameSnapshot ?? record?.snapshot; + const node = tree?.nodes.find((candidate) => `@${candidate.ref}` === refBody(target.ref)); + if (!tree || !node) throw new AppError('COMMAND_FAILED', `Ref ${target.ref} not found`); + return resolvedFromNode(config, target, node, tree); + } + const captured = await config.backend.captureSnapshot?.(context(config, options), captureOptions); + const tree = captured?.snapshot; + const node = tree?.nodes.find((candidate) => matchesSelector(candidate, target.selector)); + if (!tree || !node) { + throw new AppError('COMMAND_FAILED', `Selector ${target.selector} did not match`); + } + return { + kind: 'selector', + target: { kind: 'selector', selector: target.selector }, + point: centerOf(node) as Point, + node, + selectorChain: buildSelectorChainForNode(node, config.backend.platform), + preAction: { nodes: tree.nodes }, + }; +} + +function resolvedFromNode( + config: AgentDeviceRuntimeConfig, + target: Extract, + node: SnapshotNode, + tree: Pick, +): ResolvedInteractionTarget { + return { + kind: 'ref', + target: { kind: 'ref', ref: target.ref }, + point: centerOf(node), + node, + selectorChain: buildSelectorChainForNode(node, config.backend.platform), + preAction: { nodes: tree.nodes }, + }; +} + +function hasNativeRefRoute(backend: AgentDeviceBackend): boolean { + return Boolean(backend.tapTarget ?? backend.hoverTarget ?? backend.fillTarget); +} + +function refBody(ref: string): string { + return ref.startsWith('@') ? ref : `@${ref}`; +} + +function centerOf(node: SnapshotNode): Point | undefined { + const rect = node.rect; + if (!rect) return undefined; + return { x: Math.round(rect.x + rect.width / 2), y: Math.round(rect.y + rect.height / 2) }; +} + +/** A selector chain: `||` alternatives, each a conjunction of `key=value` terms. */ +function matchesSelector(node: SnapshotNode, selector: string): boolean { + const actual: Record = { + id: node.identifier, + label: node.label, + text: node.label ?? node.value, + value: node.value, + role: node.type?.replace(/^XCUIElementType/i, '').toLowerCase(), + }; + return selector.split(' || ').some((alternative) => { + const terms = [...alternative.matchAll(/(\w+)=(?:"((?:[^"\\]|\\.)*)"|(\S+))/g)]; + return ( + terms.length > 0 && terms.every(([, key, quoted, bare]) => actual[key!] === (quoted ?? bare)) + ); + }); +} + +function context(config: AgentDeviceRuntimeConfig, options: CommandOptions): BackendCommandContext { + return { + session: options.session, + requestId: options.requestId, + signal: options.signal ?? config.signal, + }; +} + +async function dispatch( + resolved: ResolvedInteractionTarget, + run: () => Promise, +): Promise { + try { + return await run(); + } catch (error) { + attachResolvedInteractionTarget(error, resolved); + throw error; + } +} + +function backendOperation(operation: TOperation | undefined): TOperation { + if (!operation) { + throw new AppError('UNSUPPORTED_OPERATION', 'backend does not support the action'); + } + return operation; +} + +function requirePointOf(resolved: ResolvedInteractionTarget): Point { + if (resolved.kind === 'point') return resolved.point; + if (!resolved.point) throw new AppError('COMMAND_FAILED', 'resolved target has no point'); + return resolved.point; +} + +async function dispatchTap( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + options: CommandOptions & { button?: unknown }, +): Promise { + const tapOptions = options as Parameters>[2]; + if (resolved.kind === 'ref' && !resolved.point && config.backend.tapTarget) { + return await config.backend.tapTarget(context(config, options), resolved.target, tapOptions); + } + return await backendOperation(config.backend.tap?.bind(config.backend))( + context(config, options), + requirePointOf(resolved), + tapOptions, + ); +} + +async function dispatchFill( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + text: string, + options: CommandOptions & { delayMs?: number }, +): Promise { + const fillOptions = { + delayMs: options.delayMs, + allowNonHittableCoordinateFallback: ( + options as { allowNonHittableCoordinateFallback?: boolean } + ).allowNonHittableCoordinateFallback, + }; + if (resolved.kind === 'ref' && !resolved.point && config.backend.fillTarget) { + return await config.backend.fillTarget( + context(config, options), + resolved.target, + text, + fillOptions, + ); + } + return await backendOperation(config.backend.fill?.bind(config.backend))( + context(config, options), + requirePointOf(resolved), + text, + fillOptions, + ); +} + +async function dispatchHover( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + options: CommandOptions, +): Promise { + if (resolved.kind === 'ref' && !resolved.point && config.backend.hoverTarget) { + return await config.backend.hoverTarget(context(config, options), resolved.target); + } + return await backendOperation(config.backend.hover?.bind(config.backend))( + context(config, options), + requirePointOf(resolved), + ); +} + +async function finish>( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + options: CommandOptions, + extra: TExtra, +) { + const evidence = options.verify ? await verifyEvidence(config, resolved, options) : undefined; + const settle = options.settle ? await settleObservation(config, resolved, options) : undefined; + return { + ...resolved, + ...extra, + ...(evidence ? { evidence } : {}), + ...(settle ? { settle } : {}), + } as never; +} + +async function verifyEvidence( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + options: CommandOptions, +): Promise { + const after = await config.backend.captureSnapshot?.(context(config, options), { + interactiveOnly: true, + }); + const nodes = after?.snapshot?.nodes ?? []; + const before = resolved.kind === 'point' ? undefined : resolved.preAction?.nodes; + return { + nodeCount: nodes.length, + interactiveNodeCount: nodes.length, + digest: `fake-digest-${nodes.length}`, + changedFromBefore: before === undefined || before.length !== nodes.length, + }; +} + +async function settleObservation( + config: AgentDeviceRuntimeConfig, + resolved: ResolvedInteractionTarget, + options: CommandOptions, +): Promise { + const captures = resolved.kind === 'point' ? 2 : 1; + for (let capture = 0; capture < captures; capture += 1) { + await config.backend.captureSnapshot?.(context(config, options), { interactiveOnly: true }); + } + return { + settled: true, + waitedMs: 0, + captures, + quietMs: options.settle?.quietMs ?? 0, + timeoutMs: options.settle?.timeoutMs ?? 0, + }; +} diff --git a/packages/daemon/src/__tests__/support/touch-interaction-runner.ts b/packages/daemon/src/__tests__/support/touch-interaction-runner.ts new file mode 100644 index 0000000000..902ebd2ea6 --- /dev/null +++ b/packages/daemon/src/__tests__/support/touch-interaction-runner.ts @@ -0,0 +1,60 @@ +import type { CommandFlags } from '@agent-device/contracts/command'; +import { getRuntimeBindings } from '../../execution/__tests__/interaction-get-runtime-fixture.ts'; +import { handleInteractionCommands } from '../../execution/interaction/index.ts'; +import type { SessionStore } from '../../core/session-store.ts'; +import { touchCreateCommandSurface } from './touch-command-surface-fake.ts'; + +/** The request-flag projection the interaction handler's runtime context is built from. */ +const contextFromFlags = (flags: CommandFlags | undefined) => ({ + count: flags?.count, + intervalMs: flags?.intervalMs, + delayMs: flags?.delayMs, + holdMs: flags?.holdMs, + jitterPx: flags?.jitterPx, + doubleTap: flags?.doubleTap, + clickButton: flags?.clickButton, +}); + +/** + * One interaction request through `handleInteractionCommands` on the touch command surface fake + * and the shared runtime-binding fixture, so suites state only the request they are pinning. + */ +export async function runInteraction( + sessionStore: SessionStore, + sessionName: string, + command: string, + positionals: string[], + flags: Record = {}, +) { + return await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { token: 't', session: sessionName, command, positionals, flags }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); +} + +/** The re-entry a mutating `find` makes: a click carrying the node `find` already resolved. */ +export async function runFindInternalClick( + sessionStore: SessionStore, + sessionName: string, + internal: Record, +) { + return await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['@e2'], + flags: { noRecord: true }, + internal, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); +} From 05f11e70e9bbf804508395b6b3fb8abe1550b452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:33:19 +0200 Subject: [PATCH 02/13] refactor(daemon): place the touch router ownership tests beside the interaction handler interaction-touch.test.ts is (a) daemon logic: router claims press/click/longpress/hover/fill, single inspect/bind, macOS surface refusals, hover capability refusal. All 12 tests move to execution/interaction/internal/interaction-touch-routing.test.ts on the touch surface fake. Co-Authored-By: Claude Opus 5.5 --- .../interaction-touch-routing.test.ts | 314 ++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts new file mode 100644 index 0000000000..6690831aee --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts @@ -0,0 +1,314 @@ +import { test, expect, beforeEach } from 'vitest'; +import { attachRefs } from '@agent-device/kernel/snapshot'; +import { WEB_DESKTOP_DEVICE } from '@agent-device/test-kit/device-fixtures'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { + getRuntimeBindings, + mockHoverPoint, + mockHoverRef, + mockLongPressPoint, + mockTapPoint, + resetGetRuntimeFixture, + runtimeBindingSpies, +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { + contextFromFlags, + makeMacOsDesktopSession, + makeMacOsMenubarSession, + makeSession, +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; + +// Router ownership: one representative per touch command proves +// handleInteractionCommands claims press/click/longpress/hover/fill. + +beforeEach(() => { + resetGetRuntimeFixture(); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); +}); + +test('press coordinates dispatches press and records as press', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'default'; + const storedSession = makeSession(sessionName); + sessionStore.publish(sessionName, storedSession); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['100', '200'], + flags: { count: 3, intervalMs: 1, doubleTap: true }, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toBeTruthy(); + expect(response?.ok).toBe(true); + expect(mockTapPoint).toHaveBeenCalledOnce(); + expect(mockTapPoint.mock.calls[0]?.[0]).toMatchObject({ + point: { x: 100, y: 200 }, + options: { count: 3, intervalMs: 1, doubleTap: true }, + }); + + const session = sessionStore.get(sessionName); + expect(session).toBeTruthy(); + expect(session?.actions.length).toBe(1); + expect(session?.actions[0]?.command).toBe('press'); + expect(session?.actions[0]?.positionals).toEqual(['100', '200']); +}); + +test.each([ + ['click', ['100', '200'], false], + ['press', ['100', '200'], false], + ['longpress', ['100', '200', '800'], false], + ['hover', ['100', '200'], true], + ['fill', ['100', '200', 'hello'], false], +] as const)('%s inspects once and binds once', async (command, positionals, web) => { + const sessionStore = makeSessionStore(); + const sessionName = `single-bind-${command}`; + const session = makeSession(sessionName); + if (web) session.device = WEB_DESKTOP_DEVICE; + sessionStore.publish(sessionName, session); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { token: 't', session: sessionName, command, positionals: [...positionals], flags: {} }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(runtimeBindingSpies().inspectFacts).toHaveBeenCalledOnce(); + expect(runtimeBindingSpies().bindDevice).toHaveBeenCalledOnce(); +}); + +test('click rejects macOS desktop surface interactions until helper routing exists', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'macos-desktop-click'; + sessionStore.publish(sessionName, makeMacOsDesktopSession(sessionName)); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['100', '200'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.code).toBe('UNSUPPORTED_OPERATION'); + expect(response.error.message).toMatch(/macOS desktop sessions/); + } +}); + +test('fill rejects macOS menubar surface interactions until helper routing exists', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'macos-menubar-fill'; + sessionStore.publish(sessionName, makeMacOsMenubarSession(sessionName)); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['@e2', 'hello'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.code).toBe('UNSUPPORTED_OPERATION'); + expect(response.error.message).toMatch(/macOS menubar sessions/); + } +}); + +test('longpress @ref resolves the target and dispatches coordinate longpress', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'longpress-ref'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeStaticText', + label: 'Last message', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.publish(sessionName, session); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'longpress', + positionals: ['@e1', '800'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data?.x).toBe(60); + expect(response.data?.y).toBe(40); + expect(response.data?.durationMs).toBe(800); + expect(response.data?.message).toMatch(/Long pressed @e1/); + } + expect(mockLongPressPoint).toHaveBeenCalledWith( + expect.objectContaining({ point: { x: 60, y: 40 }, durationMs: 800 }), + ); + expect(sessionStore.get(sessionName)?.actions[0]?.command).toBe('longpress'); +}); + +// #1783: hover is the pointer-only member of the targeted-touch family. It rides +// the same admission path as press/longpress, and only web admits it. On web +// the session's ref frame is minted WITHOUT rects (snapshot -i does not fetch +// boxes), so `hover @ref` cannot resolve to coordinates: like `click @ref` it +// must take the provider-native route (`hoverRef`, ADR 0011 native-ref path). +// This test therefore stores a rect-less web frame and a scoped provider — the +// production shape — and asserts no coordinate dispatch happens. +test('hover @ref on web dispatches through the provider hoverRef route, not coordinates', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'hover-ref'; + const session = makeSession(sessionName); + session.device = WEB_DESKTOP_DEVICE; + session.snapshot = { + nodes: attachRefs([{ index: 0, role: 'link', label: 'Second message', enabled: true }]), + createdAt: Date.now(), + backend: 'web', + }; + sessionStore.publish(sessionName, session); + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'hover', + positionals: ['@e1'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toMatchObject({ + ok: true, + data: { ref: 'e1', gesture: 'hover', message: expect.stringMatching(/Hovered @e1/) }, + }); + expect(mockHoverRef).toHaveBeenCalledWith(expect.objectContaining({ ref: '@e1' })); + expect(mockHoverPoint).not.toHaveBeenCalled(); + expect(sessionStore.get(sessionName)?.actions[0]?.command).toBe('hover'); +}); + +test('hover selector on web resolves the target and dispatches coordinate hover', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'hover-selector'; + const session = makeSession(sessionName); + session.device = WEB_DESKTOP_DEVICE; + sessionStore.publish(sessionName, session); + mockInteractorSnapshot.mockResolvedValue({ + nodes: attachRefs([ + { + index: 0, + role: 'link', + label: 'Second message', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + backend: 'web', + producer: 'agent-browser', + }); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'hover', + positionals: ['label="Second message"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toMatchObject({ + ok: true, + data: { x: 60, y: 40, gesture: 'hover', message: expect.stringMatching(/Hovered label/) }, + }); + expect(mockHoverPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 60, y: 40 } })); +}); + +test('hover is refused by capability on touch platforms before any dispatch', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'hover-ios'; + sessionStore.publish(sessionName, makeSession(sessionName)); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'hover', + positionals: ['100', '200'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toMatchObject({ + ok: false, + error: { + code: 'UNSUPPORTED_OPERATION', + message: 'hover is not supported on this device', + hint: 'hover raises pointer hover state and is available on web targets only. On touch platforms use longpress for hold gestures.', + }, + }); +}); From a03389a88be98741536a12dfb87098dcf7e56719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:33:26 +0200 Subject: [PATCH 03/13] refactor(daemon): split press admission and press target tests by what each asserts interaction-touch-press-admission.test.ts: (a) 12 tests move to the daemon package on the touch surface fake; the #1654 provenance-mismatch test is (c) and moves to src/commands/interaction/runtime/interactions-preresolved-target.test.ts. interaction-touch-press.test.ts: (a) secondary-button projection, the find pre-resolved wiring (adopt, control, no lookup) and read-after-press capture stay in the daemon package; the macOS menubar wrapper promotion, hittable-ancestor promotion, full-screen non-promotion (interactions- target-promotion.test.ts) and the pre-resolved adopt/control/no-lookup/shared-guard cases (interactions-preresolved-target.test.ts) are (c) and run against the real command runtime. Co-Authored-By: Claude Opus 5.5 --- .../interaction-touch-press-admission.test.ts | 106 +--- .../internal/interaction-touch-press.test.ts | 201 ++++++++ .../interaction-touch-press.test.ts | 459 ------------------ .../interactions-preresolved-target.test.ts | 161 ++++++ .../interactions-target-promotion.test.ts | 176 +++++++ 5 files changed, 560 insertions(+), 543 deletions(-) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-touch-press-admission.test.ts (76%) create mode 100644 packages/daemon/src/execution/interaction/internal/interaction-touch-press.test.ts delete mode 100644 src/__tests__/daemon-composition/interaction/interaction-touch-press.test.ts create mode 100644 src/commands/interaction/runtime/interactions-preresolved-target.test.ts create mode 100644 src/commands/interaction/runtime/interactions-target-promotion.test.ts diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-press-admission.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts similarity index 76% rename from src/__tests__/daemon-composition/interaction/interaction-touch-press-admission.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts index db81737659..3c579684c2 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-press-admission.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts @@ -1,79 +1,35 @@ -import { test, expect, vi, beforeEach } from 'vitest'; +import { test, expect, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { activateCompleteRefFrame, refFrameState } from '@agent-device/daemon/testing/ref-frame'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { activateCompleteRefFrame, refFrameState } from '../../../core/ref-frame.ts'; +import { setSessionSnapshot, STALE_SNAPSHOT_REFS_WARNING } from '../../session-snapshot.ts'; import { - setSessionSnapshot, - STALE_SNAPSHOT_REFS_WARNING, -} from '@agent-device/daemon/testing/session-snapshot'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { runInteraction } from '../../../__tests__/support/touch-interaction-runner.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, - findResolvedTarget, - makeFindPreresolvedTree, makeSession, makeStaleRefSession, makeTwoButtonNodes, - readPressPoint, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { runFindInternalClick, runInteraction } from './interaction-touch-runner.ts'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // Whether a targeted touch may act, and on what: click-option and positional -// validation, and the ADR 0014 @ref rules (staleness, pins, frame expiry, -// locator provenance) that admit or reject before any device op. - -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); +// validation, and the ADR 0014 @ref rules (staleness, pins, frame expiry) that +// admit or reject before any device op. beforeEach(() => { resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test('click --button middle on macOS fails with an explicit unsupported-operation error', async () => { @@ -95,7 +51,7 @@ test('click --button middle on macOS fails with an explicit unsupported-operatio ); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -123,7 +79,7 @@ test('press coordinates does not treat extra trailing args as selector', async ( sessionStore.publish(sessionName, makeSession(sessionName)); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -143,30 +99,12 @@ test('press coordinates does not treat extra trailing args as selector', async ( expect(sessionStore.get(sessionName)?.actions.length).toBe(1); }); -test('#1654: the resolved-target payload fails closed when its ref provenance disagrees', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'find-preresolved-mismatched-ref'; - sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); - const preresolved = makeFindPreresolvedTree(); - - const response = await runFindInternalClick(sessionStore, sessionName, { - findResolvedTarget: findResolvedTarget(preresolved, '@e9'), - }); - - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toContain('provenance does not match'); - } - expect(readPressPoint(mockTapPoint)).toBeUndefined(); -}); - test('press selector then press @ref rejects refs that outlived the stored snapshot before dispatch', async () => { const sessionStore = makeSessionStore(); const sessionName = 'stale-ref-warns'; const session = makeStaleRefSession(sessionName); sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: makeTwoButtonNodes(), backend: 'xctest', producer: 'apple-runner', @@ -202,7 +140,7 @@ test('a ref press crosses the ADR 0014 side-effect seam and expires the ref fram const sessionName = 'seam-expiry'; const session = makeStaleRefSession(sessionName); sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: makeTwoButtonNodes(), backend: 'xctest', producer: 'apple-runner', @@ -225,7 +163,7 @@ test('ADR 0014 evidence #1: a second ref mutation rejects (bare and pinned) unti // A complete snapshot issued the frame at generation 500. activateCompleteRefFrame(session); sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: makeTwoButtonNodes(), backend: 'xctest', producer: 'apple-runner', @@ -270,7 +208,7 @@ test('re-issuing a complete frame lets press @ref succeed again without warning' const sessionName = 'reissued-refs-no-warning'; const session = makeStaleRefSession(sessionName); sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: makeTwoButtonNodes(), backend: 'xctest', producer: 'apple-runner', diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.test.ts new file mode 100644 index 0000000000..fa0e023f0b --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.test.ts @@ -0,0 +1,201 @@ +import { test, expect, beforeEach } from 'vitest'; +import { attachRefs } from '@agent-device/kernel/snapshot'; +import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { runFindInternalClick } from '../../../__tests__/support/touch-interaction-runner.ts'; +import { + getRuntimeBindings, + mockTapPoint, + resetGetRuntimeFixture, +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { + contextFromFlags, + findResolvedTarget, + makeFindPreresolvedTree, + makeSession, + makeStaleRefSession, + readPressPoint, +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; + +// How an admitted press/click reaches the device: the button projection onto the bound tap, a +// mutating find's pre-resolved node (#1654) handed to the command surface, and the capture a read +// takes after a press. + +beforeEach(() => { + resetGetRuntimeFixture(); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); +}); + +test('click --button secondary on @ref dispatches a secondary press on macOS and records click', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'default'; + const session = makeSession(sessionName); + session.device = { + platform: 'apple', + appleOs: 'macos', + id: 'macos-desktop', + name: 'My Mac', + kind: 'device', + booted: true, + }; + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeCell', + label: 'failed-step.json', + rect: { x: 400, y: 500, width: 200, height: 20 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.publish(sessionName, session); + + mockTapPoint.mockResolvedValue({ button: 'secondary' }); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['@e1'], + flags: { clickButton: 'secondary' }, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toBeTruthy(); + expect(response?.ok).toBe(true); + expect(readPressPoint(mockTapPoint)).toEqual(['500', '510']); + expect(mockTapPoint.mock.calls[0]?.[0].options.button).toBe('secondary'); + if (response?.ok) { + expect(response.data?.button).toBe('secondary'); + expect(response.data?.ref).toBe('e1'); + } + + const stored = sessionStore.get(sessionName); + expect(stored).toBeTruthy(); + expect(stored?.actions[0]?.command).toBe('click'); + expect(stored?.actions[0]?.flags.clickButton).toBe('secondary'); +}); + +test('#1654: a mutating find click acts on the node find resolved, not a re-resolved @ref', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'find-preresolved-click'; + sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); + const preresolved = makeFindPreresolvedTree(); + + const response = await runFindInternalClick(sessionStore, sessionName, { + findResolvedTarget: findResolvedTarget(preresolved), + }); + + expect(response?.ok).toBe(true); + // find's node — (300,500,20,20) → (310, 510). A second resolution would read + // @e2 out of the SESSION frame tree instead and tap (60, 40). + expect(readPressPoint(mockTapPoint)).toEqual(['310', '510']); +}); + +test('#1654 control: without the resolved-target payload the same click still resolves @ref from the session tree', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'find-preresolved-control'; + sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); + + const response = await runFindInternalClick(sessionStore, sessionName, {}); + + // The ordinary ref path is untouched: @e1 is "Continue" at (10,20,100,40). + expect(response?.ok).toBe(true); + expect(readPressPoint(mockTapPoint)).toEqual(['60', '40']); +}); + +test('#1654: the leaf performs no ref lookup at all — a session that could not resolve @e2 still acts', async () => { + // Defensive proof of the structural contract: this synthetic session has no + // stored snapshot, so any second resolution necessarily fails ("Ref @e2 not + // found or has no bounds"). Production find normally stores its capture; + // this test proves lookup absence without claiming that missing state occurs. + const sessionStore = makeSessionStore(); + const sessionName = 'find-preresolved-no-session-tree'; + const session = makeSession(sessionName); + session.snapshot = undefined; + sessionStore.publish(sessionName, session); + const preresolved = makeFindPreresolvedTree(); + + const withoutPreresolution = await runFindInternalClick(sessionStore, sessionName, {}); + expect(withoutPreresolution?.ok).toBe(false); + + const response = await runFindInternalClick(sessionStore, sessionName, { + findResolvedTarget: findResolvedTarget(preresolved), + }); + + expect(response?.ok).toBe(true); + expect(readPressPoint(mockTapPoint)).toEqual(['310', '510']); +}); + +test('a read right after a press captures the post-tap screen instead of reusing the pre-tap tree', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'read-after-press'; + sessionStore.publish( + sessionName, + makeIosSession(sessionName, { appBundleId: 'com.example.app' }), + ); + const screen = (step: string) => ({ + backend: 'xctest' as const, + producer: 'apple-runner' as const, + nodes: [ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 393, height: 852 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Next', + rect: { x: 24, y: 120, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'StaticText', + label: step, + identifier: 'step', + rect: { x: 24, y: 220, width: 320, height: 24 }, + }, + ], + }); + mockInteractorSnapshot + .mockResolvedValueOnce(screen('Step 1')) + .mockResolvedValue(screen('Step 2')); + const run = async (command: string, positionals: string[]) => + await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { token: 't', session: sessionName, command, positionals, flags: {} }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(await run('is', ['visible', 'label=Next'])).toMatchObject({ ok: true }); + expect(await run('press', ['84', '142'])).toMatchObject({ ok: true }); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); + + const read = await run('get', ['text', 'id="step"']); + + expect(read).toMatchObject({ ok: true, data: { text: 'Step 2' } }); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(2); +}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-press.test.ts b/src/__tests__/daemon-composition/interaction/interaction-touch-press.test.ts deleted file mode 100644 index 3cb67024c7..0000000000 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-press.test.ts +++ /dev/null @@ -1,459 +0,0 @@ -import { test, expect, vi, beforeEach } from 'vitest'; -import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { - getRuntimeBindings, - mockTapPoint, - resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - findResolvedTarget, - makeFindPreresolvedTree, - makeMacOsMenubarSession, - makeSession, - makeStaleRefSession, - readPressPoint, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { runFindInternalClick } from './interaction-touch-runner.ts'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -// Which node an admitted press/click acts on: hittable-ancestor promotion, the -// macOS menubar wrapper, button projection, and a mutating find's pre-resolved -// node (#1654). - -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - -beforeEach(() => { - resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); -}); - -test('click on a macOS menubar wrapper ref promotes to the same-rect menu bar item', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'macos-menubar-wrapper-ref-click'; - const session = makeMacOsMenubarSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - depth: 0, - type: 'MenuBarSurface', - label: 'Menu Bar', - surface: 'menubar', - rect: { x: 0, y: 0, width: 1512, height: 982 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'MenuBar', - rect: { x: 989, y: 4.5, width: 29, height: 24 }, - hittable: true, - surface: 'menubar', - }, - { - index: 2, - depth: 2, - parentIndex: 1, - type: 'MenuBarItem', - rect: { x: 989, y: 4.5, width: 29, height: 24 }, - hittable: true, - surface: 'menubar', - }, - ]), - createdAt: Date.now(), - backend: 'macos-helper', - }; - sessionStore.publish(sessionName, session); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'click', - positionals: ['@e2'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(readPressPoint(mockTapPoint)).toEqual(['1004', '17']); - if (response?.ok) { - expect(response.data?.selectorChain).toEqual(['role="menubaritem"']); - } -}); - -test('press @ref promotes a non-hittable node to its hittable ancestor before tapping', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - parentIndex: 2, - type: 'XCUIElementTypeCell', - label: 'Settings row', - rect: { x: 20, y: 100, width: 320, height: 72 }, - enabled: true, - hittable: true, - }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeStaticText', - label: 'Settings', - rect: { x: 44, y: 124, width: 84, height: 20 }, - enabled: false, - hittable: false, - }, - { - index: 2, - type: 'XCUIElementTypeApplication', - rect: { x: 0, y: 0, width: 390, height: 844 }, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - mockTapPoint.mockResolvedValue({ pressed: true }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e2'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - if (response?.ok) { - expect(response.data?.ref).toBe('e2'); - expect(response.data?.x).toBe(180); - expect(response.data?.y).toBe(136); - // Promotion landed on a hittable ancestor, so there is nothing to flag. - expect(response.data?.targetHittable).toBeUndefined(); - expect(response.data?.hint).toBeUndefined(); - } - expect(readPressPoint(mockTapPoint)).toEqual(['180', '136']); - - const stored = sessionStore.get(sessionName); - const result = (stored?.actions[0]?.result ?? {}) as Record; - expect(result.ref).toBe('e2'); - expect(Array.isArray(result.selectorChain)).toBe(true); -}); - -test('press @ref does not promote to a full-screen hittable ancestor', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeWindow', - rect: { x: 0, y: 0, width: 402, height: 874 }, - enabled: true, - hittable: true, - }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeCell', - label: 'General', - rect: { x: 16, y: 293, width: 370, height: 52 }, - enabled: true, - hittable: false, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - mockTapPoint.mockResolvedValue({ pressed: true }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e2'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - if (response?.ok) { - expect(response.data?.x).toBe(201); - expect(response.data?.y).toBe(319); - // #1037: the press still proceeds against the non-hittable node (no stricter - // resolution), but the daemon response flags it so agents can react instead - // of assuming the tap had a visible effect. - expect(response.data?.targetHittable).toBe(false); - expect(typeof response.data?.hint).toBe('string'); - expect(response.data?.hint as string).toMatch(/hittable: false/); - } - expect(readPressPoint(mockTapPoint)).toEqual(['201', '319']); -}); - -test('click --button secondary on @ref dispatches a secondary press on macOS and records click', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.device = { - platform: 'apple', - appleOs: 'macos', - id: 'macos-desktop', - name: 'My Mac', - kind: 'device', - booted: true, - }; - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeCell', - label: 'failed-step.json', - rect: { x: 400, y: 500, width: 200, height: 20 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - mockTapPoint.mockResolvedValue({ button: 'secondary' }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'click', - positionals: ['@e1'], - flags: { clickButton: 'secondary' }, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - expect(readPressPoint(mockTapPoint)).toEqual(['500', '510']); - expect(mockTapPoint.mock.calls[0]?.[0].options.button).toBe('secondary'); - if (response?.ok) { - expect(response.data?.button).toBe('secondary'); - expect(response.data?.ref).toBe('e1'); - } - - const stored = sessionStore.get(sessionName); - expect(stored).toBeTruthy(); - expect(stored?.actions[0]?.command).toBe('click'); - expect(stored?.actions[0]?.flags.clickButton).toBe('secondary'); -}); - -test('#1654: a mutating find click acts on the node find resolved, not a re-resolved @ref', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'find-preresolved-click'; - sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); - const preresolved = makeFindPreresolvedTree(); - - const response = await runFindInternalClick(sessionStore, sessionName, { - findResolvedTarget: findResolvedTarget(preresolved), - }); - - expect(response?.ok).toBe(true); - // find's node — (300,500,20,20) → (310, 510). A second resolution would read - // @e2 out of the SESSION frame tree instead and tap (60, 40). - expect(readPressPoint(mockTapPoint)).toEqual(['310', '510']); -}); - -test('#1654 control: without the resolved-target payload the same click still resolves @ref from the session tree', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'find-preresolved-control'; - sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); - - const response = await runFindInternalClick(sessionStore, sessionName, {}); - - // The ordinary ref path is untouched: @e1 is "Continue" at (10,20,100,40). - expect(response?.ok).toBe(true); - expect(readPressPoint(mockTapPoint)).toEqual(['60', '40']); -}); - -test('#1654: the leaf performs no ref lookup at all — a session that could not resolve @e2 still acts', async () => { - // Defensive proof of the structural contract: this synthetic session has no - // stored snapshot, so any second resolution necessarily fails ("Ref @e2 not - // found or has no bounds"). Production find normally stores its capture; - // this test proves lookup absence without claiming that missing state occurs. - const sessionStore = makeSessionStore(); - const sessionName = 'find-preresolved-no-session-tree'; - const session = makeSession(sessionName); - session.snapshot = undefined; - sessionStore.publish(sessionName, session); - const preresolved = makeFindPreresolvedTree(); - - const withoutPreresolution = await runFindInternalClick(sessionStore, sessionName, {}); - expect(withoutPreresolution?.ok).toBe(false); - - const response = await runFindInternalClick(sessionStore, sessionName, { - findResolvedTarget: findResolvedTarget(preresolved), - }); - - expect(response?.ok).toBe(true); - expect(readPressPoint(mockTapPoint)).toEqual(['310', '510']); -}); - -test('#1654: the shared guards still run on the pre-resolved node', async () => { - // The pre-resolution replaces the lookup, not the guarantees: an occluded - // node is still refused, at the same ADR 0011 `runtime-ref` occlusion cell. - const sessionStore = makeSessionStore(); - const sessionName = 'find-preresolved-occluded'; - sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); - const preresolved = makeFindPreresolvedTree(); - const blocked = { - ...preresolved.node, - interactionBlocked: { by: 'overlay', reason: 'covered' as const }, - }; - - const response = await runFindInternalClick(sessionStore, sessionName, { - findResolvedTarget: { - ref: '@e2', - node: blocked, - tree: { nodes: [preresolved.nodes[0]!, blocked] }, - }, - }); - - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toContain('covered by another visible element'); - } - expect(readPressPoint(mockTapPoint)).toBeUndefined(); -}); - -test('a read right after a press captures the post-tap screen instead of reusing the pre-tap tree', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'read-after-press'; - sessionStore.publish( - sessionName, - makeIosSession(sessionName, { appBundleId: 'com.example.app' }), - ); - const screen = (step: string) => ({ - backend: 'xctest' as const, - producer: 'apple-runner' as const, - nodes: [ - { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 393, height: 852 } }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Next', - rect: { x: 24, y: 120, width: 120, height: 44 }, - enabled: true, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'StaticText', - label: step, - identifier: 'step', - rect: { x: 24, y: 220, width: 320, height: 24 }, - }, - ], - }); - mockCaptureSnapshotForSession - .mockResolvedValueOnce(screen('Step 1')) - .mockResolvedValue(screen('Step 2')); - const run = async (command: string, positionals: string[]) => - await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { token: 't', session: sessionName, command, positionals, flags: {} }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(await run('is', ['visible', 'label=Next'])).toMatchObject({ ok: true }); - expect(await run('press', ['84', '142'])).toMatchObject({ ok: true }); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); - - const read = await run('get', ['text', 'id="step"']); - - expect(read).toMatchObject({ ok: true, data: { text: 'Step 2' } }); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(2); -}); diff --git a/src/commands/interaction/runtime/interactions-preresolved-target.test.ts b/src/commands/interaction/runtime/interactions-preresolved-target.test.ts new file mode 100644 index 0000000000..cda78e6147 --- /dev/null +++ b/src/commands/interaction/runtime/interactions-preresolved-target.test.ts @@ -0,0 +1,161 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Point, RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { PreresolvedInteractionTarget } from '@agent-device/contracts/interaction'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { ref } from './selector-read-utils.ts'; +import { createLocalArtifactAdapter } from '../../../io.ts'; +import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +import { createAgentDevice } from '../../../runtime.ts'; + +// #1654: a mutating `find` hands the interaction leaf the node it already resolved, so the leaf +// adopts that node instead of resolving the same `@ref` a second time, while the shared guards +// keep running on it. + +const SESSION_TREE: RawSnapshotNode[] = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Cancel', + rect: { x: 10, y: 80, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, +]; + +// The tree a mutating `find` matched against, deliberately NOT the tree stored on the session. +// `@e2` names the "Continue" button in both trees, but at a distinctive point here and at (60, 40) +// in the session tree, so the tap coordinates say which tree the leaf actually resolved against. +function findResolved(): PreresolvedInteractionTarget & { node: SnapshotNode } { + const nodes = makeSnapshotState([ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + rect: { x: 300, y: 500, width: 20, height: 20 }, + enabled: true, + hittable: true, + }, + ]).nodes; + return { ref: '@e2', node: nodes[1]!, tree: { nodes } }; +} + +function deviceWithSessionTree(snapshot: ReturnType | undefined) { + const taps: Point[] = []; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: snapshot ?? makeSnapshotState([]) }), + tap: async (_context, point) => { + taps.push(point); + }, + }, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + }); + return { device, taps }; +} + +test('runtime click acts on the node find resolved, not a re-resolved @ref', async () => { + const { device, taps } = deviceWithSessionTree(makeSnapshotState(SESSION_TREE)); + + const result = await device.interactions.click(ref('@e2'), { + session: 'default', + preresolvedTarget: findResolved(), + }); + + assert.equal(result.kind, 'ref'); + // find's node (300,500,20,20) -> (310, 510). A second resolution would read @e2 out of the + // SESSION tree instead and tap (60, 40). + assert.deepEqual(taps, [{ x: 310, y: 510 }]); +}); + +test('runtime click without the resolved-target payload still resolves @ref from the session tree', async () => { + const { device, taps } = deviceWithSessionTree(makeSnapshotState(SESSION_TREE)); + + await device.interactions.click(ref('@e2'), { session: 'default' }); + + // The ordinary ref path is untouched: @e2 is "Continue" at (10,20,100,40). + assert.deepEqual(taps, [{ x: 60, y: 40 }]); +}); + +test('runtime click with a resolved-target payload performs no ref lookup at all', async () => { + // This synthetic session has no stored snapshot, so any second resolution necessarily fails + // ("Ref @e2 not found or has no bounds"). Production find normally stores its capture; this + // proves lookup absence without claiming that missing state occurs. + const { device, taps } = deviceWithSessionTree(undefined); + + await assert.rejects(() => device.interactions.click(ref('@e2'), { session: 'default' })); + assert.deepEqual(taps, []); + + await device.interactions.click(ref('@e2'), { + session: 'default', + preresolvedTarget: findResolved(), + }); + + assert.deepEqual(taps, [{ x: 310, y: 510 }]); +}); + +test('runtime click refuses a resolved-target payload whose ref provenance disagrees', async () => { + const { device, taps } = deviceWithSessionTree(makeSnapshotState(SESSION_TREE)); + + await assert.rejects( + () => + device.interactions.click(ref('@e2'), { + session: 'default', + preresolvedTarget: { ...findResolved(), ref: '@e9' }, + }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /provenance does not match/); + return true; + }, + ); + assert.deepEqual(taps, []); +}); + +test('runtime click keeps the shared guards running on a pre-resolved node', async () => { + // The pre-resolution replaces the lookup, not the guarantees: an occluded node is still refused, + // at the same ADR 0011 `runtime-ref` occlusion cell. + const { device, taps } = deviceWithSessionTree(makeSnapshotState(SESSION_TREE)); + const resolved = findResolved(); + const blocked: SnapshotNode = { + ...resolved.node, + interactionBlocked: 'covered', + }; + + await assert.rejects( + () => + device.interactions.click(ref('@e2'), { + session: 'default', + preresolvedTarget: { + ref: '@e2', + node: blocked, + tree: { nodes: [resolved.tree.nodes[0]!, blocked] }, + }, + }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /covered by another visible element/); + return true; + }, + ); + assert.deepEqual(taps, []); +}); diff --git a/src/commands/interaction/runtime/interactions-target-promotion.test.ts b/src/commands/interaction/runtime/interactions-target-promotion.test.ts new file mode 100644 index 0000000000..ffaa657559 --- /dev/null +++ b/src/commands/interaction/runtime/interactions-target-promotion.test.ts @@ -0,0 +1,176 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Point } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { ref } from './selector-read-utils.ts'; +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; + +// Which node an admitted press/click acts on: hittable-ancestor promotion, the macOS menubar +// wrapper, and the point the action lands on. + +test('runtime click on a macOS menubar wrapper ref promotes to the same-rect menu bar item', async () => { + const snapshot = makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'MenuBarSurface', + label: 'Menu Bar', + surface: 'menubar', + rect: { x: 0, y: 0, width: 1512, height: 982 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'MenuBar', + rect: { x: 989, y: 4.5, width: 29, height: 24 }, + hittable: true, + surface: 'menubar', + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'MenuBarItem', + rect: { x: 989, y: 4.5, width: 29, height: 24 }, + hittable: true, + surface: 'menubar', + }, + ], + { backend: 'macos-helper' }, + ); + const taps: Point[] = []; + const device = createInteractionDevice(snapshot, { + platform: 'macos', + sessionMetadata: { surface: 'menubar' }, + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.click(ref('@e2'), { session: 'default' }); + + assert.deepEqual(taps, [{ x: 1004, y: 17 }]); + assert.equal(result.kind, 'ref'); + assert.deepEqual(result.selectorChain, ['role="menubaritem"']); +}); + +test('runtime press @ref promotes a non-hittable node to its hittable ancestor before tapping', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + parentIndex: 2, + type: 'XCUIElementTypeCell', + label: 'Settings row', + rect: { x: 20, y: 100, width: 320, height: 72 }, + enabled: true, + hittable: true, + }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Settings', + rect: { x: 44, y: 124, width: 84, height: 20 }, + enabled: false, + hittable: false, + }, + { + index: 2, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, + ]); + const taps: Point[] = []; + const device = createInteractionDevice(snapshot, { + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.press(ref('@e2'), { session: 'default' }); + + assert.deepEqual(taps, [{ x: 180, y: 136 }]); + assert.equal(result.kind, 'ref'); + assert.deepEqual(result.target, { kind: 'ref', ref: '@e2' }); + // Promotion landed on a hittable ancestor, so there is nothing to flag. + assert.equal(result.targetHittable, undefined); + assert.equal(result.hint, undefined); + assert.ok(Array.isArray(result.selectorChain)); +}); + +test('runtime press @ref does not promote to a full-screen hittable ancestor', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + type: 'XCUIElementTypeWindow', + rect: { x: 0, y: 0, width: 402, height: 874 }, + enabled: true, + hittable: true, + }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeCell', + label: 'General', + rect: { x: 16, y: 293, width: 370, height: 52 }, + enabled: true, + hittable: false, + }, + ]); + const taps: Point[] = []; + const device = createInteractionDevice(snapshot, { + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.press(ref('@e2'), { session: 'default' }); + + assert.deepEqual(taps, [{ x: 201, y: 319 }]); + // #1037: the press still proceeds against the non-hittable node (no stricter resolution), but the + // result flags it so agents can react instead of assuming the tap had a visible effect. + assert.equal(result.kind, 'ref'); + assert.equal(result.targetHittable, false); + assert.match(result.hint ?? '', /hittable: false/); +}); + +test('runtime fill @ref keeps the original editable node when its parent is the hittable ancestor', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + type: 'XCUIElementTypeCell', + label: 'Email row', + rect: { x: 20, y: 100, width: 320, height: 72 }, + enabled: true, + hittable: true, + }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + identifier: 'auth_email', + rect: { x: 44, y: 120, width: 200, height: 32 }, + enabled: true, + hittable: false, + }, + ]); + const fills: Array<{ point: Point; text: string }> = []; + const device = createInteractionDevice(snapshot, { + fill: async (_context, point, text) => { + fills.push({ point, text }); + }, + }); + + const result = await device.interactions.fill(ref('@e2'), 'hello@example.com', { + session: 'default', + }); + + assert.deepEqual(fills, [{ point: { x: 144, y: 136 }, text: 'hello@example.com' }]); + assert.equal(result.kind, 'ref'); + assert.deepEqual(result.target, { kind: 'ref', ref: '@e2' }); + assert.equal(result.node?.identifier, 'auth_email'); +}); From 330c4b2e16ece6371025993a07c05680007462a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:33:33 +0200 Subject: [PATCH 04/13] refactor(daemon): split fill and runtime-dispatch tests by what each asserts interaction-touch-fill.test.ts: (a) expired-frame rejection, pinned stale/current ref, and the find-internal fill without a stale-ref warning move to the daemon package (plus one new test that the resolved point, typing delay and recorded ref reach the bound fill). The off-screen refusal, invalid-bounds refusal and editable-node retention are (c): interactions-ref-frame-evidence.test.ts and interactions-target-promotion.test.ts. interaction-touch-runtime.test.ts: (a) the resolved-point tap/record test and the #1542 rescue hook wiring (hook present iff the owner advertises querySelector) move to the daemon package; the invalid-bounds, off-screen refusal and in-frame label recovery cases are (c) in interactions-ref-frame-evidence.test.ts (the missing-rect refusal is already ref-target-resolution.test.ts). Co-Authored-By: Claude Opus 5.5 --- .../internal/interaction-touch-fill.test.ts | 219 +++++++++ .../interaction-touch-runtime.test.ts | 129 +++++ .../interaction-touch-fill.test.ts | 393 --------------- .../interaction-touch-runtime.test.ts | 447 ------------------ .../interactions-ref-frame-evidence.test.ts | 194 ++++++++ 5 files changed, 542 insertions(+), 840 deletions(-) create mode 100644 packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts create mode 100644 packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.test.ts delete mode 100644 src/__tests__/daemon-composition/interaction/interaction-touch-fill.test.ts delete mode 100644 src/__tests__/daemon-composition/interaction/interaction-touch-runtime.test.ts create mode 100644 src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts new file mode 100644 index 0000000000..6b3565ce17 --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts @@ -0,0 +1,219 @@ +import { test, expect, beforeEach } from 'vitest'; +import { attachRefs } from '@agent-device/kernel/snapshot'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { activateCompleteRefFrame, expireRefFrame } from '../../../core/ref-frame.ts'; +import { STALE_SNAPSHOT_REFS_WARNING } from '../../session-snapshot.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { runInteraction } from '../../../__tests__/support/touch-interaction-runner.ts'; +import { + getRuntimeBindings, + mockFillPoint, + resetGetRuntimeFixture, +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { + contextFromFlags, + makeSession, + makeStaleRefSession, +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; + +// How fill is admitted and projected: its @ref admission and pins, and the bound fill the resolved +// point reaches. + +beforeEach(() => { + resetGetRuntimeFixture(); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); +}); + +test('fill @ref dispatches the resolved point with the typing delay and records the ref', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'fill-ref-record'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + identifier: 'auth_email', + rect: { x: 44, y: 120, width: 200, height: 32 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.publish(sessionName, session); + + const response = await runInteraction( + sessionStore, + sessionName, + 'fill', + ['@e1', 'hello@example.com'], + { delayMs: 25 }, + ); + + expect(response?.ok).toBe(true); + expect(mockFillPoint).toHaveBeenCalledWith( + expect.objectContaining({ + point: { x: 144, y: 136 }, + text: 'hello@example.com', + delayMs: 25, + }), + ); + const result = (sessionStore.get(sessionName)?.actions[0]?.result ?? {}) as Record< + string, + unknown + >; + expect(result.ref).toBe('e1'); +}); + +test('fill @ref rejects after a device action expired the frame', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'stale-ref-fill'; + const session = makeStaleRefSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 200, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + // ADR 0014: an unobserved device action expired the frame. + expireRefFrame(session); + sessionStore.publish(sessionName, session); + mockFillPoint.mockRejectedValue( + new Error('dispatch should not be called for an expired-frame ref'), + ); + + const response = await runInteraction(sessionStore, sessionName, 'fill', [ + '@e1', + 'hello@example.com', + ]); + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.code).toBe('COMMAND_FAILED'); + expect(response.error.message).toMatch(/expired ref frame/); + expect(response.error.details?.reason).toBe('ref_frame_expired'); + expect(response.error.details?.hint).toBe(STALE_SNAPSHOT_REFS_WARNING); + } + expect(mockFillPoint).not.toHaveBeenCalled(); +}); + +test('fill with a pinned stale ref rejects; pinned current is clean', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'pinned-fill'; + const session = makeStaleRefSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 200, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + session.snapshotGeneration = 3; + // Re-issue the frame over the overridden snapshot so its source tree matches. + activateCompleteRefFrame(session); + sessionStore.publish(sessionName, session); + + const stale = await runInteraction(sessionStore, sessionName, 'fill', ['@e1~s2', 'hello']); + expect(stale?.ok).toBe(false); + if (stale && !stale.ok) { + expect(stale.error.code).toBe('COMMAND_FAILED'); + expect(stale.error.details?.hint).toBe( + "Ref @e1 was minted from snapshot s2 but the session's ref frame is now s3 — re-run snapshot -i.", + ); + } + expect(mockFillPoint).not.toHaveBeenCalled(); + + const current = await runInteraction(sessionStore, sessionName, 'fill', ['@e1~s3', 'hello']); + expect(current?.ok).toBe(true); + if (current?.ok) { + expect(current.data?.warning).toBeUndefined(); + } +}); + +test("ADR 0014 blocker-2: a mutating find's internal fill from an expired frame carries no stale-ref warning", async () => { + // Removing the coarse marker (step 8) means an expired frame now derives read + // staleness. A mutating `find fill` re-resolves the locator itself and re-enters + // the fill leaf with a LOCATOR-minted ref (`internal.findResolvedTarget`) — the + // caller never consumed a `@ref`, so the public find response must not claim + // stale refs even though the frame is expired. + const sessionStore = makeSessionStore(); + const sessionName = 'find-internal-fill-expired'; + const session = makeStaleRefSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 200, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + // Re-issue the frame over the overridden snapshot, then expire it as if a prior + // device side effect changed the screen. + activateCompleteRefFrame(session); + expireRefFrame(session); + sessionStore.publish(sessionName, session); + mockFillPoint.mockResolvedValue({ filled: true }); + + // Contrast: a user-supplied `@ref` against the expired frame rejects before + // dispatch (it consumed a stale ref). + const userRef = await runInteraction(sessionStore, sessionName, 'fill', ['@e1', 'hello']); + expect(userRef?.ok).toBe(false); + if (userRef && !userRef.ok) { + expect(userRef.error.details?.reason).toBe('ref_frame_expired'); + } + + // The mutating find's internal dispatch (the exact request find.ts hands the + // leaf) bypasses admission AND attaches no stale-ref warning. + const findNodes = session.snapshot.nodes; + const findNode = findNodes[0]!; + const internal = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['@e1', 'hello'], + flags: {}, + internal: { + findResolvedTarget: { ref: '@e1', node: findNode, tree: { nodes: findNodes } }, + }, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + expect(internal?.ok).toBe(true); + if (internal?.ok) { + expect(internal.data?.warning).toBeUndefined(); + } +}); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.test.ts new file mode 100644 index 0000000000..47da4cab06 --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.test.ts @@ -0,0 +1,129 @@ +import { test, expect, beforeEach } from 'vitest'; +import { attachRefs } from '@agent-device/kernel/snapshot'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + resetTouchSurfaceFake, + touchSurfaceConfigs, +} from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { runInteraction } from '../../../__tests__/support/touch-interaction-runner.ts'; +import { + elementReadFixtureState, + mockQuerySelector, + mockTapPoint, + resetGetRuntimeFixture, +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { makeSession } from './__tests__/interaction-touch-fixtures.ts'; + +// What the shared runtime dispatch hands the command surface: the tapped point and the recorded +// `@ref` the caller wrote, and the off-screen rescue hook that rides the request-bound binding. + +beforeEach(() => { + resetGetRuntimeFixture(); + resetTouchSurfaceFake(); +}); + +test('press @ref taps the resolved point once and records the ref', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'press-ref'; + const session = makeSession(sessionName); + const nodes = attachRefs([ + { + index: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + identifier: 'auth_continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ]); + session.snapshot = { nodes, createdAt: Date.now(), backend: 'xctest' }; + sessionStore.publish(sessionName, session); + + const response = await runInteraction(sessionStore, sessionName, 'press', ['@e1']); + + expect(response?.ok).toBe(true); + expect(mockTapPoint).toHaveBeenCalledTimes(1); + expect(mockTapPoint.mock.calls[0]?.[0]?.point).toEqual({ x: 60, y: 40 }); + const stored = sessionStore.get(sessionName); + expect(stored?.actions[0]?.positionals).toEqual(['@e1']); + expect(stored?.actions[0]?.flags).toEqual({}); +}); + +// #1542 wiring, daemon route: the off-screen rescue's live read rides the SAME request-bound +// binding the touch command was admitted for. These two tests pin both declared sides of the +// CONDITIONAL `querySelector` fact: an owner that advertises it hands the command surface a +// rescue hook that answers with the live rect; an owner that does not hands it none, so the guard +// refuses on the bulk-tree verdict with ZERO queries. Deleting `conditional: ['querySelector']` +// from the captured touch uses, or making the rescue hook return undefined, breaks one of these +// two; the Apple binding's own spread is pinned in platform-apple's runtime.test.ts. + +const RESCUE_ROUTE_SESSION = 'rescue-route'; + +function offscreenRefSessionStore() { + const sessionStore = makeSessionStore(); + const session = makeSession(RESCUE_ROUTE_SESSION); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + depth: 0, + type: 'Window', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Far item', + identifier: 'far_item', + rect: { x: 20, y: 1200, width: 120, height: 44 }, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.publish(RESCUE_ROUTE_SESSION, session); + return sessionStore; +} + +const FAR_ITEM = { identifier: 'far_item', label: 'Far item' }; +const ROOT_VIEWPORT = { x: 0, y: 0, width: 390, height: 844 }; + +test('press @ref hands the command surface a rescue hook that reads the live rect when the owner advertises querySelector', async () => { + const sessionStore = offscreenRefSessionStore(); + elementReadFixtureState.querySelectorAvailable = true; + mockQuerySelector.mockResolvedValue({ + found: true, + node: { rect: { x: 20, y: 400, width: 120, height: 44 }, hittable: true }, + }); + + const response = await runInteraction(sessionStore, RESCUE_ROUTE_SESSION, 'press', ['@e2']); + + expect(response?.ok).toBe(true); + const confirm = touchSurfaceConfigs.at(-1)?.backend.confirmOffscreenTargetVisible; + expect(confirm).toBeDefined(); + // The rescue answers with the LIVE rect, not the stale bulk-tree row's off-screen geometry, and + // the live read carried the derived selector. + const rescued = await confirm!({}, FAR_ITEM, ROOT_VIEWPORT); + expect(rescued).toEqual({ x: 20, y: 400, width: 120, height: 44 }); + expect(mockQuerySelector).toHaveBeenCalledTimes(1); + expect(mockQuerySelector.mock.calls[0]?.[0].selector).toEqual({ + key: 'id', + value: 'far_item', + }); +}); + +test('press @ref hands the command surface no rescue hook when the owner does not advertise querySelector', async () => { + // The fixture default: the owner's facts declare the CONDITIONAL read unavailable, so the + // route holds no hook and the guard's refusal is built from the bulk tree alone. + const sessionStore = offscreenRefSessionStore(); + + const response = await runInteraction(sessionStore, RESCUE_ROUTE_SESSION, 'press', ['@e2']); + + expect(response?.ok).toBe(true); + expect(touchSurfaceConfigs.at(-1)?.backend.confirmOffscreenTargetVisible).toBeUndefined(); + expect(mockQuerySelector).not.toHaveBeenCalled(); +}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-fill.test.ts b/src/__tests__/daemon-composition/interaction/interaction-touch-fill.test.ts deleted file mode 100644 index 35087c1e0b..0000000000 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-fill.test.ts +++ /dev/null @@ -1,393 +0,0 @@ -import { test, expect, vi, beforeEach } from 'vitest'; -import { attachRefs } from '@agent-device/kernel/snapshot'; -import { withAppleRunnerProvider } from '@agent-device/platform-apple/runner'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { activateCompleteRefFrame, expireRefFrame } from '@agent-device/daemon/testing/ref-frame'; -import { STALE_SNAPSHOT_REFS_WARNING } from '@agent-device/daemon/testing/session-snapshot'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { - getRuntimeBindings, - mockFillPoint, - resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - makeSession, - makeStaleRefSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { runInteraction } from './interaction-touch-runner.ts'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -// How fill is admitted and projected: its @ref admission and pins, the editable -// node it keeps, and the guards that refuse before typing. - -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -async function withRunner(operation: () => Promise): Promise { - return await withAppleRunnerProvider(mockRunAppleRunnerCommand, { deviceId: 'sim-1' }, operation); -} - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - -beforeEach(() => { - resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); -}); - -test('fill @ref fails fast when the target is off-screen', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'fill-offscreen-ref'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - depth: 0, - type: 'Window', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - rect: { x: 20, y: 1180, width: 180, height: 44 }, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - const response = await withRunner(() => - handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'fill', - positionals: ['@e2', 'hello@example.com'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }), - ); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(false); - expect(mockFillPoint).not.toHaveBeenCalled(); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/off-screen/i); - // #1366: direction-named, selector-first recovery hint (see press test above). - expect(response.error.hint).toMatch(/scroll down/i); - expect(response.error.hint).toMatch(/selector/i); - expect(response.error.details?.reason).toBe('offscreen_ref'); - expect(response.error.details?.scrollDirection).toBe('down'); - } -}); - -test('fill @ref fails closed when stored ref bounds are invalid (ADR 0014)', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.device = { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel 8 Pro', - kind: 'emulator', - booted: true, - }; - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'android.widget.EditText', - label: 'Email', - rect: { x: 20, y: 40, width: Number.NaN, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'android', - }; - sessionStore.publish(sessionName, session); - - mockFillPoint.mockRejectedValue( - new Error('dispatch must not run: no positional recapture on unusable frame evidence'), - ); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'fill', - positionals: ['@e1', 'hello@example.com'], - flags: { delayMs: 25 }, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - // ADR 0014: the authorized frame's @e1 has an unusable rect, so the fill FAILS - // rather than recapturing a fresh tree and filling the same index. - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/Ref @e1 has no usable bounds/); - expect(response.error.details).toMatchObject({ reason: 'target_bounds_invalid', ref: 'e1' }); - } - expect(mockFillPoint).not.toHaveBeenCalled(); -}); - -test('fill @ref rejects after a device action expired the frame', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'stale-ref-fill'; - const session = makeStaleRefSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - rect: { x: 10, y: 20, width: 200, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - // ADR 0014: an unobserved device action expired the frame. - expireRefFrame(session); - sessionStore.publish(sessionName, session); - mockFillPoint.mockRejectedValue( - new Error('dispatch should not be called for an expired-frame ref'), - ); - - const response = await runInteraction(sessionStore, sessionName, 'fill', [ - '@e1', - 'hello@example.com', - ]); - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/expired ref frame/); - expect(response.error.details?.reason).toBe('ref_frame_expired'); - expect(response.error.details?.hint).toBe(STALE_SNAPSHOT_REFS_WARNING); - } - expect(mockFillPoint).not.toHaveBeenCalled(); -}); - -test('fill with a pinned stale ref rejects; pinned current is clean', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'pinned-fill'; - const session = makeStaleRefSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - rect: { x: 10, y: 20, width: 200, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - session.snapshotGeneration = 3; - // Re-issue the frame over the overridden snapshot so its source tree matches. - activateCompleteRefFrame(session); - sessionStore.publish(sessionName, session); - - const stale = await runInteraction(sessionStore, sessionName, 'fill', ['@e1~s2', 'hello']); - expect(stale?.ok).toBe(false); - if (stale && !stale.ok) { - expect(stale.error.code).toBe('COMMAND_FAILED'); - expect(stale.error.details?.hint).toBe( - "Ref @e1 was minted from snapshot s2 but the session's ref frame is now s3 — re-run snapshot -i.", - ); - } - expect(mockFillPoint).not.toHaveBeenCalled(); - - const current = await runInteraction(sessionStore, sessionName, 'fill', ['@e1~s3', 'hello']); - expect(current?.ok).toBe(true); - if (current?.ok) { - expect(current.data?.warning).toBeUndefined(); - } -}); - -test("ADR 0014 blocker-2: a mutating find's internal fill from an expired frame carries no stale-ref warning", async () => { - // Removing the coarse marker (step 8) means an expired frame now derives read - // staleness. A mutating `find fill` re-resolves the locator itself and re-enters - // the fill leaf with a LOCATOR-minted ref (`internal.findResolvedTarget`) — the - // caller never consumed a `@ref`, so the public find response must not claim - // stale refs even though the frame is expired. - const sessionStore = makeSessionStore(); - const sessionName = 'find-internal-fill-expired'; - const session = makeStaleRefSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - rect: { x: 10, y: 20, width: 200, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - // Re-issue the frame over the overridden snapshot, then expire it as if a prior - // device side effect changed the screen. - activateCompleteRefFrame(session); - expireRefFrame(session); - sessionStore.publish(sessionName, session); - mockFillPoint.mockResolvedValue({ filled: true }); - - // Contrast: a user-supplied `@ref` against the expired frame rejects before - // dispatch (it consumed a stale ref). - const userRef = await runInteraction(sessionStore, sessionName, 'fill', ['@e1', 'hello']); - expect(userRef?.ok).toBe(false); - if (userRef && !userRef.ok) { - expect(userRef.error.details?.reason).toBe('ref_frame_expired'); - } - - // The mutating find's internal dispatch (the exact request find.ts hands the - // leaf) bypasses admission AND attaches no stale-ref warning. - const findNodes = session.snapshot.nodes; - const findNode = findNodes[0]!; - const internal = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'fill', - positionals: ['@e1', 'hello'], - flags: {}, - internal: { - findResolvedTarget: { ref: '@e1', node: findNode, tree: { nodes: findNodes } }, - }, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - expect(internal?.ok).toBe(true); - if (internal?.ok) { - expect(internal.data?.warning).toBeUndefined(); - } -}); - -test('fill @ref keeps the original editable node when its parent is the hittable ancestor', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeCell', - label: 'Email row', - rect: { x: 20, y: 100, width: 320, height: 72 }, - enabled: true, - hittable: true, - }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - identifier: 'auth_email', - rect: { x: 44, y: 120, width: 200, height: 32 }, - enabled: true, - hittable: false, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - mockFillPoint.mockResolvedValue({ filled: true }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'fill', - positionals: ['@e2', 'hello@example.com'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - expect(mockFillPoint).toHaveBeenCalledWith( - expect.objectContaining({ - point: { x: 144, y: 136 }, - text: 'hello@example.com', - }), - ); - - const stored = sessionStore.get(sessionName); - const result = (stored?.actions[0]?.result ?? {}) as Record; - expect(result.ref).toBe('e2'); -}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-runtime.test.ts b/src/__tests__/daemon-composition/interaction/interaction-touch-runtime.test.ts deleted file mode 100644 index 839ada4962..0000000000 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-runtime.test.ts +++ /dev/null @@ -1,447 +0,0 @@ -import { test, expect, vi, beforeEach } from 'vitest'; -import { attachRefs } from '@agent-device/kernel/snapshot'; -import { withAppleRunnerProvider } from '@agent-device/platform-apple/runner'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { - elementReadFixtureState, - getRuntimeBindings, - mockQuerySelector, - mockTapPoint, - resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - makeSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -// What the shared runtime dispatch does with the resolved target: refuse -// unusable frame evidence rather than recapture positionally (ADR 0014), refuse -// off-screen targets, and record the `@ref` the caller wrote. - -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -async function withRunner(operation: () => Promise): Promise { - return await withAppleRunnerProvider(mockRunAppleRunnerCommand, { deviceId: 'sim-1' }, operation); -} - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - -beforeEach(() => { - resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); -}); - -test('press @ref taps the resolved point once and records the ref', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'press-ref'; - const session = makeSession(sessionName); - const nodes = attachRefs([ - { - index: 0, - type: 'XCUIElementTypeButton', - label: 'Continue', - identifier: 'auth_continue', - rect: { x: 10, y: 20, width: 100, height: 40 }, - enabled: true, - hittable: true, - }, - ]); - session.snapshot = { nodes, createdAt: Date.now(), backend: 'xctest' }; - sessionStore.publish(sessionName, session); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e1'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(mockTapPoint).toHaveBeenCalledTimes(1); - expect(mockTapPoint.mock.calls[0]?.[0]?.point).toEqual({ x: 60, y: 40 }); - const stored = sessionStore.get(sessionName); - expect(stored?.actions[0]?.positionals).toEqual(['@e1']); - expect(stored?.actions[0]?.flags).toEqual({}); -}); - -test('press @ref fails closed when the authorized ref has no usable bounds (ADR 0014)', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'stale-ref-refresh'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeButton', - label: 'Continue', - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - mockTapPoint.mockRejectedValue( - new Error('dispatch must not run: no positional recapture on missing frame evidence'), - ); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e1'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - // ADR 0014: the authorized frame's @e1 has no usable rect, so the ref FAILS - // rather than recapturing and accepting the same index from a newer tree by - // positional coincidence. No fresh capture, no dispatch. - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/Ref @e1 has no usable bounds/); - expect(response.error.details).toMatchObject({ reason: 'target_bounds_invalid', ref: 'e1' }); - } - expect(mockTapPoint).not.toHaveBeenCalled(); -}); - -test('press @ref fails closed when stored ref bounds are invalid (ADR 0014)', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.device = { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel 8 Pro', - kind: 'emulator', - booted: true, - }; - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'android.widget.TextView', - label: 'My App', - rect: { x: 20, y: 40, width: Number.NaN, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'android', - }; - sessionStore.publish(sessionName, session); - - mockTapPoint.mockRejectedValue( - new Error('dispatch must not run: no positional recapture on unusable frame evidence'), - ); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e1'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - // ADR 0014: the authorized frame's @e1 has an unusable rect (NaN), so it FAILS - // rather than recapturing and accepting the same index from a newer tree. - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/Ref @e1 has no usable bounds/); - expect(response.error.details).toMatchObject({ reason: 'target_bounds_invalid', ref: 'e1' }); - } - expect(mockTapPoint).not.toHaveBeenCalled(); -}); - -// #1542 wiring, daemon route: the off-screen rescue's live read rides the SAME request-bound -// binding the touch command was admitted for. These two tests pin both declared sides of the -// CONDITIONAL `querySelector` fact: an owner that advertises it answers the guard with a live -// rect the tap lands on; an owner that does not refuses on the bulk-tree verdict with ZERO -// queries. Deleting `conditional: ['querySelector']` from the captured touch uses, or making the -// rescue hook return undefined, breaks one of these two; the Apple binding's own spread is pinned -// in platform-apple's runtime.test.ts. - -const RESCUE_ROUTE_SESSION = 'rescue-route'; - -function offscreenRefSessionStore() { - const sessionStore = makeSessionStore(); - const session = makeSession(RESCUE_ROUTE_SESSION); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - depth: 0, - type: 'Window', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeButton', - label: 'Far item', - identifier: 'far_item', - rect: { x: 20, y: 1200, width: 120, height: 44 }, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(RESCUE_ROUTE_SESSION, session); - return sessionStore; -} - -async function pressOffscreenRef(sessionStore: ReturnType) { - return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: RESCUE_ROUTE_SESSION, - command: 'press', - positionals: ['@e2'], - flags: {}, - }, - sessionName: RESCUE_ROUTE_SESSION, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); -} - -test('press @ref an off-screen node is rescued to the live rect when the owner advertises querySelector', async () => { - const sessionStore = offscreenRefSessionStore(); - elementReadFixtureState.querySelectorAvailable = true; - mockQuerySelector.mockResolvedValue({ - found: true, - node: { rect: { x: 20, y: 400, width: 120, height: 44 }, hittable: true }, - }); - - const response = await pressOffscreenRef(sessionStore); - - // The rescue wins: the press lands at the CENTER OF THE LIVE RECT, not the stale - // bulk-tree row's off-screen geometry, and the live read carried the derived selector. - expect(response?.ok).toBe(true); - expect(mockQuerySelector).toHaveBeenCalledTimes(1); - expect(mockQuerySelector.mock.calls[0]?.[0].selector).toEqual({ - key: 'id', - value: 'far_item', - }); - expect(mockTapPoint).toHaveBeenCalledTimes(1); - expect(mockTapPoint.mock.calls[0]?.[0]?.point).toEqual({ x: 80, y: 422 }); -}); - -test('press @ref an off-screen node refuses with zero queries when the owner does not advertise querySelector', async () => { - // The fixture default: the owner's facts declare the CONDITIONAL read unavailable, so the - // route holds no hook and the guard's refusal is built from the bulk tree alone. - const sessionStore = offscreenRefSessionStore(); - - const response = await pressOffscreenRef(sessionStore); - - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.details?.reason).toBe('offscreen_ref'); - } - expect(mockQuerySelector).not.toHaveBeenCalled(); - expect(mockTapPoint).not.toHaveBeenCalled(); -}); - -test('press @ref fails fast when the target is off-screen', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'press-offscreen-ref'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - depth: 0, - type: 'Window', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeButton', - label: 'Far item', - rect: { x: 20, y: 1200, width: 120, height: 44 }, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - const response = await withRunner(() => - handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e2'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }), - ); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(false); - expect(mockTapPoint).not.toHaveBeenCalled(); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/off-screen/i); - // #1366: the hint names the concrete scroll direction and steers to a selector-based retry - // (a @ref would be rejected as expired after the scroll). A ref refusal has no selector to - // check between passes, so this hint names the move and leaves the stop condition to the - // selector retry it prescribes. - expect(response.error.hint).toMatch(/scroll down/i); - expect(response.error.hint).toMatch(/selector/i); - expect(response.error.details?.reason).toBe('offscreen_ref'); - expect(response.error.details?.scrollDirection).toBe('down'); - } -}); - -test('press @ref with a trailing label recovers within the authorized frame (no positional recapture)', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - session.device = { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel 8 Pro', - kind: 'emulator', - booted: true, - }; - // @e1's rect is unusable, but the SAME frame tree carries another node with - // the trailing label at usable bounds. ADR 0014 label recovery resolves within - // the authorized frame — never by recapturing a fresh tree. - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'android.widget.TextView', - label: 'Different', - rect: { x: 20, y: 40, width: Number.NaN, height: 40 }, - enabled: true, - hittable: true, - }, - { - index: 1, - type: 'android.widget.TextView', - label: 'My App', - rect: { x: 100, y: 200, width: 80, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'android', - }; - sessionStore.publish(sessionName, session); - - mockCaptureSnapshotForSession.mockRejectedValue( - new Error('no positional recapture: recovery stays in-frame'), - ); - mockTapPoint.mockResolvedValue({ pressed: true }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['@e1', 'My App'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - expect(mockTapPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 140, y: 220 } })); - if (response?.ok) { - expect(response.data?.x).toBe(140); - expect(response.data?.y).toBe(220); - expect(response.data?.resolution).toEqual({ - source: 'ref', - phase: 'pre-action', - kind: 'label-fallback', - }); - } -}); diff --git a/src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts b/src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts new file mode 100644 index 0000000000..cc0025e6b5 --- /dev/null +++ b/src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts @@ -0,0 +1,194 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Point } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { ref } from './selector-read-utils.ts'; +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; + +// What a `@ref` interaction does with the frame evidence it is handed (ADR 0014): refuse unusable +// bounds rather than recapture positionally, refuse off-screen targets with a direction, and +// recover a label only inside the authorized frame. + +function androidTextViewSnapshot(type: string, label: string) { + return makeSnapshotState( + [ + { + index: 0, + type, + label, + rect: { x: 20, y: 40, width: Number.NaN, height: 40 }, + enabled: true, + hittable: true, + }, + ], + { backend: 'android' }, + ); +} + +function offscreenSnapshot(type: string, label: string, y: number) { + return makeSnapshotState([ + { index: 0, depth: 0, type: 'Window', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type, + label, + rect: { x: 20, y, width: 180, height: 44 }, + hittable: true, + }, + ]); +} + +function assertBoundsInvalid(error: unknown): boolean { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /Ref @e1 has no usable bounds/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.ref, 'e1'); + return true; +} + +function assertOffscreenRefusal(error: unknown): boolean { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /off-screen/i); + // #1366: the hint names the concrete scroll direction and steers to a selector-based retry (a + // @ref would be rejected as expired after the scroll). + const details = (error as { details?: Record }).details; + const hint = String(details?.hint ?? ''); + assert.match(hint, /scroll down/i); + assert.match(hint, /selector/i); + assert.equal(details?.reason, 'offscreen_ref'); + assert.equal(details?.scrollDirection, 'down'); + return true; +} + +test('runtime press @ref fails closed when stored ref bounds are invalid (ADR 0014)', async () => { + const taps: Point[] = []; + const device = createInteractionDevice( + androidTextViewSnapshot('android.widget.TextView', 'My App'), + { + platform: 'android', + tap: async (_context, point) => { + taps.push(point); + }, + }, + ); + + await assert.rejects( + () => device.interactions.press(ref('@e1'), { session: 'default' }), + assertBoundsInvalid, + ); + assert.deepEqual(taps, []); +}); + +test('runtime fill @ref fails closed when stored ref bounds are invalid (ADR 0014)', async () => { + const fills: Point[] = []; + const device = createInteractionDevice( + androidTextViewSnapshot('android.widget.EditText', 'Email'), + { + platform: 'android', + fill: async (_context, point) => { + fills.push(point); + }, + }, + ); + + await assert.rejects( + () => + device.interactions.fill(ref('@e1'), 'hello@example.com', { + session: 'default', + delayMs: 25, + }), + assertBoundsInvalid, + ); + assert.deepEqual(fills, []); +}); + +test('runtime press @ref fails fast when the target is off-screen', async () => { + const taps: Point[] = []; + const device = createInteractionDevice( + offscreenSnapshot('XCUIElementTypeButton', 'Far item', 1200), + { + tap: async (_context, point) => { + taps.push(point); + }, + }, + ); + + await assert.rejects( + () => device.interactions.press(ref('@e2'), { session: 'default' }), + assertOffscreenRefusal, + ); + assert.deepEqual(taps, []); +}); + +test('runtime fill @ref fails fast when the target is off-screen', async () => { + const fills: Point[] = []; + const device = createInteractionDevice( + offscreenSnapshot('XCUIElementTypeTextField', 'Email', 1180), + { + fill: async (_context, point) => { + fills.push(point); + }, + }, + ); + + await assert.rejects( + () => device.interactions.fill(ref('@e2'), 'hello@example.com', { session: 'default' }), + assertOffscreenRefusal, + ); + assert.deepEqual(fills, []); +}); + +test('runtime press @ref with a trailing label recovers within the authorized frame (no positional recapture)', async () => { + // @e1's rect is unusable, but the SAME frame tree carries another node with the trailing label at + // usable bounds. ADR 0014 label recovery resolves within the authorized frame, never by + // recapturing a fresh tree. + const snapshot = makeSnapshotState( + [ + { + index: 0, + type: 'android.widget.TextView', + label: 'Different', + rect: { x: 20, y: 40, width: Number.NaN, height: 40 }, + enabled: true, + hittable: true, + }, + { + index: 1, + type: 'android.widget.TextView', + label: 'My App', + rect: { x: 100, y: 200, width: 80, height: 40 }, + enabled: true, + hittable: true, + }, + ], + { backend: 'android' }, + ); + const taps: Point[] = []; + const device = createInteractionDevice(snapshot, { + platform: 'android', + captureSnapshot: async () => { + throw new Error('no positional recapture: recovery stays in-frame'); + }, + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.press(ref('@e1', { fallbackLabel: 'My App' }), { + session: 'default', + }); + + assert.deepEqual(taps, [{ x: 140, y: 220 }]); + assert.equal(result.kind, 'ref'); + assert.deepEqual(result.point, { x: 140, y: 220 }); + assert.deepEqual(result.resolution, { + source: 'ref', + phase: 'pre-action', + kind: 'label-fallback', + }); +}); From 0aaa48c9eda102c960b791629e38d00d60813b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:33:39 +0200 Subject: [PATCH 05/13] refactor(daemon): place touch response, payload and prepare tests beside their modules interaction-touch-response.test.ts (6), interaction-touch-payload.test.ts (7) and interaction-touch-prepare.test.ts (5) are (a) daemon logic: evidence rides the response allowlist only when --verify ran, recording/visualization payloads and reference frames, and the settle observation capture the request admitted. They move to execution/interaction/internal/ on the touch surface fake, the interactor-resolution fixture and daemonPlatformServicesFixture. The --verify evidence computation itself stays covered by the real-runtime tests in src/commands/interaction/runtime/interactions.test.ts. Co-Authored-By: Claude Opus 5.5 --- .../interaction-touch-payload.test.ts | 57 +++++------- .../interaction-touch-prepare.test.ts | 38 ++++---- ...nteraction-touch-response-evidence.test.ts | 86 ++++++------------- 3 files changed, 65 insertions(+), 116 deletions(-) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-touch-payload.test.ts (86%) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-touch-prepare.test.ts (88%) rename src/__tests__/daemon-composition/interaction/interaction-touch-response.test.ts => packages/daemon/src/execution/interaction/internal/interaction-touch-response-evidence.test.ts (76%) diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-payload.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-payload.test.ts similarity index 86% rename from src/__tests__/daemon-composition/interaction/interaction-touch-payload.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-payload.test.ts index ffbd8b9a8f..00cf236c95 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-payload.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-payload.test.ts @@ -1,28 +1,28 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, installTestScreenRecording, makeAndroidSession, makeSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // What a built touch payload carries: the recorded action entry and touch // visualization, the coordinates and reference frame each platform resolves, // and the native timing the recorded result preserves. -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { const actual = await importOriginal(); return { @@ -33,26 +33,14 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - import { getAndroidAppState, getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); beforeEach(() => { resetGetRuntimeFixture(); @@ -62,9 +50,8 @@ beforeEach(() => { mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); mockGetAndroidScreenSize.mockReset(); mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); // fallow-ignore-next-line complexity @@ -98,7 +85,7 @@ test('press coordinates appends touch-visualization events while recording', asy }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -146,7 +133,7 @@ test('press coordinates on iOS recording captures a full snapshot for the touch mockTapPoint.mockResolvedValue({ x: 220, y: 600 }); // Regression: a filtered snapshot has no Application/Window node, so viewport inference would // return a leaf-element bounding box and the recording overlay would misplace tap markers. - mockCaptureSnapshotForSession.mockResolvedValueOnce({ + mockInteractorSnapshot.mockResolvedValueOnce({ nodes: attachRefs([ { index: 0, @@ -165,7 +152,7 @@ test('press coordinates on iOS recording captures a full snapshot for the touch }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -180,8 +167,8 @@ test('press coordinates on iOS recording captures a full snapshot for the touch }); expect(response?.ok).toBe(true); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); - expect(mockCaptureSnapshotForSession.mock.calls[0]?.[0].options).toMatchObject({ + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); + expect(mockInteractorSnapshot.mock.calls[0]?.[0].options).toMatchObject({ interactiveOnly: true, }); const event = sessionStore.get(sessionName)?.screenRecording?.handle.inspect().gestureEvents[0]; @@ -206,7 +193,7 @@ test('press coordinates on Android recording uses physical screen size when no s mockTapPoint.mockResolvedValue({ x: 300, y: 2300 }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -243,7 +230,7 @@ test('press coordinates on Android recording caches physical screen size across mockTapPoint.mockResolvedValue({ x: 300, y: 2300 }); await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -260,7 +247,7 @@ test('press coordinates on Android recording caches physical screen size across mockTapPoint.mockResolvedValue({ x: 320, y: 2200 }); await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -292,7 +279,7 @@ test('press coordinates without recording skips Android screen-size lookup', asy mockTapPoint.mockResolvedValue({ x: 300, y: 2300 }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -327,7 +314,7 @@ test('press coordinates during recording still dispatches when Android screen-si mockGetAndroidScreenSize.mockRejectedValue(new Error('adb unavailable')); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -392,7 +379,7 @@ test('press @ref preserves native timing in recorded result and touch visualizat }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-prepare.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-prepare.test.ts similarity index 88% rename from src/__tests__/daemon-composition/interaction/interaction-touch-prepare.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-prepare.test.ts index 7c0fb0f86a..531de7cb23 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-prepare.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-prepare.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, expect, test, vi } from 'vitest'; +import { beforeEach, expect, test } from 'vitest'; import type { CommandFlags } from '@agent-device/contracts/command'; import { type DeviceBinding, @@ -16,16 +16,20 @@ import { import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { createUnavailableRuntimeFactsForTest } from '@agent-device/test-kit/runtime-operation-facts'; -import { makeIosAppSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; +import { makeIosAppSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; import type { BindDeviceRuntime, InspectDeviceRuntimeFacts, -} from '@agent-device/daemon/testing/request-runtime-binding'; -import { composedPlatformServicesFixture } from '../platform-services-fixture.ts'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import type { SessionStore } from '@agent-device/daemon/testing/session-store'; +} from '../../../core/request-runtime-binding.ts'; +import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import type { SessionStore } from '../../../core/session-store.ts'; +import { handleInteractionCommands } from '../index.ts'; // #3510: the capture an interaction's post-action observation polls. A coordinate-targeted press or // fill acts through a use that declares no capture, so before this its settle loop read the screen @@ -55,11 +59,6 @@ const BRIDGE_PRODUCED = Object.freeze({ producer: 'simulator-ax-bridge' as const, }); -const mockInteractorCapture = vi.hoisted(() => vi.fn()); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: mockInteractorCapture, -})); - const contextFromFlags = (_flags: CommandFlags | undefined) => ({}); /** An owner that captures, taps points, fills points, and does nothing else. */ @@ -141,12 +140,12 @@ async function runCommand( inspectFacts: InspectDeviceRuntimeFacts = bindings.inspectFacts, ) { return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, command, positionals, flags: { ...flags } }, sessionName, sessionStore, contextFromFlags, - platformServices: composedPlatformServicesFixture(), + platformServices: daemonPlatformServicesFixture(), inspectFacts, bindDevice: bindings.bindDevice, }); @@ -157,7 +156,8 @@ function settled(response: unknown) { } beforeEach(() => { - mockInteractorCapture.mockReset(); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); // The observation's bind is a second, capture-only use beside the action's exact one — taken once @@ -190,7 +190,7 @@ test.each([ // issued resolve against the tree the caller's next `snapshot` reads. expect(session.snapshot?.producer).toBe('simulator-ax-bridge'); expect(bindings.captureCount()).toBeGreaterThan(1); - expect(mockInteractorCapture).not.toHaveBeenCalled(); + expect(mockInteractorSnapshot).not.toHaveBeenCalled(); }, ); @@ -214,7 +214,7 @@ test('an observation capture the owner cannot admit still lets the coordinate pr const sessionName = 'point-press-probe-failure'; publishSession(sessionStore, sessionName); const bindings = settleFixture(); - mockInteractorCapture.mockResolvedValue(BRIDGE_PRODUCED); + mockInteractorSnapshot.mockResolvedValue(BRIDGE_PRODUCED); let inspections = 0; const response = await runCommand( @@ -235,7 +235,7 @@ test('an observation capture the owner cannot admit still lets the coordinate pr expect(response?.ok).toBe(true); expect(settled(response)).toMatchObject({ settled: true }); - expect(mockInteractorCapture).toHaveBeenCalled(); + expect(mockInteractorSnapshot).toHaveBeenCalled(); }); test('a tree-resolved press reuses the capture its action binding already carries', async () => { diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-response.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-response-evidence.test.ts similarity index 76% rename from src/__tests__/daemon-composition/interaction/interaction-touch-response.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-response-evidence.test.ts index 55b01f4330..a19eecc5f0 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-response.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-response-evidence.test.ts @@ -1,70 +1,32 @@ -import { test, expect, vi, beforeEach } from 'vitest'; +import { test, expect, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockFillPoint, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, installTestScreenRecording, makeSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // The identity extras the one response site composes: --verify evidence rides // the interactionResultExtra allowlist on every branch, and no branch invents // an evidence field without it. -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - beforeEach(() => { resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test('press @ref --verify surfaces evidence through the interactionResultExtra allowlist', async () => { @@ -89,7 +51,7 @@ test('press @ref --verify surfaces evidence through the interactionResultExtra a // Post-action capture reports an extra node, so changedFromBefore should // read true against the pre-action (stored) snapshot's single node. - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: [ { index: 0, @@ -113,7 +75,7 @@ test('press @ref --verify surfaces evidence through the interactionResultExtra a }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -147,7 +109,7 @@ test('press @ref --verify surfaces evidence through the interactionResultExtra a // without a fresh pre-action capture (zero extra cost, per #1047's design) — // only the post-action verify capture issues a 'snapshot' dispatch, after 'press'. expect(mockTapPoint).toHaveBeenCalledOnce(); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledOnce(); + expect(mockInteractorSnapshot).toHaveBeenCalledOnce(); }); test('press @ref without --verify never includes an evidence field', async () => { @@ -172,7 +134,7 @@ test('press @ref without --verify never includes an evidence field', async () => mockTapPoint.mockResolvedValue({ pressed: true }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -192,7 +154,7 @@ test('press @ref without --verify never includes an evidence field', async () => } // No verify flag means no post-action snapshot capture at all. expect(mockTapPoint).toHaveBeenCalledOnce(); - expect(mockCaptureSnapshotForSession).not.toHaveBeenCalled(); + expect(mockInteractorSnapshot).not.toHaveBeenCalled(); }); test('fill selector --verify surfaces evidence through the interactionResultExtra allowlist', async () => { @@ -215,7 +177,7 @@ test('fill selector --verify surfaces evidence through the interactionResultExtr }; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: [ { index: 0, @@ -231,7 +193,7 @@ test('fill selector --verify surfaces evidence through the interactionResultExtr }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -277,7 +239,7 @@ test('fill @ref --verify surfaces evidence in the ref response branch', async () }; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: [ { index: 0, @@ -301,7 +263,7 @@ test('fill @ref --verify surfaces evidence in the ref response branch', async () }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -348,7 +310,7 @@ test('fill @ref without --verify never includes an evidence field', async () => sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -367,7 +329,7 @@ test('fill @ref without --verify never includes an evidence field', async () => expect(response.data?.evidence).toBeUndefined(); } // No verify flag means no post-action snapshot capture at all. - expect(mockCaptureSnapshotForSession).not.toHaveBeenCalled(); + expect(mockInteractorSnapshot).not.toHaveBeenCalled(); }); test('fill @ref preserves fallback coordinates for recording when platform result is sparse', async () => { @@ -399,7 +361,7 @@ test('fill @ref preserves fallback coordinates for recording when platform resul mockFillPoint.mockResolvedValue({ filled: true }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, From e1b47b7263d0984f2637ac57dfd4f1d72f25fbae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:33:46 +0200 Subject: [PATCH 06/13] refactor(daemon): place the Android freshness and readiness tests beside their modules interaction-touch-android-freshness.test.ts (4) and interaction-touch-android-readiness.test.ts (7) are (a) daemon logic: the freshness refresh may not retarget an admitted ref, the escape and permission-prompt outcomes, and recovery aborting an admitted ref. They move to execution/interaction/internal/. The readiness suite now mocks the exact android-system-dialog module the handler imports. interaction-touch-android-readiness-spawns.test.ts is (a): the two-checks-per-tap dumpsys budget is pinned through the real Android observation adapter over a counting runAdb host port instead of the root adb provider and host binding. Co-Authored-By: Claude Opus 5.5 --- ...nteraction-touch-android-freshness.test.ts | 60 +++++++------------ ...ion-touch-android-readiness-spawns.test.ts | 47 +++++++++++++++ ...nteraction-touch-android-readiness.test.ts | 55 ++++++----------- ...ion-touch-android-readiness-spawns.test.ts | 43 ------------- 4 files changed, 88 insertions(+), 117 deletions(-) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-touch-android-freshness.test.ts (80%) create mode 100644 packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-touch-android-readiness.test.ts (82%) delete mode 100644 src/__tests__/daemon-composition/interaction-touch-android-readiness-spawns.test.ts diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-android-freshness.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-freshness.test.ts similarity index 80% rename from src/__tests__/daemon-composition/interaction/interaction-touch-android-freshness.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-android-freshness.test.ts index c9329f67b1..af6788fa6c 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-android-freshness.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-freshness.test.ts @@ -1,26 +1,24 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { activateCompleteRefFrame } from '../../../core/ref-frame.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - makeAndroidSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { contextFromFlags, makeAndroidSession } from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // The Android ref-refresh capture a @ref mutation takes before dispatch: when // it runs, what it may not retarget, and the comparison-safe baseline it hands // finalization when the refresh fails. -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { const actual = await importOriginal(); return { @@ -31,27 +29,14 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - import { getAndroidAppState, getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -import { activateCompleteRefFrame } from '@agent-device/daemon/testing/ref-frame'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); beforeEach(() => { resetGetRuntimeFixture(); @@ -61,9 +46,8 @@ beforeEach(() => { mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); mockGetAndroidScreenSize.mockReset(); mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test('press @ref refreshes Android snapshot when freshness tracking is active', async () => { @@ -93,7 +77,7 @@ test('press @ref refreshes Android snapshot when freshness tracking is active', }; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: [ { index: 0, @@ -109,7 +93,7 @@ test('press @ref refreshes Android snapshot when freshness tracking is active', }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -124,8 +108,8 @@ test('press @ref refreshes Android snapshot when freshness tracking is active', }); expect(response?.ok).toBe(true); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); - expect(mockCaptureSnapshotForSession.mock.calls[0]?.[0].options).toMatchObject({ + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); + expect(mockInteractorSnapshot.mock.calls[0]?.[0].options).toMatchObject({ interactiveOnly: true, }); expect(mockTapPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 140, y: 220 } })); @@ -168,7 +152,7 @@ test('ADR 0014: Android freshness cannot retarget an admitted ref by positional // The freshness refresh returns a DIFFERENT element at @e1's index — after // navigation the button at that position is now "Cancel", not "Continue". - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: [ { index: 0, @@ -184,7 +168,7 @@ test('ADR 0014: Android freshness cannot retarget an admitted ref by positional }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -232,10 +216,10 @@ test('press @ref falls back to cached Android ref when freshness refresh fails', }; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockRejectedValueOnce(new Error('uiautomator timeout')); + mockInteractorSnapshot.mockRejectedValueOnce(new Error('uiautomator timeout')); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -250,7 +234,7 @@ test('press @ref falls back to cached Android ref when freshness refresh fails', }); expect(response?.ok).toBe(true); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); expect(mockTapPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 60, y: 40 } })); expect(sessionStore.get(sessionName)?.androidSnapshotFreshness).toMatchObject({ action: 'press', @@ -303,7 +287,7 @@ test('coordinate press preserves Android route freshness from last comparable sn sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts new file mode 100644 index 0000000000..acf7a0c82f --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts @@ -0,0 +1,47 @@ +import { expect, test, vi } from 'vitest'; +import type { AndroidObservationHost } from '@agent-device/contracts/android-observation'; +import { createAndroidObservationAdapter } from '@agent-device/platform-android'; +import { makeAndroidSession } from '../../../__tests__/support/session-factories.ts'; +import { expireRefFrame } from '../../../core/ref-frame.ts'; +import { runWithAndroidDialogReadinessCheck } from './interaction-touch-android-readiness.ts'; + +// What blocking-dialog readiness costs a run of taps, end to end. Every one of these `dumpsys` +// spawns is a device round trip on the critical path of an interaction, so the budget is pinned +// here rather than left to whatever the composition of the readiness seams happens to produce. + +const NORMAL_FOCUS_DUMP = + ' mCurrentFocus=Window{a1b2c3 u0 com.agentdevice.tester/com.agentdevice.tester.MainActivity}'; + +test('a run of taps on a clear screen costs one window dump per tap', async () => { + const session = makeAndroidSession('readiness-spawn-budget'); + const runAdb = vi.fn(async (_device, args) => ({ + exitCode: 0, + stdout: args.includes('dumpsys') ? NORMAL_FOCUS_DUMP : '', + stderr: '', + })); + const observation = createAndroidObservationAdapter({ + runAdb, + readSnapshotNodes: async () => [], + openApp: async () => undefined, + }); + + for (let tap = 0; tap < 3; tap += 1) { + const outcome = await runWithAndroidDialogReadinessCheck( + session, + 'press', + { refContext: undefined }, + async () => { + // The dispatch seam every interaction crosses (ADR 0014). + expireRefFrame(session); + return { pressed: true }; + }, + observation, + ); + expect(outcome.aborted).toBe(false); + } + + // Two checks per tap, one dump each: the focused-window line answers the question, so the + // second `dumpsys` variant never runs on a clear screen. Neither check may be skipped — a dialog + // surfaces with no adb traffic at all, so the previous check is evidence about nothing. + expect(runAdb.mock.calls.filter(([, args]) => args.includes('dumpsys'))).toHaveLength(6); +}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-android-readiness.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness.test.ts similarity index 82% rename from src/__tests__/daemon-composition/interaction/interaction-touch-android-readiness.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness.test.ts index e6a5fba15b..05482a34b8 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-android-readiness.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness.test.ts @@ -1,25 +1,26 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { expireRefFrame } from '@agent-device/daemon/testing/ref-frame'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { expireRefFrame } from '../../../core/ref-frame.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - makeAndroidSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { contextFromFlags, makeAndroidSession } from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // The Android device state a dispatch has to survive: an escape to launcher or // Settings fails the press, a permission prompt warns instead, and // blocking-dialog recovery aborts an admitted @ref rather than retargeting it. type EnsureAndroidBlockingSystemDialogReady = - typeof import('@agent-device/daemon/testing/android-system-dialog').ensureAndroidBlockingSystemDialogReady; + typeof import('../../android-system-dialog.ts').ensureAndroidBlockingSystemDialogReady; const { androidDialogReadiness } = vi.hoisted(() => ({ androidDialogReadiness: { @@ -28,10 +29,6 @@ const { androidDialogReadiness } = vi.hoisted(() => ({ }, })); -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { const actual = await importOriginal(); return { @@ -42,21 +39,10 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - // The blocking-dialog readiness check runs for real (its focus probe is mocked // above); the recovery regression below swaps in a recovering implementation. -vi.mock('@agent-device/daemon/testing/android-system-dialog', async (importOriginal) => { - const actual = - await importOriginal(); +vi.mock('../../android-system-dialog.ts', async (importOriginal) => { + const actual = await importOriginal(); androidDialogReadiness.actual = actual.ensureAndroidBlockingSystemDialogReady; return { ...actual, ensureAndroidBlockingSystemDialogReady: androidDialogReadiness.spy }; }); @@ -66,11 +52,9 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); beforeEach(() => { resetGetRuntimeFixture(); @@ -80,9 +64,8 @@ beforeEach(() => { mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); mockGetAndroidScreenSize.mockReset(); mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); androidDialogReadiness.spy.mockReset(); androidDialogReadiness.spy.mockImplementation((params) => androidDialogReadiness.actual(params)); }); @@ -115,7 +98,7 @@ test('press @ref fails when Android tap escapes to launcher', async () => { await expect( handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -164,7 +147,7 @@ test('press @ref fails when Android tap escapes to Settings', async () => { await expect( handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -219,7 +202,7 @@ test.each(ANDROID_PERMISSION_PROMPT_PACKAGES)( }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -279,7 +262,7 @@ test('a ref action aborts with the shared ref_frame_expired rejection after Andr sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, command: 'press', positionals: ['@e1'], flags: {} }, sessionName, sessionStore, diff --git a/src/__tests__/daemon-composition/interaction-touch-android-readiness-spawns.test.ts b/src/__tests__/daemon-composition/interaction-touch-android-readiness-spawns.test.ts deleted file mode 100644 index 4a01cdd4ca..0000000000 --- a/src/__tests__/daemon-composition/interaction-touch-android-readiness-spawns.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { expect, test } from 'vitest'; -import { makeAndroidSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { withFakeAdb } from '../test-utils/fake-adb.ts'; -import { expireRefFrame } from '@agent-device/daemon/testing/ref-frame'; -import { runWithAndroidDialogReadinessCheck } from '@agent-device/daemon/testing/interaction/internal/interaction-touch-android-readiness'; -import { androidObservationPolicyFixture as androidObservation } from '@agent-device/daemon/testing/__tests__/android-observation-fixture'; - -// What blocking-dialog readiness costs a run of taps, end to end. Every one of these `dumpsys` -// spawns is a device round trip on the critical path of an interaction, so the budget is pinned -// here rather than left to whatever the composition of the readiness seams happens to produce. - -const NORMAL_FOCUS_DUMP = - ' mCurrentFocus=Window{a1b2c3 u0 com.agentdevice.tester/com.agentdevice.tester.MainActivity}'; - -test('a run of taps on a clear screen costs one window dump per tap', async () => { - const session = makeAndroidSession('readiness-spawn-budget'); - - const dumps = await withFakeAdb( - (args) => (args[1] === 'dumpsys' ? NORMAL_FOCUS_DUMP : ''), - async ({ calls }) => { - for (let tap = 0; tap < 3; tap += 1) { - const outcome = await runWithAndroidDialogReadinessCheck( - session, - 'press', - { refContext: undefined }, - async () => { - // The dispatch seam every interaction crosses (ADR 0014). - expireRefFrame(session); - return { pressed: true }; - }, - androidObservation, - ); - expect(outcome.aborted).toBe(false); - } - return calls.filter((args) => args[1] === 'dumpsys').length; - }, - ); - - // Two checks per tap, one dump each: the focused-window line answers the question, so the - // second `dumpsys` variant never runs on a clear screen. Neither check may be skipped — a dialog - // surfaces with no adb traffic at all, so the previous check is evidence about nothing. - expect(dumps).toBe(6); -}); From c0e77558ffef45179ca1a76eb055cf1539706a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:33:52 +0200 Subject: [PATCH 07/13] refactor(daemon): place the direct-iOS selector tap tests beside the press handler interaction-touch-direct-ios.test.ts (6) and interaction-touch-direct-ios-eligibility.test.ts (4) are (a) daemon logic: the Maestro-only direct selector route, its error passthrough, the fused ref-frame seam, and the maestro-direct dispatch-disclosure rows. They move to execution/interaction/internal/. The dispatch-disclosure driver owner and the boundary-fault-matrix evidence path follow the moved driver file. Co-Authored-By: Claude Opus 5.5 --- .../src/dispatch-disclosure.fixtures.ts | 2 +- ...ction-touch-direct-ios-eligibility.test.ts | 82 +++++-------------- .../interaction-touch-direct-ios.test.ts | 44 ++++------ .../test-kit/src/boundary-fault-matrix.ts | 2 +- 4 files changed, 41 insertions(+), 89 deletions(-) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-touch-direct-ios-eligibility.test.ts (66%) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-touch-direct-ios.test.ts (73%) diff --git a/packages/contracts/src/dispatch-disclosure.fixtures.ts b/packages/contracts/src/dispatch-disclosure.fixtures.ts index 89ed8507ce..9bee01c77d 100644 --- a/packages/contracts/src/dispatch-disclosure.fixtures.ts +++ b/packages/contracts/src/dispatch-disclosure.fixtures.ts @@ -56,7 +56,7 @@ export const DISPATCH_DISCLOSURE_DRIVER_OWNERS: Readonly> 'web-agent-browser.': 'packages/platform-web/src/__tests__/agent-browser-provider-dispatch-disclosure.test.ts', 'maestro-direct.': - 'src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios.test.ts', + 'packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.test.ts', }; /** The per-row driving loop every driver file carries. */ diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios-eligibility.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios-eligibility.test.ts similarity index 66% rename from src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios-eligibility.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios-eligibility.test.ts index 37c14bb245..7c27f2f5e3 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios-eligibility.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios-eligibility.test.ts @@ -1,67 +1,29 @@ -import { test, expect, vi, beforeEach } from 'vitest'; +import { test, expect, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; import { getRuntimeBindings, mockFillPoint, mockTapElementSelector, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { contextFromFlags } from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { contextFromFlags } from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; // Ordinary selectors stay capture-backed. Only the explicit Maestro // non-hittable fallback may use the direct iOS selector route. -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - beforeEach(() => { resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test('ordinary click uses canonical capture even when the owner binds selector tap', async () => { @@ -71,7 +33,7 @@ test('ordinary click uses canonical capture even when the owner binds selector t sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' }), ); - mockCaptureSnapshotForSession.mockResolvedValueOnce({ + mockInteractorSnapshot.mockResolvedValueOnce({ nodes: attachRefs([ { index: 0, @@ -93,7 +55,7 @@ test('ordinary click uses canonical capture even when the owner binds selector t }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -109,7 +71,7 @@ test('ordinary click uses canonical capture even when the owner binds selector t expect(response?.ok).toBe(true); expect(mockTapElementSelector).not.toHaveBeenCalled(); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); expect(mockTapPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 90, y: 100 } })); }); @@ -121,7 +83,7 @@ test('fill simple iOS id selector resolves runtime text input evidence before co makeIosSession(sessionName, { appBundleId: 'com.example.app' }), ); - mockCaptureSnapshotForSession.mockResolvedValueOnce({ + mockInteractorSnapshot.mockResolvedValueOnce({ nodes: attachRefs([ { index: 0, @@ -151,7 +113,7 @@ test('fill simple iOS id selector resolves runtime text input evidence before co }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -166,7 +128,7 @@ test('fill simple iOS id selector resolves runtime text input evidence before co }); expect(response?.ok).toBe(true); - expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(1); expect(mockFillPoint).toHaveBeenCalledWith( expect.objectContaining({ point: { x: 190, y: 102 }, @@ -199,7 +161,7 @@ test('click simple iOS selector forwards Maestro non-hittable coordinate fallbac }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -243,7 +205,7 @@ test('click simple iOS id selector waits for snapshot path after pending gesture }; sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: attachRefs([ { index: 0, @@ -265,7 +227,7 @@ test('click simple iOS id selector waits for snapshot path after pending gesture }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.test.ts similarity index 73% rename from src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.test.ts index 49f4c5f381..659329ae34 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.test.ts @@ -6,43 +6,33 @@ import { DISPATCH_DISCLOSURE_TABLE_PATH, dispatchDisclosureRowsOwnedBy, } from '@agent-device/contracts/dispatch-disclosure-fixtures'; -import { beforeEach, expect, test, vi } from 'vitest'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { beforeEach, expect, test } from 'vitest'; +import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { runInteraction } from '../../../__tests__/support/touch-interaction-runner.ts'; +import { refFrameState } from '../../../core/ref-frame.ts'; import { getRuntimeBindings, mockTapElementSelector, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeStaleRefSession, makeTwoButtonNodes, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { runInteraction } from './interaction-touch-runner.ts'; -import { refFrameState } from '@agent-device/daemon/testing/ref-frame'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); +} from './__tests__/interaction-touch-fixtures.ts'; +import { handleInteractionCommands } from '../index.ts'; beforeEach(() => { resetGetRuntimeFixture(); - vi.mocked(captureSnapshotWithInteractor).mockReset(); + installInteractorSnapshotResolution(); + mockInteractorSnapshot.mockReset(); }); test.each([ @@ -60,7 +50,7 @@ test.each([ mockTapElementSelector.mockRejectedValue(new AppError(code, message)); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -128,7 +118,7 @@ for (const row of ROWS) { const sessionName = `maestro-direct-${row.id}`; sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); mockTapElementSelector.mockRejectedValueOnce(failure()); - vi.mocked(captureSnapshotWithInteractor).mockResolvedValue({ + mockInteractorSnapshot.mockResolvedValue({ nodes: makeTwoButtonNodes(), backend: 'xctest', producer: 'apple-runner', diff --git a/packages/test-kit/src/boundary-fault-matrix.ts b/packages/test-kit/src/boundary-fault-matrix.ts index 7b55945c2a..4a8a7c0055 100644 --- a/packages/test-kit/src/boundary-fault-matrix.ts +++ b/packages/test-kit/src/boundary-fault-matrix.ts @@ -196,7 +196,7 @@ export const BOUNDARY_FAULT_MATRIX = { mutation: { kind: 'covered', evidence: [ - 'src/__tests__/daemon-composition/interaction/interaction-touch-direct-ios.test.ts', + 'packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.test.ts', ], invariants: ['best-effort-degradation'], }, From 48c5540cedd9083802a6b930231455557b909c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:34:02 +0200 Subject: [PATCH 08/13] refactor(daemon): place the iOS tap outcome and drag tests beside their modules interaction-ios-tap-outcome.test.ts (22) and interaction-gesture-drag.test.ts (2) are (a) daemon logic: same-scope corroboration of a failed iOS tap, the recorded drag with both endpoint identities, one inspect/bind per drag, and frame expiry. They move to execution/interaction/internal/ on the touch surface fake (which now answers a target-authored drag through one backend plan). The drag endpoint resolution and disclosure stay covered by src/commands/interaction/runtime/gestures.test.ts. Co-Authored-By: Claude Opus 5.5 --- .../interaction-gesture-drag.test.ts | 20 +++---- .../interaction-ios-tap-outcome.test.ts | 55 +++++++++---------- 2 files changed, 36 insertions(+), 39 deletions(-) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-gesture-drag.test.ts (85%) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-ios-tap-outcome.test.ts (94%) diff --git a/src/__tests__/daemon-composition/interaction/interaction-gesture-drag.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-gesture-drag.test.ts similarity index 85% rename from src/__tests__/daemon-composition/interaction/interaction-gesture-drag.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-gesture-drag.test.ts index e9b4d7a1be..b8315a8f12 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-gesture-drag.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-gesture-drag.test.ts @@ -1,21 +1,21 @@ -import { composedPlatformServicesFixture } from '../platform-services-fixture.ts'; import { beforeEach, expect, test } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { makeIosSession, authoringPublication, -} from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { activateCompleteRefFrame, refFrameState } from '@agent-device/daemon/testing/ref-frame'; +} from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; +import { activateCompleteRefFrame, refFrameState } from '../../../core/ref-frame.ts'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { gestureRuntimeBindingsFixture } from '@agent-device/daemon/testing/interaction/internal/__tests__/gesture-runtime-bindings.fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +import { handleInteractionCommands } from '../index.ts'; +import { gestureRuntimeBindingsFixture } from './__tests__/gesture-runtime-bindings.fixtures.ts'; const contextFromFlags = () => ({}); const captureSnapshotForSession = async ({ session, -}: import('@agent-device/daemon/testing/session-state').SessionRef) => session.snapshot!; +}: import('../../../core/session-state.ts').SessionRef) => session.snapshot!; let gestures = gestureRuntimeBindingsFixture(); beforeEach(() => { @@ -63,8 +63,8 @@ function makeDragSession(sessionName: string) { async function runDrag(sessionStore: ReturnType, sessionName: string) { return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - platformServices: composedPlatformServicesFixture(), + createCommandSurface: touchCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), req: { token: 't', session: sessionName, diff --git a/src/__tests__/daemon-composition/interaction/interaction-ios-tap-outcome.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.test.ts similarity index 94% rename from src/__tests__/daemon-composition/interaction/interaction-ios-tap-outcome.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.test.ts index c960608bf2..361270a012 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-ios-tap-outcome.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.test.ts @@ -1,5 +1,8 @@ import type { CommandFlags } from '@agent-device/contracts/command'; -import { legacyDispatchCapture } from '@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'; +import { + legacyDispatchCapture, + resetLegacySnapshotCapture, +} from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; import { beforeEach, expect, test, vi } from 'vitest'; import fs from 'node:fs'; @@ -7,47 +10,40 @@ import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot'; import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; +import { handleInteractionCommands } from '../index.ts'; import { makeIosSession, authoringPublication, -} from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { SessionScriptWriter } from '@agent-device/daemon/testing/session-script-writer'; -import { runReplayForTest } from '@agent-device/daemon/testing/__tests__/replay-runtime/replay-command-fixture'; +} from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { SessionScriptWriter } from '../../../core/session-script-writer.ts'; +import { runReplayForTest } from '../../../sessions/__tests__/replay-runtime/replay-command-fixture.ts'; import { replayScriptSourceBundleFor } from '@agent-device/replay-port/replay-script-source-fixtures'; import { imageViewerNodes, profileNodes, snapshot, snapshotPayload, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures'; +} from './__tests__/interaction-ios-tap-outcome-fixtures.ts'; import { appCaptureComparisonIdentity, appCaptureComparisonKey, systemSurfaceCaptureComparisonKey, -} from '@agent-device/daemon/testing/__tests__/ios-comparison-key-fixture'; +} from '../../../core/__tests__/ios-comparison-key-fixture.ts'; import { IOS_SIMULATOR } from '@agent-device/test-kit/device-fixtures'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -import { corroborateIosTapFailure } from '@agent-device/daemon/testing/interaction/internal/interaction-ios-tap-outcome'; -import { movingTargetBaseline } from '@agent-device/daemon/testing/deferred-interaction-outcome'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../../../__tests__/support/interactor-resolution-fixture.ts'; +import { corroborateIosTapFailure } from './interaction-ios-tap-outcome.ts'; +import { movingTargetBaseline } from '../../../core/deferred-interaction-outcome.ts'; import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', async () => { - const fixture = - await import('@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'); - return { - captureSnapshotWithInteractor: vi.fn(fixture.captureSnapshotThroughLegacyDispatchFixture), - }; -}); - -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); +import { touchCreateCommandSurface } from '../../../__tests__/support/touch-command-surface-fake.ts'; const contextFromFlags = (flags: CommandFlags | undefined) => ({ count: flags?.count, @@ -70,7 +66,7 @@ async function runClick( const session = sessionStore.get(sessionName); if (session?.snapshot && !session.scriptPublication && options.positionals?.length !== 2) { const baseline = session.snapshot; - mockCaptureSnapshotForSession.mockResolvedValueOnce({ + mockInteractorSnapshot.mockResolvedValueOnce({ nodes: baseline.nodes, backend: 'xctest', producer: 'apple-runner', @@ -78,7 +74,7 @@ async function runClick( }); } return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 'test', session: sessionName, @@ -95,7 +91,8 @@ async function runClick( beforeEach(() => { resetGetRuntimeFixture(); - mockCaptureSnapshotForSession.mockClear(); + installInteractorSnapshotResolution(); + resetLegacySnapshotCapture(mockInteractorSnapshot); legacyDispatchCapture.mockReset(); mockTapPoint.mockImplementation(async (input) => { return await legacyDispatchCapture( @@ -580,7 +577,7 @@ test('corroborates a tap when the request carries no flags and the baseline used // mirrors the raw daemon/JSON-RPC production boundary, which can omit the // key entirely. CLI and batch paths always materialize a flags object. const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 'test', session: sessionName, @@ -839,7 +836,7 @@ test('corroborated runtime taps retain target evidence through save and replay', sessionStore: replayStore, invoke: async (req) => { const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req, sessionName: replaySessionName, sessionStore: replayStore, @@ -873,7 +870,7 @@ test('a tap corroborates against its admitted snapshot after a same-lifetime rec return {}; }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 'test', session: name, command: 'click', positionals: ['104', '222'], flags: {} }, sessionName: name, sessionRef: ref, From b1252a46cadb061f151f032b3671a781821384b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:34:02 +0200 Subject: [PATCH 09/13] refactor(daemon): split the get and is read tests by what each asserts interaction-get.test.ts: (a) the 9 get tests (bound read expansion, no live read, retired direct selector, frame-tree @ref reads and their warnings) move to execution/selector-runtime-get-reads .test.ts and the 5 is tests that pin capture wiring and the Android foreground blocker move to execution/selector-runtime-is-reads.test.ts. The two predicate-semantics tests (is visible inheriting ancestor viewport visibility, is visible failing outside the viewport) are (c) in src/commands/interaction/runtime/selector-is-visibility.test.ts, and the same-label disambiguation read is (c) in selector-read-disambiguation.test.ts. Nothing imports the root interaction-touch-runner helper any more, so it is deleted. Co-Authored-By: Claude Opus 5.5 --- .../selector-runtime-get-reads.test.ts | 409 ++---------------- .../selector-runtime-is-reads.test.ts | 269 ++++++++++++ .../interaction/interaction-touch-runner.ts | 50 --- .../runtime/selector-is-visibility.test.ts | 68 +++ .../selector-read-disambiguation.test.ts | 72 +++ 5 files changed, 436 insertions(+), 432 deletions(-) rename src/__tests__/daemon-composition/interaction/interaction-get.test.ts => packages/daemon/src/execution/selector-runtime-get-reads.test.ts (52%) create mode 100644 packages/daemon/src/execution/selector-runtime-is-reads.test.ts delete mode 100644 src/__tests__/daemon-composition/interaction/interaction-touch-runner.ts create mode 100644 src/commands/interaction/runtime/selector-is-visibility.test.ts create mode 100644 src/commands/interaction/runtime/selector-read-disambiguation.test.ts diff --git a/src/__tests__/daemon-composition/interaction/interaction-get.test.ts b/packages/daemon/src/execution/selector-runtime-get-reads.test.ts similarity index 52% rename from src/__tests__/daemon-composition/interaction/interaction-get.test.ts rename to packages/daemon/src/execution/selector-runtime-get-reads.test.ts index 0694700b81..5d6d13a490 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-get.test.ts +++ b/packages/daemon/src/execution/selector-runtime-get-reads.test.ts @@ -1,50 +1,33 @@ import { test, expect, vi, beforeEach } from 'vitest'; -import { legacyDispatchCapture } from '@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'; -import { attachRefs } from '@agent-device/kernel/snapshot'; -import { WEB_DESKTOP_DEVICE } from '@agent-device/test-kit/device-fixtures'; import { - makeAndroidSession as makeBaseAndroidSession, - makeIosSession, -} from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { expireRefFrame } from '@agent-device/daemon/testing/ref-frame'; + legacyDispatchCapture, + resetLegacySnapshotCapture, +} from '../__tests__/support/legacy-snapshot-capture-fixture.ts'; +import { attachRefs } from '@agent-device/kernel/snapshot'; +import { makeIosSession } from '../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../__tests__/support/store-factory.ts'; +import { expireRefFrame } from '../core/ref-frame.ts'; +import { setSessionSnapshot, STALE_SNAPSHOT_REFS_WARNING } from './session-snapshot.ts'; import { - setSessionSnapshot, - STALE_SNAPSHOT_REFS_WARNING, -} from '@agent-device/daemon/testing/session-snapshot'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../__tests__/support/touch-command-surface-fake.ts'; +import { runInteraction } from '../__tests__/support/touch-interaction-runner.ts'; +import { handleInteractionCommands } from './interaction/index.ts'; import { contextFromFlags, makeSession, makeStaleRefSession, - makeVisibleButtonSnapshot, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { runInteraction } from './interaction-touch-runner.ts'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './interaction/internal/__tests__/interaction-touch-fixtures.ts'; -// Non-touch interaction routing: the `get` and `is` reads the public handler -// owns. Touch commands live in the interaction-touch* test files. +// The `get` reads the public interaction handler owns: which capture or bound operation answers a +// selector or `@ref` read, and the frame-derived warnings a `@ref` read carries. const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn(), })); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', async () => { - const fixture = - await import('@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'); - return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; -}); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { const actual = await importOriginal(); @@ -60,20 +43,11 @@ import { getRuntimeBindings, mockReadTextAtPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, -} from '@agent-device/platform-android/mechanics'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); +} from './__tests__/interaction-get-runtime-fixture.ts'; + beforeEach(() => { - legacyDispatchCapture.mockReset(); - legacyDispatchCapture.mockResolvedValue({}); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); + installInteractorSnapshotResolution(); + resetLegacySnapshotCapture(mockInteractorSnapshot); mockRunAppleRunnerCommand.mockReset(); mockRunAppleRunnerCommand.mockResolvedValue({}); resetGetRuntimeFixture(); @@ -104,7 +78,7 @@ test('get text prefers underlying value for text surfaces and avoids recording g ); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -156,7 +130,7 @@ test('get text uses backend read expansion when the resolved node has a rect', a ); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -203,7 +177,7 @@ test('get text answers from the captured tree when the bound owner advertises no elementReadFixtureState.readTextAtPointAvailable = false; const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -246,7 +220,7 @@ test('an eligible direct iOS selector cannot operate before admission', async () }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -306,7 +280,7 @@ test('get text simple iOS id selector resolves through the bound capture, not a }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -386,7 +360,7 @@ test('get text iOS label selector uses snapshot disambiguation instead of runner }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchCreateCommandSurface, req: { token: 't', session: sessionName, @@ -410,335 +384,6 @@ test('get text iOS label selector uses snapshot disambiguation instead of runner } }); -test('is visible preserves CLI snapshot flags during runtime snapshot capture', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'snapshot-flags'; - sessionStore.publish(sessionName, makeSession(sessionName)); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeWindow', - label: 'Login', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeButton', - label: 'Continue', - identifier: 'auth_continue', - rect: { x: 10, y: 20, width: 100, height: 40 }, - enabled: true, - hittable: true, - visible: true, - }, - ], - backend: 'xctest', - producer: 'apple-runner', - }; - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'id=auth_continue'], - flags: { snapshotDepth: 2, snapshotScope: 'Login', snapshotRaw: true }, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ - snapshotDepth: 2, - snapshotScope: 'Login', - snapshotRaw: true, - snapshotInteractiveOnly: false, - snapshotIncludeRects: true, - }); -}); - -test('is visible reuses fresh cached iOS snapshots with rects', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'ios-visible-cached'; - const session = makeSession(sessionName); - session.snapshot = makeVisibleButtonSnapshot('Cached action', { - backend: 'xctest', - producer: 'apple-runner', - }); - sessionStore.publish(sessionName, session); - legacyDispatchCapture.mockRejectedValue(new Error('unexpected fresh snapshot')); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Cached action"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(legacyDispatchCapture).not.toHaveBeenCalled(); -}); - -test('is visible recaptures web snapshots when cached nodes may lack rects', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'web-visible-refreshes-rects'; - const session = makeSession(sessionName); - session.device = WEB_DESKTOP_DEVICE; - session.snapshot = buildSnapshotState( - { - nodes: [{ index: 0, type: 'button', label: 'Submit order' }], - backend: 'web', - producer: 'agent-browser', - }, - { snapshotInteractiveOnly: false }, - ); - sessionStore.publish(sessionName, session); - legacyDispatchCapture.mockResolvedValue( - makeVisibleButtonSnapshot('Submit order', { - backend: 'web', - producer: 'agent-browser', - }), - ); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Submit order"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ - snapshotIncludeRects: true, - }); -}); - -// PIN CHANGED TWICE (#1739, R37). #557 asserted `ok: true` with `pass: false` and zero snapshots -// here, from the direct-iOS shortcut. That broke `is`'s documented contract — it "exits non-zero -// on failure" (website/docs/docs/commands.md) — and on device printed `Passed: is text` with exit -// 0 for a failed assertion. The reversal made the shortcut answer only when the predicate held; -// the shortcut is now retired outright, so the bound capture answers every predicate and this is -// simply what `is` does. The assertion below is unchanged across both edits because it was always -// about the OUTCOME, not about which path produced it. -test('a failing is predicate is COMMAND_FAILED, never a zero-exit pass', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'is-selected-ios-direct-selector-false'; - sessionStore.publish( - sessionName, - makeIosSession(sessionName, { appBundleId: 'com.example.app' }), - ); - mockRunAppleRunnerCommand.mockResolvedValue({ - found: true, - nodes: [ - { - index: 0, - depth: 0, - type: 'Button', - label: 'Submit', - identifier: 'submit', - selected: false, - rect: { x: 126, y: 555, width: 75, height: 38 }, - }, - ], - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['selected', 'id="submit"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - // The session snapshot has no `id=submit`, so the bound capture reports the typed selector - // failure. Nothing can report a failed assertion as a completed command. - expect(response?.ok).toBe(false); - expect(legacyDispatchCapture.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(1); - if (response?.ok === false) { - expect(response.error?.code).toBe('COMMAND_FAILED'); - } -}); - -test('is visible passes for list text that inherits viewport visibility from an ancestor', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'visible-list-item'; - sessionStore.publish(sessionName, makeSession(sessionName)); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeCell', - rect: { x: 0, y: 160, width: 390, height: 44 }, - hittable: false, - }, - { - index: 2, - parentIndex: 1, - type: 'XCUIElementTypeStaticText', - label: 'Trip ideas', - hittable: false, - }, - ], - backend: 'xctest', - producer: 'apple-runner', - }; - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Trip ideas"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - if (response?.ok) { - expect(response.data?.predicate).toBe('visible'); - expect(response.data?.pass).toBe(true); - expect(response.data?.selector).toBe('label="Trip ideas"'); - } -}); - -test('is visible fails for nodes outside the current viewport', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'visible-offscreen'; - sessionStore.publish(sessionName, makeSession(sessionName)); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeStaticText', - label: 'Far item', - rect: { x: 20, y: 2600, width: 120, height: 40 }, - hittable: false, - }, - ], - backend: 'xctest', - producer: 'apple-runner', - }; - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Far item"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/actual=\{"visible":false/); - } -}); - -test('is reports Android permission dialog blocker when app content assertion fails', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'android-permission-blocked'; - sessionStore.publish( - sessionName, - makeBaseAndroidSession(sessionName, { appBundleId: 'com.example.demo' }), - ); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { nodes: [], backend: 'uiautomator' }; - }); - mockGetAndroidAppState.mockResolvedValue({ - package: 'com.google.android.permissioncontroller', - activity: 'com.android.permissioncontroller.permission.ui.GrantPermissionsActivity', - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Metro Ready"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.message).toMatch(/permission dialog is blocking/); - expect(response.error.details).toMatchObject({ - blockedBy: 'android_foreground_surface', - expectedPackage: 'com.example.demo', - foregroundPackage: 'com.google.android.permissioncontroller', - }); - } -}); - // fallow-ignore-next-line complexity test('ADR 0014 evidence #17: get text @ref reads the retained frame tree, not a newer observation', async () => { const sessionStore = makeSessionStore(); diff --git a/packages/daemon/src/execution/selector-runtime-is-reads.test.ts b/packages/daemon/src/execution/selector-runtime-is-reads.test.ts new file mode 100644 index 0000000000..b2c28697ea --- /dev/null +++ b/packages/daemon/src/execution/selector-runtime-is-reads.test.ts @@ -0,0 +1,269 @@ +import { test, expect, vi, beforeEach } from 'vitest'; +import { + legacyDispatchCapture, + resetLegacySnapshotCapture, +} from '../__tests__/support/legacy-snapshot-capture-fixture.ts'; +import { WEB_DESKTOP_DEVICE } from '@agent-device/test-kit/device-fixtures'; +import { + makeAndroidSession as makeBaseAndroidSession, + makeIosSession, +} from '../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../__tests__/support/store-factory.ts'; +import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; +import { + installInteractorSnapshotResolution, + mockInteractorSnapshot, +} from '../__tests__/support/interactor-resolution-fixture.ts'; +import { touchCreateCommandSurface } from '../__tests__/support/touch-command-surface-fake.ts'; +import { handleInteractionCommands } from './interaction/index.ts'; +import { + contextFromFlags, + makeSession, + makeVisibleButtonSnapshot, +} from './interaction/internal/__tests__/interaction-touch-fixtures.ts'; + +// The `is` reads the public interaction handler owns: the capture each predicate takes, and the +// Android foreground blocker a failed assertion discloses. + +vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getAndroidAppState: vi.fn(async () => ({})), + getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), + }; +}); + +import { + getRuntimeBindings, + resetGetRuntimeFixture, +} from './__tests__/interaction-get-runtime-fixture.ts'; +import { + getAndroidAppState, + getAndroidBlockingDialogObservation, +} from '@agent-device/platform-android/mechanics'; +const mockGetAndroidAppState = vi.mocked(getAndroidAppState); +const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); + +beforeEach(() => { + installInteractorSnapshotResolution(); + resetLegacySnapshotCapture(mockInteractorSnapshot); + mockGetAndroidAppState.mockReset(); + mockGetAndroidAppState.mockResolvedValue({}); + mockGetAndroidBlockingDialogObservation.mockReset(); + mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); + resetGetRuntimeFixture(); +}); + +test('is visible preserves CLI snapshot flags during runtime snapshot capture', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'snapshot-flags'; + sessionStore.publish(sessionName, makeSession(sessionName)); + + legacyDispatchCapture.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); + return { + nodes: [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeWindow', + label: 'Login', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + identifier: 'auth_continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + visible: true, + }, + ], + backend: 'xctest', + producer: 'apple-runner', + }; + }); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'id=auth_continue'], + flags: { snapshotDepth: 2, snapshotScope: 'Login', snapshotRaw: true }, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ + snapshotDepth: 2, + snapshotScope: 'Login', + snapshotRaw: true, + snapshotInteractiveOnly: false, + snapshotIncludeRects: true, + }); +}); + +test('is visible reuses fresh cached iOS snapshots with rects', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-visible-cached'; + const session = makeSession(sessionName); + session.snapshot = makeVisibleButtonSnapshot('Cached action', { + backend: 'xctest', + producer: 'apple-runner', + }); + sessionStore.publish(sessionName, session); + legacyDispatchCapture.mockRejectedValue(new Error('unexpected fresh snapshot')); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Cached action"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(legacyDispatchCapture).not.toHaveBeenCalled(); +}); + +test('is visible recaptures web snapshots when cached nodes may lack rects', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'web-visible-refreshes-rects'; + const session = makeSession(sessionName); + session.device = WEB_DESKTOP_DEVICE; + session.snapshot = buildSnapshotState( + { + nodes: [{ index: 0, type: 'button', label: 'Submit order' }], + backend: 'web', + producer: 'agent-browser', + }, + { snapshotInteractiveOnly: false }, + ); + sessionStore.publish(sessionName, session); + legacyDispatchCapture.mockResolvedValue( + makeVisibleButtonSnapshot('Submit order', { + backend: 'web', + producer: 'agent-browser', + }), + ); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Submit order"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ + snapshotIncludeRects: true, + }); +}); + +// PIN CHANGED TWICE (#1739, R37). #557 asserted `ok: true` with `pass: false` and zero snapshots +// here, from the direct-iOS shortcut. That broke `is`'s documented contract — it "exits non-zero +// on failure" (website/docs/docs/commands.md) — and on device printed `Passed: is text` with exit +// 0 for a failed assertion. The reversal made the shortcut answer only when the predicate held; +// the shortcut is now retired outright, so the bound capture answers every predicate and this is +// simply what `is` does. The assertion below is unchanged across both edits because it was always +// about the OUTCOME, not about which path produced it. +test('a failing is predicate is COMMAND_FAILED, never a zero-exit pass', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'is-selected-ios-direct-selector-false'; + sessionStore.publish( + sessionName, + makeIosSession(sessionName, { appBundleId: 'com.example.app' }), + ); + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['selected', 'id="submit"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + // The session snapshot has no `id=submit`, so the bound capture reports the typed selector + // failure. Nothing can report a failed assertion as a completed command. + expect(response?.ok).toBe(false); + expect(legacyDispatchCapture.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(1); + if (response?.ok === false) { + expect(response.error?.code).toBe('COMMAND_FAILED'); + } +}); + +test('is reports Android permission dialog blocker when app content assertion fails', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'android-permission-blocked'; + sessionStore.publish( + sessionName, + makeBaseAndroidSession(sessionName, { appBundleId: 'com.example.demo' }), + ); + + legacyDispatchCapture.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); + return { nodes: [], backend: 'uiautomator' }; + }); + mockGetAndroidAppState.mockResolvedValue({ + package: 'com.google.android.permissioncontroller', + activity: 'com.android.permissioncontroller.permission.ui.GrantPermissionsActivity', + }); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Metro Ready"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toBeTruthy(); + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.message).toMatch(/permission dialog is blocking/); + expect(response.error.details).toMatchObject({ + blockedBy: 'android_foreground_surface', + expectedPackage: 'com.example.demo', + foregroundPackage: 'com.google.android.permissioncontroller', + }); + } +}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch-runner.ts b/src/__tests__/daemon-composition/interaction/interaction-touch-runner.ts deleted file mode 100644 index c540cc58ba..0000000000 --- a/src/__tests__/daemon-composition/interaction/interaction-touch-runner.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { getRuntimeBindings } from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { contextFromFlags } from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; -import type { SessionStore } from '@agent-device/daemon/testing/session-store'; - -/** - * The real command-surface legs the relocated interaction suites drive through the handler. - * The daemon-policy suites keep their fake surface and the shared factories in - * `interaction-touch-fixtures.ts`; running the real command surface is the composition these - * suites moved here to pin, so the runners live beside them. - */ -export async function runInteraction( - sessionStore: SessionStore, - sessionName: string, - command: string, - positionals: string[], - flags: Record = {}, -) { - return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { token: 't', session: sessionName, command, positionals, flags }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); -} - -export async function runFindInternalClick( - sessionStore: SessionStore, - sessionName: string, - internal: Record, -) { - return await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'click', - positionals: ['@e2'], - flags: { noRecord: true }, - internal, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); -} diff --git a/src/commands/interaction/runtime/selector-is-visibility.test.ts b/src/commands/interaction/runtime/selector-is-visibility.test.ts new file mode 100644 index 0000000000..f3405c2d79 --- /dev/null +++ b/src/commands/interaction/runtime/selector-is-visibility.test.ts @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { selector } from './selector-read-utils.ts'; +import { createSelectorDevice } from './__tests__/test-utils/index.ts'; + +// What `is visible` decides about a matched node: it inherits the viewport visibility of an +// ancestor that has a rect, and a node outside the viewport fails the assertion. + +test('runtime is visible passes for list text that inherits viewport visibility from an ancestor', async () => { + const snapshot = makeSnapshotState( + [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeCell', + rect: { x: 0, y: 160, width: 390, height: 44 }, + hittable: false, + }, + { + index: 2, + parentIndex: 1, + type: 'XCUIElementTypeStaticText', + label: 'Trip ideas', + hittable: false, + }, + ], + { backend: 'xctest', producer: 'apple-runner' }, + ); + const device = createSelectorDevice(snapshot); + + const result = await device.selectors.isVisible(selector('label="Trip ideas"'), { + session: 'default', + }); + + assert.equal(result.predicate, 'visible'); + assert.equal(result.pass, true); + assert.equal(result.selector, 'label="Trip ideas"'); +}); + +test('runtime is visible fails for nodes outside the current viewport', async () => { + const snapshot = makeSnapshotState( + [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Far item', + rect: { x: 20, y: 2600, width: 120, height: 40 }, + hittable: false, + }, + ], + { backend: 'xctest', producer: 'apple-runner' }, + ); + const device = createSelectorDevice(snapshot); + + await assert.rejects( + () => device.selectors.isVisible(selector('label="Far item"'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /actual=\{"visible":false/); + return true; + }, + ); +}); diff --git a/src/commands/interaction/runtime/selector-read-disambiguation.test.ts b/src/commands/interaction/runtime/selector-read-disambiguation.test.ts new file mode 100644 index 0000000000..2070b265f8 --- /dev/null +++ b/src/commands/interaction/runtime/selector-read-disambiguation.test.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { selector } from './selector-read-utils.ts'; +import { createSelectorDevice } from './__tests__/test-utils/index.ts'; + +// A label selector that matches several nodes of one row resolves through the captured tree's +// disambiguation, never through a runner query. + +test('runtime get text on an iOS label selector disambiguates among same-label nodes in one capture', async () => { + const snapshot = makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 393, height: 852 }, + enabled: true, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Cell', + label: 'General', + rect: { x: 0, y: 100, width: 393, height: 48 }, + enabled: true, + hittable: true, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'General', + rect: { x: 0, y: 100, width: 393, height: 48 }, + enabled: true, + hittable: true, + }, + { + index: 3, + depth: 2, + parentIndex: 1, + type: 'StaticText', + label: 'General', + rect: { x: 18, y: 112, width: 80, height: 24 }, + enabled: true, + hittable: false, + }, + ], + { backend: 'xctest', producer: 'apple-runner' }, + ); + let captures = 0; + const device = createSelectorDevice(snapshot, { + captureSnapshot: () => { + captures += 1; + return { snapshot }; + }, + }); + + const result = await device.selectors.get({ + session: 'default', + property: 'text', + target: selector('label="General"'), + }); + + assert.equal(result.kind, 'text'); + assert.equal(result.text, 'General'); + assert.deepEqual(result.target, { kind: 'selector', selector: 'label="General"' }); + assert.equal(captures, 1); +}); From 6207002454c128fb752124f6f0bc46ba78959109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 15:39:58 +0200 Subject: [PATCH 10/13] test(daemon): delete the touch routing test the package copy replaces Co-Authored-By: Claude Opus 5.5 --- packages/daemon/package.json | 16 - .../interaction/interaction-touch.test.ts | 351 ------------------ 2 files changed, 367 deletions(-) delete mode 100644 src/__tests__/daemon-composition/interaction/interaction-touch.test.ts diff --git a/packages/daemon/package.json b/packages/daemon/package.json index d606016ff5..99b5ac44d0 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -56,10 +56,6 @@ "types": "./src/execution/__tests__/interaction-get-runtime-fixture.ts", "default": "./src/execution/__tests__/interaction-get-runtime-fixture.ts" }, - "./testing/__tests__/ios-comparison-key-fixture": { - "types": "./src/core/__tests__/ios-comparison-key-fixture.ts", - "default": "./src/core/__tests__/ios-comparison-key-fixture.ts" - }, "./testing/__tests__/legacy-snapshot-capture-fixture": { "types": "./src/__tests__/support/legacy-snapshot-capture-fixture.ts", "default": "./src/__tests__/support/legacy-snapshot-capture-fixture.ts" @@ -68,10 +64,6 @@ "types": "./src/__tests__/support/platform-services-fixture.ts", "default": "./src/__tests__/support/platform-services-fixture.ts" }, - "./testing/__tests__/replay-runtime/replay-command-fixture": { - "types": "./src/sessions/__tests__/replay-runtime/replay-command-fixture.ts", - "default": "./src/sessions/__tests__/replay-runtime/replay-command-fixture.ts" - }, "./testing/__tests__/request-router-dispatch-mocks": { "types": "./src/__tests__/support/request-router-dispatch-mocks.ts", "default": "./src/__tests__/support/request-router-dispatch-mocks.ts" @@ -196,14 +188,6 @@ "types": "./src/execution/interaction/internal/interaction-android-escape.ts", "default": "./src/execution/interaction/internal/interaction-android-escape.ts" }, - "./testing/interaction/internal/interaction-ios-tap-outcome": { - "types": "./src/execution/interaction/internal/interaction-ios-tap-outcome.ts", - "default": "./src/execution/interaction/internal/interaction-ios-tap-outcome.ts" - }, - "./testing/interaction/internal/interaction-touch-android-readiness": { - "types": "./src/execution/interaction/internal/interaction-touch-android-readiness.ts", - "default": "./src/execution/interaction/internal/interaction-touch-android-readiness.ts" - }, "./testing/interaction/internal/interaction-touch-response": { "types": "./src/execution/interaction/internal/interaction-touch-response.ts", "default": "./src/execution/interaction/internal/interaction-touch-response.ts" diff --git a/src/__tests__/daemon-composition/interaction/interaction-touch.test.ts b/src/__tests__/daemon-composition/interaction/interaction-touch.test.ts deleted file mode 100644 index 8139f29792..0000000000 --- a/src/__tests__/daemon-composition/interaction/interaction-touch.test.ts +++ /dev/null @@ -1,351 +0,0 @@ -import { test, expect, vi, beforeEach } from 'vitest'; -import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { WEB_DESKTOP_DEVICE } from '@agent-device/test-kit/device-fixtures'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { - getRuntimeBindings, - mockHoverPoint, - mockHoverRef, - mockLongPressPoint, - mockTapPoint, - resetGetRuntimeFixture, - runtimeBindingSpies, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - makeMacOsDesktopSession, - makeMacOsMenubarSession, - makeSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -// Router ownership: one representative per touch command proves -// handleInteractionCommands claims press/click/longpress/hover/fill. - -const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ - mockRunAppleRunnerCommand: vi.fn(), -})); - -vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - getAndroidScreenSize: vi.fn(async () => ({ width: 1344, height: 2992 })), - getAndroidAppState: vi.fn(async () => ({})), - getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), - }; -}); - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; -}); - -import { - getAndroidAppState, - getAndroidBlockingDialogObservation, - getAndroidScreenSize, -} from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -const mockGetAndroidAppState = vi.mocked(getAndroidAppState); -const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); -const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); -const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotWithInteractor); - -beforeEach(() => { - resetGetRuntimeFixture(); - mockGetAndroidAppState.mockReset(); - mockGetAndroidAppState.mockResolvedValue({}); - mockGetAndroidBlockingDialogObservation.mockReset(); - mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); - mockGetAndroidScreenSize.mockReset(); - mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); - mockCaptureSnapshotForSession.mockReset(); - mockRunAppleRunnerCommand.mockReset(); - mockRunAppleRunnerCommand.mockResolvedValue({}); -}); - -test('press coordinates dispatches press and records as press', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const storedSession = makeSession(sessionName); - sessionStore.publish(sessionName, storedSession); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['100', '200'], - flags: { count: 3, intervalMs: 1, doubleTap: true }, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - expect(mockTapPoint).toHaveBeenCalledOnce(); - expect(mockTapPoint.mock.calls[0]?.[0]).toMatchObject({ - point: { x: 100, y: 200 }, - options: { count: 3, intervalMs: 1, doubleTap: true }, - }); - - const session = sessionStore.get(sessionName); - expect(session).toBeTruthy(); - expect(session?.actions.length).toBe(1); - expect(session?.actions[0]?.command).toBe('press'); - expect(session?.actions[0]?.positionals).toEqual(['100', '200']); -}); - -test.each([ - ['click', ['100', '200'], false], - ['press', ['100', '200'], false], - ['longpress', ['100', '200', '800'], false], - ['hover', ['100', '200'], true], - ['fill', ['100', '200', 'hello'], false], -] as const)('%s inspects once and binds once', async (command, positionals, web) => { - const sessionStore = makeSessionStore(); - const sessionName = `single-bind-${command}`; - const session = makeSession(sessionName); - if (web) session.device = WEB_DESKTOP_DEVICE; - sessionStore.publish(sessionName, session); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { token: 't', session: sessionName, command, positionals: [...positionals], flags: {} }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(runtimeBindingSpies().inspectFacts).toHaveBeenCalledOnce(); - expect(runtimeBindingSpies().bindDevice).toHaveBeenCalledOnce(); -}); - -test('click rejects macOS desktop surface interactions until helper routing exists', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'macos-desktop-click'; - sessionStore.publish(sessionName, makeMacOsDesktopSession(sessionName)); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'click', - positionals: ['100', '200'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('UNSUPPORTED_OPERATION'); - expect(response.error.message).toMatch(/macOS desktop sessions/); - } -}); - -test('fill rejects macOS menubar surface interactions until helper routing exists', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'macos-menubar-fill'; - sessionStore.publish(sessionName, makeMacOsMenubarSession(sessionName)); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'fill', - positionals: ['@e2', 'hello'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('UNSUPPORTED_OPERATION'); - expect(response.error.message).toMatch(/macOS menubar sessions/); - } -}); - -test('longpress @ref resolves the target and dispatches coordinate longpress', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'longpress-ref'; - const session = makeSession(sessionName); - session.snapshot = { - nodes: attachRefs([ - { - index: 0, - type: 'XCUIElementTypeStaticText', - label: 'Last message', - rect: { x: 10, y: 20, width: 100, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - createdAt: Date.now(), - backend: 'xctest', - }; - sessionStore.publish(sessionName, session); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'longpress', - positionals: ['@e1', '800'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - if (response?.ok) { - expect(response.data?.x).toBe(60); - expect(response.data?.y).toBe(40); - expect(response.data?.durationMs).toBe(800); - expect(response.data?.message).toMatch(/Long pressed @e1/); - } - expect(mockLongPressPoint).toHaveBeenCalledWith( - expect.objectContaining({ point: { x: 60, y: 40 }, durationMs: 800 }), - ); - expect(sessionStore.get(sessionName)?.actions[0]?.command).toBe('longpress'); -}); - -// #1783: hover is the pointer-only member of the targeted-touch family. It rides -// the same admission path as press/longpress, and only web admits it. On web -// the session's ref frame is minted WITHOUT rects (snapshot -i does not fetch -// boxes), so `hover @ref` cannot resolve to coordinates: like `click @ref` it -// must take the provider-native route (`hoverRef`, ADR 0011 native-ref path). -// This test therefore stores a rect-less web frame and a scoped provider — the -// production shape — and asserts no coordinate dispatch happens. -test('hover @ref on web dispatches through the provider hoverRef route, not coordinates', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'hover-ref'; - const session = makeSession(sessionName); - session.device = WEB_DESKTOP_DEVICE; - session.snapshot = { - nodes: attachRefs([{ index: 0, role: 'link', label: 'Second message', enabled: true }]), - createdAt: Date.now(), - backend: 'web', - }; - sessionStore.publish(sessionName, session); - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'hover', - positionals: ['@e1'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toMatchObject({ - ok: true, - data: { ref: 'e1', gesture: 'hover', message: expect.stringMatching(/Hovered @e1/) }, - }); - expect(mockHoverRef).toHaveBeenCalledWith(expect.objectContaining({ ref: '@e1' })); - expect(sessionStore.get(sessionName)?.actions[0]?.command).toBe('hover'); -}); - -test('hover selector on web resolves the target and dispatches coordinate hover', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'hover-selector'; - const session = makeSession(sessionName); - session.device = WEB_DESKTOP_DEVICE; - sessionStore.publish(sessionName, session); - mockCaptureSnapshotForSession.mockResolvedValue({ - nodes: attachRefs([ - { - index: 0, - role: 'link', - label: 'Second message', - rect: { x: 10, y: 20, width: 100, height: 40 }, - enabled: true, - hittable: true, - }, - ]), - backend: 'web', - producer: 'agent-browser', - }); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'hover', - positionals: ['label="Second message"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toMatchObject({ - ok: true, - data: { x: 60, y: 40, gesture: 'hover', message: expect.stringMatching(/Hovered label/) }, - }); - expect(mockHoverPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 60, y: 40 } })); -}); - -test('hover is refused by capability on touch platforms before any dispatch', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'hover-ios'; - sessionStore.publish(sessionName, makeSession(sessionName)); - - const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'hover', - positionals: ['100', '200'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toMatchObject({ - ok: false, - error: { - code: 'UNSUPPORTED_OPERATION', - message: 'hover is not supported on this device', - hint: 'hover raises pointer hover state and is available on web targets only. On touch platforms use longpress for hold gestures.', - }, - }); -}); From 41e540586a09bd93e2ab28ef74efcf03cd266014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 16:22:28 +0200 Subject: [PATCH 11/13] test: assert typed refusal details, count captures, and make the touch fake's drag and ref routes follow production Co-Authored-By: Claude Opus 5.5 --- .../support/touch-command-surface-fake.ts | 39 ++++++++++++------ ...ion-touch-android-readiness-spawns.test.ts | 2 +- .../internal/interaction-touch-fill.test.ts | 4 ++ .../selector-runtime-is-reads.test.ts | 1 + .../interactions-preresolved-target.test.ts | 3 ++ .../interactions-ref-frame-evidence.test.ts | 40 +++++++++++-------- .../runtime/selector-is-visibility.test.ts | 3 ++ .../selector-read-disambiguation.test.ts | 1 + 8 files changed, 63 insertions(+), 30 deletions(-) diff --git a/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts index a652e26df2..2b4c551abd 100644 --- a/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts +++ b/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts @@ -18,6 +18,10 @@ import type { } from '@agent-device/contracts/interaction'; import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; import { attachResolvedInteractionTarget } from '@agent-device/host-kit/interaction-outcome'; +import { + buildDragGesturePlan, + singlePointerPlanEndpoints, +} from '@agent-device/contracts/gesture-plan'; import { AppError } from '@agent-device/kernel/errors'; import type { Point, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; import { extractReadableText } from '@agent-device/capture-kit/text-surface'; @@ -124,7 +128,7 @@ function touchInteractions( target: InteractionTarget, options: CommandOptions & { button?: unknown; [key: string]: unknown } = {}, ) => { - const resolved = await resolveTarget(config, target, options); + const resolved = await resolveTarget(config, target, options, undefined, 'tapTarget'); const backendResult = await dispatch(resolved, () => dispatchTap(config, resolved, options)); return await finish(config, resolved, options, { backendResult }); }; @@ -134,7 +138,7 @@ function touchInteractions( press, gesture: async (options) => await dragOrInert(config, inert, options), fill: async (target, text, options = {}) => { - const resolved = await resolveTarget(config, target, options); + const resolved = await resolveTarget(config, target, options, undefined, 'fillTarget'); const backendResult = await dispatch(resolved, () => dispatchFill(config, resolved, text, options), ); @@ -155,7 +159,7 @@ function touchInteractions( }); }, hover: async (target, options = {}) => { - const resolved = await resolveTarget(config, target, options); + const resolved = await resolveTarget(config, target, options, undefined, 'hoverTarget'); const backendResult = await dispatch(resolved, () => dispatchHover(config, resolved, options), ); @@ -190,14 +194,26 @@ async function dragOrInert( ), ), ); - const from = requirePointOf(source!); - const to = requirePointOf(destination!); - const durationMs = - (gesture.sourceHoldMs ?? 0) + (gesture.moveMs ?? 0) + (gesture.destinationHoldMs ?? 0); + const viewport = + (await config.backend.resolveGestureViewport?.(context(config, commandOptions))) ?? + source!.preAction?.nodes.find((node) => node.rect)?.rect; + if (!viewport) throw new AppError('COMMAND_FAILED', 'gesture viewport is unavailable'); + const plan = buildDragGesturePlan( + { + from: requirePointOf(source!), + to: requirePointOf(destination!), + sourceHoldMs: gesture.sourceHoldMs, + moveMs: gesture.moveMs, + destinationHoldMs: gesture.destinationHoldMs, + }, + viewport, + ); const backendResult = await backendOperation(config.backend.performGesture?.bind(config.backend))( context(config, commandOptions), - { topology: 'single', intent: 'drag', durationMs } as never, + plan, ); + const { start: from, end: to } = singlePointerPlanEndpoints(plan); + const durationMs = plan.durationMs; const endpoint = (resolved: ResolvedInteractionTarget) => { if (resolved.kind === 'point' || !resolved.node || !resolved.selectorChain) { throw new AppError('COMMAND_FAILED', 'drag endpoint did not resolve to a node'); @@ -239,12 +255,13 @@ async function resolveTarget( target: InteractionTarget, options: CommandOptions, captureOptions: BackendSnapshotOptions = { interactiveOnly: true }, + nativeRefRoute?: 'tapTarget' | 'fillTarget' | 'hoverTarget', ): Promise { if (target.kind === 'point') return { kind: 'point', point: { x: target.x, y: target.y } }; const pre = options.preresolvedTarget; if (pre && target.kind === 'ref') return resolvedFromNode(config, target, pre.node, pre.tree); if (target.kind === 'ref') { - if (hasNativeRefRoute(config.backend)) return { kind: 'ref', target }; + if (nativeRefRoute && config.backend[nativeRefRoute]) return { kind: 'ref', target }; const record = await config.sessions?.get(options.session ?? 'default'); const tree = record?.refFrameSnapshot ?? record?.snapshot; const node = tree?.nodes.find((candidate) => `@${candidate.ref}` === refBody(target.ref)); @@ -283,10 +300,6 @@ function resolvedFromNode( }; } -function hasNativeRefRoute(backend: AgentDeviceBackend): boolean { - return Boolean(backend.tapTarget ?? backend.hoverTarget ?? backend.fillTarget); -} - function refBody(ref: string): string { return ref.startsWith('@') ? ref : `@${ref}`; } diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts index acf7a0c82f..ab006e0055 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-android-readiness-spawns.test.ts @@ -12,7 +12,7 @@ import { runWithAndroidDialogReadinessCheck } from './interaction-touch-android- const NORMAL_FOCUS_DUMP = ' mCurrentFocus=Window{a1b2c3 u0 com.agentdevice.tester/com.agentdevice.tester.MainActivity}'; -test('a run of taps on a clear screen costs one window dump per tap', async () => { +test('a run of taps on a clear screen costs two window dumps per tap', async () => { const session = makeAndroidSession('readiness-spawn-budget'); const runAdb = vi.fn(async (_device, args) => ({ exitCode: 0, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts index 6b3565ce17..eb7d1b9080 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.test.ts @@ -140,6 +140,8 @@ test('fill with a pinned stale ref rejects; pinned current is clean', async () = expect(stale?.ok).toBe(false); if (stale && !stale.ok) { expect(stale.error.code).toBe('COMMAND_FAILED'); + expect(stale.error.details?.reason).toBe('ref_generation_mismatch'); + expect(stale.error.details?.dispatched).toBe('no'); expect(stale.error.details?.hint).toBe( "Ref @e1 was minted from snapshot s2 but the session's ref frame is now s3 — re-run snapshot -i.", ); @@ -189,7 +191,9 @@ test("ADR 0014 blocker-2: a mutating find's internal fill from an expired frame expect(userRef?.ok).toBe(false); if (userRef && !userRef.ok) { expect(userRef.error.details?.reason).toBe('ref_frame_expired'); + expect(userRef.error.details?.dispatched).toBe('no'); } + expect(mockFillPoint).not.toHaveBeenCalled(); // The mutating find's internal dispatch (the exact request find.ts hands the // leaf) bypasses admission AND attaches no stale-ref warning. diff --git a/packages/daemon/src/execution/selector-runtime-is-reads.test.ts b/packages/daemon/src/execution/selector-runtime-is-reads.test.ts index b2c28697ea..96c90db1d7 100644 --- a/packages/daemon/src/execution/selector-runtime-is-reads.test.ts +++ b/packages/daemon/src/execution/selector-runtime-is-reads.test.ts @@ -264,6 +264,7 @@ test('is reports Android permission dialog blocker when app content assertion fa blockedBy: 'android_foreground_surface', expectedPackage: 'com.example.demo', foregroundPackage: 'com.google.android.permissioncontroller', + permissionDialog: true, }); } }); diff --git a/src/commands/interaction/runtime/interactions-preresolved-target.test.ts b/src/commands/interaction/runtime/interactions-preresolved-target.test.ts index cda78e6147..9bd4677dca 100644 --- a/src/commands/interaction/runtime/interactions-preresolved-target.test.ts +++ b/src/commands/interaction/runtime/interactions-preresolved-target.test.ts @@ -154,6 +154,9 @@ test('runtime click keeps the shared guards running on a pre-resolved node', asy assert.ok(error instanceof Error); assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); assert.match(error.message, /covered by another visible element/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_covered'); + assert.equal(details?.dispatched, 'no'); return true; }, ); diff --git a/src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts b/src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts index cc0025e6b5..0ee355f146 100644 --- a/src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts +++ b/src/commands/interaction/runtime/interactions-ref-frame-evidence.test.ts @@ -67,34 +67,41 @@ function assertOffscreenRefusal(error: unknown): boolean { test('runtime press @ref fails closed when stored ref bounds are invalid (ADR 0014)', async () => { const taps: Point[] = []; - const device = createInteractionDevice( - androidTextViewSnapshot('android.widget.TextView', 'My App'), - { - platform: 'android', - tap: async (_context, point) => { - taps.push(point); - }, + let captures = 0; + const snapshot = androidTextViewSnapshot('android.widget.TextView', 'My App'); + const device = createInteractionDevice(snapshot, { + platform: 'android', + captureSnapshot: async () => { + captures += 1; + return { snapshot }; }, - ); + tap: async (_context, point) => { + taps.push(point); + }, + }); await assert.rejects( () => device.interactions.press(ref('@e1'), { session: 'default' }), assertBoundsInvalid, ); assert.deepEqual(taps, []); + assert.equal(captures, 0, 'a ref with invalid bounds is refused without a positional recapture'); }); test('runtime fill @ref fails closed when stored ref bounds are invalid (ADR 0014)', async () => { const fills: Point[] = []; - const device = createInteractionDevice( - androidTextViewSnapshot('android.widget.EditText', 'Email'), - { - platform: 'android', - fill: async (_context, point) => { - fills.push(point); - }, + let captures = 0; + const snapshot = androidTextViewSnapshot('android.widget.EditText', 'Email'); + const device = createInteractionDevice(snapshot, { + platform: 'android', + captureSnapshot: async () => { + captures += 1; + return { snapshot }; }, - ); + fill: async (_context, point) => { + fills.push(point); + }, + }); await assert.rejects( () => @@ -105,6 +112,7 @@ test('runtime fill @ref fails closed when stored ref bounds are invalid (ADR 001 assertBoundsInvalid, ); assert.deepEqual(fills, []); + assert.equal(captures, 0, 'a ref with invalid bounds is refused without a positional recapture'); }); test('runtime press @ref fails fast when the target is off-screen', async () => { diff --git a/src/commands/interaction/runtime/selector-is-visibility.test.ts b/src/commands/interaction/runtime/selector-is-visibility.test.ts index f3405c2d79..2a6dfbe1e7 100644 --- a/src/commands/interaction/runtime/selector-is-visibility.test.ts +++ b/src/commands/interaction/runtime/selector-is-visibility.test.ts @@ -62,6 +62,9 @@ test('runtime is visible fails for nodes outside the current viewport', async () assert.ok(error instanceof Error); assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); assert.match(error.message, /actual=\{"visible":false/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'predicate_failed'); + assert.equal(details?.predicate, 'visible'); return true; }, ); diff --git a/src/commands/interaction/runtime/selector-read-disambiguation.test.ts b/src/commands/interaction/runtime/selector-read-disambiguation.test.ts index 2070b265f8..0b4f38a53e 100644 --- a/src/commands/interaction/runtime/selector-read-disambiguation.test.ts +++ b/src/commands/interaction/runtime/selector-read-disambiguation.test.ts @@ -67,6 +67,7 @@ test('runtime get text on an iOS label selector disambiguates among same-label n assert.equal(result.kind, 'text'); assert.equal(result.text, 'General'); + assert.notEqual(result.node.type, 'Cell'); assert.deepEqual(result.target, { kind: 'selector', selector: 'label="General"' }); assert.equal(captures, 1); }); From f31172531d80a30023d9d9109553b2e76af84bf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 18:00:28 +0200 Subject: [PATCH 12/13] test(daemon): reuse the shared contextFromFlags in the touch runner and say which fake values are fake-supplied Co-Authored-By: Claude Opus 5.5 --- .../__tests__/support/touch-command-surface-fake.ts | 6 ++++++ .../__tests__/support/touch-interaction-runner.ts | 13 +------------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts index 2b4c551abd..fee6559c29 100644 --- a/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts +++ b/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts @@ -40,6 +40,12 @@ import { fakeCreateCommandSurface } from './command-surface-fake.ts'; * observation) names, and a selector is the first node of one fresh capture that matches one of its `||` alternatives * (each a conjunction of `key=value` terms). Promotion, occlusion, off-screen and bounds guards, label recovery, and * `--verify` evidence are command-family behavior and are asserted beside that code. + * + * Values the fake supplies itself rather than reading from a backend: an `is` read always passes + * (`pass: true`, `matches: 1`); reads capture with `READ_CAPTURE_OPTIONS`; `--verify` evidence is a + * placeholder digest (`fake-digest-`). Daemon suites assert how the daemon passes, records and + * discloses these values, never their content; the real predicate, capture and evidence values are + * asserted against the real command surface in `src/commands/interaction/runtime/`. */ /** The runtime configs this surface was assembled with, oldest first. */ diff --git a/packages/daemon/src/__tests__/support/touch-interaction-runner.ts b/packages/daemon/src/__tests__/support/touch-interaction-runner.ts index 902ebd2ea6..70136e479f 100644 --- a/packages/daemon/src/__tests__/support/touch-interaction-runner.ts +++ b/packages/daemon/src/__tests__/support/touch-interaction-runner.ts @@ -1,20 +1,9 @@ -import type { CommandFlags } from '@agent-device/contracts/command'; import { getRuntimeBindings } from '../../execution/__tests__/interaction-get-runtime-fixture.ts'; import { handleInteractionCommands } from '../../execution/interaction/index.ts'; +import { contextFromFlags } from '../../execution/interaction/internal/__tests__/interaction-touch-fixtures.ts'; import type { SessionStore } from '../../core/session-store.ts'; import { touchCreateCommandSurface } from './touch-command-surface-fake.ts'; -/** The request-flag projection the interaction handler's runtime context is built from. */ -const contextFromFlags = (flags: CommandFlags | undefined) => ({ - count: flags?.count, - intervalMs: flags?.intervalMs, - delayMs: flags?.delayMs, - holdMs: flags?.holdMs, - jitterPx: flags?.jitterPx, - doubleTap: flags?.doubleTap, - clickButton: flags?.clickButton, -}); - /** * One interaction request through `handleInteractionCommands` on the touch command surface fake * and the shared runtime-binding fixture, so suites state only the request they are pinning. From 75f4e4ca332306ce2ae90b4927aff2f239f7435a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 18:02:46 +0200 Subject: [PATCH 13/13] docs(test): say that only selector reads capture in the touch fake Co-Authored-By: Claude Opus 5.5 --- .../daemon/src/__tests__/support/touch-command-surface-fake.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts index fee6559c29..1791d8760a 100644 --- a/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts +++ b/packages/daemon/src/__tests__/support/touch-command-surface-fake.ts @@ -42,7 +42,8 @@ import { fakeCreateCommandSurface } from './command-surface-fake.ts'; * `--verify` evidence are command-family behavior and are asserted beside that code. * * Values the fake supplies itself rather than reading from a backend: an `is` read always passes - * (`pass: true`, `matches: 1`); reads capture with `READ_CAPTURE_OPTIONS`; `--verify` evidence is a + * (`pass: true`, `matches: 1`); selector reads (`get`, `is`) capture once with + * `READ_CAPTURE_OPTIONS`, while a point or `@ref` read captures nothing; `--verify` evidence is a * placeholder digest (`fake-digest-`). Daemon suites assert how the daemon passes, records and * discloses these values, never their content; the real predicate, capture and evidence values are * asserted against the real command surface in `src/commands/interaction/runtime/`.