Skip to content

Commit c4fae2b

Browse files
committed
fix(desktop): confirm the click window and the highlighted menu item before input
- A coordinate click must land in the observed window, or in a sheet, popover, panel or menu above it; another document window of the same app is refused - The menu-completion Return is sent only when the pressed item reports itself highlighted
1 parent 4774742 commit c4fae2b

1 file changed

Lines changed: 19 additions & 9 deletions

File tree

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,9 @@ func completeMenuPress(_ item: AXUIElement, pid: pid_t) async throws {
6767
guard let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]],
6868
entries.contains(where: { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid && ($0[kCGWindowLayer as String] as? NSNumber)?.intValue == menuLevel }) else { return }
6969
try checkCancellation()
70+
// Return runs whatever is highlighted, so it is sent only when this exact item is.
7071
_ = AXUIElementSetAttributeValue(item, kAXSelectedAttribute as CFString, kCFBooleanTrue)
72+
guard attribute(item, kAXSelectedAttribute) as? Bool == true else { return }
7173
try checkCancellation()
7274
try postKey(pid: pid, code: 36)
7375
}
@@ -404,20 +406,28 @@ func mapPoint(frame: CGRect, x: Double, y: Double) throws -> CGPoint {
404406
func requireForegroundTarget(pid: pid_t, windowID: String, point: CGPoint, frame: CGRect) throws {
405407
guard NSWorkspace.shared.frontmostApplication?.processIdentifier == pid else { throw ComputerError("foreground_required", "Coordinate mouse input requires the target app in front. Call activate_app, then observe again.") }
406408
guard try windowRect(windowID, pid: pid) == frame else { throw ComputerError("stale_window", "Window moved during input; observe it again.") }
407-
guard let owner = pointOwner(point), owner == pid else { throw ComputerError("window_occluded", "The target point is covered by another app's window; bring the intended window forward and observe again.") }
409+
guard let hit = elementAtPoint(point) else { throw ComputerError("window_occluded", "Cannot confirm which window receives this point; observe again before retrying.") }
410+
var owner: pid_t = 0
411+
guard AXUIElementGetPid(hit, &owner) == .success, owner == pid else { throw ComputerError("window_occluded", "The target point is covered by another app's window; bring the intended window forward and observe again.") }
412+
// The app's own sheets, popovers, panels and menus may sit above the observed window; another
413+
// of its document windows may not.
414+
if let rawWindow = attribute(hit, kAXWindowAttribute), CFGetTypeID(rawWindow) == AXUIElementGetTypeID() {
415+
let window = unsafeBitCast(rawWindow, to: AXUIElement.self)
416+
guard elementRect(window) == frame || attribute(window, kAXSubroleAttribute) as? String != kAXStandardWindowSubrole else {
417+
throw ComputerError("window_occluded", "Another of this app's windows covers the target point; observe that window and target it instead.")
418+
}
419+
}
408420
}
409-
/// The process that receives a click at a global point. Notification Center and the Dock keep
421+
/// The element that receives a click at a global point. Notification Center and the Dock keep
410422
/// full-screen click-through windows above every app, so the topmost window in the window list is not
411-
/// the click target; Accessibility hit-testing follows the window server's routing. The app's own
412-
/// sheets, popovers and menus above the observed window resolve to the app itself. Without an answer
413-
/// the point's owner is unknown, and the caller refuses the input.
414-
func pointOwner(_ point: CGPoint) -> pid_t? {
423+
/// the click target; Accessibility hit-testing follows the window server's routing. Without an answer
424+
/// the target is unknown, and the caller refuses the input.
425+
func elementAtPoint(_ point: CGPoint) -> AXUIElement? {
415426
let system = AXUIElementCreateSystemWide()
416427
AXUIElementSetMessagingTimeout(system, 0.25)
417428
var hit: AXUIElement?
418-
var owner: pid_t = 0
419-
guard AXUIElementCopyElementAtPosition(system, Float(point.x), Float(point.y), &hit) == .success, let hit, AXUIElementGetPid(hit, &owner) == .success else { return nil }
420-
return owner
429+
guard AXUIElementCopyElementAtPosition(system, Float(point.x), Float(point.y), &hit) == .success else { return nil }
430+
return hit
421431
}
422432
final class CoordinateDispatch {
423433
private(set) var hasDispatched = false

0 commit comments

Comments
 (0)