From 1b138cab6ca2d3ce84e3e90dd25c6681c78ddfcb Mon Sep 17 00:00:00 2001 From: iamnbutler Date: Thu, 8 Oct 2026 08:25:35 -0400 Subject: [PATCH 1/2] feat(desktop): consume native desktop tools from githubnext/desktop-tools --- apps/desktop/native/Package.resolved | 60 - apps/desktop/native/Package.swift | 42 - apps/desktop/native/patches/README.md | 182 --- .../native/patches/peekaboo-click.patch | 197 --- .../peekaboo-clipboard-files-write.patch | 298 ----- .../patches/peekaboo-clipboard-files.patch | 252 ---- .../peekaboo-clipboard-image-write.patch | 282 ----- .../patches/peekaboo-clipboard-image.patch | 376 ------ .../patches/peekaboo-clipboard-text.patch | 310 ----- .../native/patches/peekaboo-close.patch | 156 --- .../native/patches/peekaboo-insert.patch | 899 -------------- .../native/patches/peekaboo-launch.patch | 27 - .../native/patches/peekaboo-menu.patch | 454 ------- .../native/patches/peekaboo-open.patch | 19 - .../native/patches/peekaboo-point-focus.patch | 255 ---- .../patches/peekaboo-pointer-window.patch | 35 - .../native/patches/peekaboo-quit.patch | 41 - .../native/patches/peekaboo-stale-click.patch | 23 - .../{sources/desktop => }/project.swift | 15 +- .../native/sources/client/actions.swift | 409 ------- .../native/sources/client/client.swift | 368 ------ .../native/sources/client/clipboard.swift | 133 -- .../native/sources/client/launch.swift | 163 --- .../native/sources/client/management.swift | 280 ----- apps/desktop/native/sources/client/menu.swift | 108 -- .../desktop/native/sources/client/menus.swift | 119 -- .../native/sources/desktop/desktop.swift | 174 --- .../native/sources/signing/signing.swift | 49 - apps/desktop/package.json | 1 + apps/desktop/scripts/helper.ts | 144 +-- apps/desktop/src/native.ts | 49 +- apps/host/package.json | 1 + apps/host/src/desktop.ts | 1069 +---------------- bun.lock | 8 +- docs/architecture.md | 4 +- docs/desktop-tools.md | 5 +- docs/updates.md | 17 +- package.json | 1 + packages/channel/package.json | 1 + packages/channel/src/desktop.ts | 213 +--- 40 files changed, 125 insertions(+), 7114 deletions(-) delete mode 100644 apps/desktop/native/Package.resolved delete mode 100644 apps/desktop/native/Package.swift delete mode 100644 apps/desktop/native/patches/README.md delete mode 100644 apps/desktop/native/patches/peekaboo-click.patch delete mode 100644 apps/desktop/native/patches/peekaboo-clipboard-files-write.patch delete mode 100644 apps/desktop/native/patches/peekaboo-clipboard-files.patch delete mode 100644 apps/desktop/native/patches/peekaboo-clipboard-image-write.patch delete mode 100644 apps/desktop/native/patches/peekaboo-clipboard-image.patch delete mode 100644 apps/desktop/native/patches/peekaboo-clipboard-text.patch delete mode 100644 apps/desktop/native/patches/peekaboo-close.patch delete mode 100644 apps/desktop/native/patches/peekaboo-insert.patch delete mode 100644 apps/desktop/native/patches/peekaboo-launch.patch delete mode 100644 apps/desktop/native/patches/peekaboo-menu.patch delete mode 100644 apps/desktop/native/patches/peekaboo-open.patch delete mode 100644 apps/desktop/native/patches/peekaboo-point-focus.patch delete mode 100644 apps/desktop/native/patches/peekaboo-pointer-window.patch delete mode 100644 apps/desktop/native/patches/peekaboo-quit.patch delete mode 100644 apps/desktop/native/patches/peekaboo-stale-click.patch rename apps/desktop/native/{sources/desktop => }/project.swift (91%) delete mode 100644 apps/desktop/native/sources/client/actions.swift delete mode 100644 apps/desktop/native/sources/client/client.swift delete mode 100644 apps/desktop/native/sources/client/clipboard.swift delete mode 100644 apps/desktop/native/sources/client/launch.swift delete mode 100644 apps/desktop/native/sources/client/management.swift delete mode 100644 apps/desktop/native/sources/client/menu.swift delete mode 100644 apps/desktop/native/sources/client/menus.swift delete mode 100644 apps/desktop/native/sources/desktop/desktop.swift delete mode 100644 apps/desktop/native/sources/signing/signing.swift diff --git a/apps/desktop/native/Package.resolved b/apps/desktop/native/Package.resolved deleted file mode 100644 index 65b5cbd..0000000 --- a/apps/desktop/native/Package.resolved +++ /dev/null @@ -1,60 +0,0 @@ -{ - "originHash" : "1d80ce2cea94149fb16a0e67353938267a00d1ed31116b46fed1dd4bb7235fe9", - "pins" : [ - { - "identity" : "axorcist", - "kind" : "remoteSourceControl", - "location" : "https://github.com/openclaw/AXorcist.git", - "state" : { - "revision" : "31732870e1fd758247418db9239256d2da497146", - "version" : "0.1.11" - } - }, - { - "identity" : "commander", - "kind" : "remoteSourceControl", - "location" : "https://github.com/steipete/Commander.git", - "state" : { - "revision" : "bd219c4ee9032fee3e009856f81fcc6ec09a85f4", - "version" : "0.2.4" - } - }, - { - "identity" : "peekaboo", - "kind" : "remoteSourceControl", - "location" : "https://github.com/openclaw/Peekaboo.git", - "state" : { - "revision" : "4d43dc9d80cd2aa3787a27f54b76d692db1dcf8f", - "version" : "4.8.0" - } - }, - { - "identity" : "swift-algorithms", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-algorithms", - "state" : { - "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", - "version" : "1.2.1" - } - }, - { - "identity" : "swift-log", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-log.git", - "state" : { - "revision" : "9c6fb14227f55d8f711ce3847dc2f419fb0ecacb", - "version" : "1.15.1" - } - }, - { - "identity" : "swift-numerics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-numerics.git", - "state" : { - "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", - "version" : "1.1.1" - } - } - ], - "version" : 3 -} diff --git a/apps/desktop/native/Package.swift b/apps/desktop/native/Package.swift deleted file mode 100644 index ae9d845..0000000 --- a/apps/desktop/native/Package.swift +++ /dev/null @@ -1,42 +0,0 @@ -// swift-tools-version: 6.2 - -import PackageDescription - -let package = Package( - name: "AceNative", - platforms: [.macOS(.v15)], - products: [ - .library(name: "AceDesktop", type: .dynamic, targets: ["AceDesktop"]), - .executable(name: "ace-desktop-client", targets: ["AceDesktopClient"]), - ], - dependencies: [ - .package(url: "https://github.com/openclaw/Peekaboo.git", exact: "4.8.0"), - ], - targets: [ - .target( - name: "AceSigning", - path: "sources/signing", - linkerSettings: [.linkedFramework("Security")] - ), - .target( - name: "AceDesktop", - dependencies: [ - "AceSigning", - .product(name: "PeekabooBridge", package: "Peekaboo"), - .product(name: "PeekabooAutomationKit", package: "Peekaboo"), - ], - path: "sources/desktop" - ), - .executableTarget( - name: "AceDesktopClient", - dependencies: [ - "AceSigning", - .product(name: "PeekabooBridge", package: "Peekaboo"), - .product(name: "PeekabooAutomationKit", package: "Peekaboo"), - .product(name: "PeekabooFoundation", package: "Peekaboo"), - ], - path: "sources/client" - ), - ], - swiftLanguageModes: [.v6] -) diff --git a/apps/desktop/native/patches/README.md b/apps/desktop/native/patches/README.md deleted file mode 100644 index cdfba4c..0000000 --- a/apps/desktop/native/patches/README.md +++ /dev/null @@ -1,182 +0,0 @@ -# Native dependency patches - -These patches apply to Peekaboo 4.8.0, revision -`4d43dc9d80cd2aa3787a27f54b76d692db1dcf8f`. - -`peekaboo-click.patch` addresses -[self-targeted Accessibility clicks blocking Ace's native bridge](https://github.com/githubnext/ace2/issues/61). - -The patch moves semantic `AXPress` off MainActor so Ace can service its own Accessibility request. -It captures result metadata before dispatch and avoids querying removed non-tab controls after a -press. Exact target validation remains in place. The operation keeps its coordinator lane until -the native call returns, including after client cancellation; no timeout is treated as completed -input. -Ambiguous native press failures retain an indeterminate outcome, preventing Peekaboo from -falling back to another click after input may already have been delivered. - -The patch also addresses [editable WebKit controls accepting a press without keyboard focus](https://github.com/githubnext/ace2/issues/106). -A single element click prefers the existing verified focus write for writable `AXTextField` and -`AXTextArea` controls when Accessibility value delivery is allowed. Other controls retain their -normal press behavior. The focus write runs off MainActor while the operation keeps its lane; -the original exact target checks and focus readback remain in force. An ambiguous write failure -retains an indeterminate outcome instead of allowing another input route. Point-click occlusion -and background paste behavior are separate parts of that issue. - -`peekaboo-pointer-window.patch` addresses -[WebKit controls being reported as occluded](https://github.com/githubnext/ace2/issues/106). -Positional click validation and pointer receiver identification use Peekaboo's existing -containing-window resolver, including the native `AXWindow` link when a leaf has no direct window -ID. A different or unresolved window remains refused; process, generation, bounds, and target -checks are unchanged. It adds no coordinate-routing or input fallback. - -`peekaboo-insert.patch` addresses -[literal newlines submitting web composers](https://github.com/githubnext/ace2/issues/76). -Unicode keyboard events are still keyboard events: WebKit can treat a newline as Return. -The patch exposes one GUI-owned literal insertion operation using a temporary plain-text paste. -It reuses the native clipboard transaction gate, preserves bounded prior contents privately, -and owns the snapshot lease and native process mutation lane through delivery, verification, and cleanup. -Exact process, window, focused receiver, text, and UTF-16 selection establish the intended edit. -Both the original and intended text must fit the complete 65,536-unit verification limit. -An exact receiver in the active frontmost app uses a direct targeted Cmd+V chord. That route -revalidates the retained editor, selection, active app, and process generation before every input -unit. Other targets use Peekaboo's target-only window preparation, including its guarded blank -native title-bar click. The route is fixed before input and never switches after partial delivery. -Preparation and delivery form one native outcome; partial preparation remains uncertain input and -cannot authorize a retry. If the paste key was never posted, clipboard restoration is safe even -when preparation or modifier input was emitted. Typed refusal causes -retain the native guard diagnostic without exposing clipboard contents or the compared text. -An observed meaningful edit authorizes generation-checked restoration; uncertain consumption -leaves the replacement or preserves newer contents, never restoring private prior contents -while a paste may still be pending. It has no typing fallback or delayed restore journal. -The existing clipboard gate durably reserves the target process generation before the paste key. -Unresolved delivery blocks later automated clipboard writes until a live read confirms the intended -edit or that exact process generation ends. Only reservation metadata survives a GUI restart; -no clipboard contents, hashes, or deferred restoration are persisted. - -`peekaboo-quit.patch` preserves accepted but unfinished normal quit in -[native computer use](https://github.com/githubnext/ace2/issues/8). -A normal quit can leave an application waiting for an unsaved-work decision. That is one -dispatched operation with unverified completion and unsafe retry, not a safely repeatable no-op. -The quit-specific result validator permits this canonical outcome with `false` termination so -the existing bridge returns the boolean and its signed process receipt. Other false/success -contradictions stay errors. Force-quit behavior, target revalidation, and mutation lanes are unchanged. - -`peekaboo-close.patch` makes background window close a single request for -[window management](https://github.com/githubnext/ace2/issues/73). It selects a supported `AXClose` -or exact close-button `AXPress` from read-only evidence before input, then rechecks the original -window ID, process generation and bounds immediately before that one action. No accepted or -ambiguous native result falls through to another close route. An accepted close whose window -remains open is unverified with unsafe retry, including when unsaved work opens a dialog. -Cancellation stops admission before input; once admitted, the existing mutation lane remains -held until the detached AX call actually settles. Confirmed disappearance retains the existing -bridge postcondition checks. Foreground fallback behavior is unchanged and is not exposed by Ace. - -`peekaboo-clipboard-text.patch` adds explicit plain-text clipboard reads and persistent writes for -[clipboard access](https://github.com/githubnext/ace2/issues/72). It applies after the insertion patch -and reuses its GUI clipboard service and reservation gate. Reads require silent clipboard access, -one complete item, a stable generation, and a complete JSON result of at most 24,000 bytes; ordinary -alternate representations do not prevent reading its plain text. Reads do not enter or release the -gate. Writes accept at most 8,192 UTF-16 units, enter the existing gate, and retain native mutation -outcomes without returning clipboard contents. They never paste, snapshot prior contents, restore, -or add a separate journal. Unresolved paste ownership refuses writes across channels and GUI restarts. - -`peekaboo-clipboard-image.patch` adds the read-only image format for the same -[clipboard access](https://github.com/githubnext/ace2/issues/72). The GUI requires silent read access -and one stable clipboard item, then validates a complete PNG/JPEG/TIFF representation within -10 MiB, one frame, and 64 million pixels. ImageIO renders an oriented preview off MainActor: -PNG at bounded sizes first, then explicitly white-composited JPEG if needed. Previews fit -1,600 pixels per side and 900,000 bytes and carry separate source/conversion metadata. The -generation is checked after rendering too. The operation has no mutation lane, gate admission, -temporary file, clipboard backup, or paste behavior; it reuses the existing read-only Bridge -and channel image result. - -`peekaboo-launch.patch` classifies synchronous selector preparation failures before application -launch as refused without dispatch. A missing LaunchServices registration or invalid launch request -therefore receives the existing signed targetless refusal receipt, instead of an indeterminate -mutation receipt. The catch surrounds only preparation; native opening, activation, readiness, -mutation lane ownership, and all errors after dispatch keep their existing semantics. - -`peekaboo-clipboard-image-write.patch` adds persistent image writes for #72. The host reads one -bounded regular file into memory; neither the native client nor GUI opens the source path. -ImageIO validates a complete PNG/JPEG/TIFF frame (10 MiB, 64 million pixels) off MainActor before -the existing clipboard reservation gate admits the write. The GUI writes the original bytes and -UTI through `setActionResult`, preserving native accepted/uncertain outcomes. The response contains -source metadata, never image bytes or prior clipboard contents. Cancellation before admission -does not write; cancellation or response loss after admission cannot imply undo or safe replay. -No temporary paste, file reference, backup, new lock, or journal is added. - -`peekaboo-clipboard-files.patch` adds the read-only file-reference format for #72. The GUI -uses the same silent permission and stable-generation checks, with at most 32 advertised local -file URL items and a complete 24 KB JSON result. It preserves URLs, order, and decoded paths; -no file existence/content checks, path canonicalization, or remote access occurs. Unsupported -legacy filename lists, promises, mixed item kinds, invalid/nonlocal URLs and unreadable data -refuse without truncation when visible to Ace. macOS may filter references before delivery, so -absence does not establish what the originating app published. The typed read-only Bridge has -no mutation gate, backup or journal. - -`peekaboo-point-focus.patch` gives single left coordinate clicks the same verified editable-field -focus behavior as element clicks for [native computer use](https://github.com/githubnext/ace2/issues/106). -The raw Accessibility hit test runs on the existing bounded read lane so a self-targeted app can -answer it. Only that read leaves MainActor; generation, exact window bounds, and containing-window -checks still run before input. Settable text fields choose focus before AXPress, with the existing -detached focus write and verified native focus receipt. Original identity and bounds are checked -again immediately before mutation. Positional AXPress waits for its actual return and retains the -operation lane through cancellation; an uncertain return remains unsafe to retry. No input runs in -the detached read, no failed action falls back to another route, and no caret position is promised. - -`peekaboo-clipboard-files-write.patch` adds persistent file-reference writes for -[clipboard access](https://github.com/githubnext/ace2/issues/72). The host checks metadata for -existing regular files, directories, or symbolic links without reading contents or resolving links. -The GUI validates 1–32 literal paths and prepares a complete file URL list within the reader's -24 KB bound. A focused multi-item service method publishes one `public.file-url` representation -per item in one `writeObjects` call, with generation checks around publication. It reuses the -existing mutation result owner and outer pending-paste gate; it adds no backup, temporary paste, -promise, or new authority. Silent same-generation item readback can confirm publication; -unavailable verification retains the existing dispatched/unverified result. There is no fallback -or restore after publication, and no claim about a receiver's later handling of the references. - -`peekaboo-open.patch` distinguishes accepted document/URL delivery from verified application launch -for [native computer use](https://github.com/githubnext/ace2/issues/8). The existing launch service -returns `dispatched_unverified` with `delivery_accepted` when its request contains items to open. -The original native delivery, accepted unit count, global lane ownership and signed process target -are retained. Readiness and activation do not prove that the app loaded the item. Launches without -items and all refusal/uncertainty paths are unchanged; the existing bridge contract already permits -this accepted outcome. - -`peekaboo-menu.patch` adds literal external-application menu commands for -[native computer use](https://github.com/githubnext/ace2/issues/8). A distinct typed Bridge -operation carries the exact process generation and title array, avoiding the existing String -API's splitting, fuzzy normalization and intermediate presses. Fresh bounded raw AX traversal -requires a unique path and supported enabled leaf before one AXPress; lazy missing paths refuse. -The whole native operation runs off MainActor while retaining the existing process write lane -until actual return, with cancellation checked before dispatch. Per-element AX messaging timeouts -do not race or abandon native work. `cannotComplete` and other ambiguous delivery failures remain -one attempted, unsafe unknown action; successful delivery does not establish command completion. -Signed application receipts and canonical outcome validation remain mandatory. Commands targeting -the native Ace process itself refuse before input; its menu inventory remains available. No -intermediate presses, separate activation request, fallback, automatic screenshot, or command retry -is added. The target app or macOS may still bring the app forward in response. - -`peekaboo-stale-click.patch` reports exact-window clicks whose window already changed as refused for -[moved-window clicks](https://github.com/githubnext/ace2/issues/155). Click preparation runs before -any strategy route. It now checks the captured window identity and bounds there and throws a typed -pre-dispatch `target_unavailable` refusal with the `SNAPSHOT_STALE` code, a refresh hint, and the -captured exact-window target. Previously the first such check ran inside a route and its untyped -error reached the bridge's generic mapping, which conservatively reported possible dispatch. -Later route checks are unchanged: a route can follow an earlier Accessibility attempt that may have -dispatched, so a window change detected there still reports uncertain input. Long press keeps its -existing exact-window exemption, and process-generation checks are unchanged. - -The desktop build resolves only `Package.resolved` versions, checks the pin and checkout revision, -and assembles the ordered patch stack in a private Git index. A build compares the checkout -with each complete ordered prefix, since later patches can change earlier patch contexts. -Only the missing suffix is applied, then verified against the complete expected source before -compiling Swift. Keep new patches at the end so an existing complete stack remains a valid prefix. -The real dependency index is unchanged. Drift fails the build instead of producing unexpected -source. Patch files are excluded from formatting. - -When updating Peekaboo, review the upstream fixes and these patches together. Update the revision -guard and patches deliberately, or remove each patch when the dependency includes its fix. -Revalidate real button clicks, changed-focus refusal, literal multiline insertion, clipboard -restoration, partial input, and interruption before shipping -the update. diff --git a/apps/desktop/native/patches/peekaboo-click.patch b/apps/desktop/native/patches/peekaboo-click.patch deleted file mode 100644 index ed555d6..0000000 --- a/apps/desktop/native/patches/peekaboo-click.patch +++ /dev/null @@ -1,197 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift -@@ -171,8 +171,16 @@ - allowAccessibilityValueFallback: Bool, - beforeMutation: @MainActor () throws -> Void = {}) async throws -> UIInputExecutionResult.Action - { -+ // WebKit text controls may accept AXPress without moving keyboard focus. -+ if allowAccessibilityValueFallback, -+ let role = element.role, -+ role == "AXTextField" || role == "AXTextArea", -+ element.isFocusedSettable -+ { -+ return try await self.focusForClick(element, beforeMutation: beforeMutation) -+ } - do { -- return try self.performAction(AXActionNames.kAXPressAction, on: element, beforeMutation: beforeMutation) -+ return try await self.performPressAction(on: element, beforeMutation: beforeMutation) - } catch let error as ActionInputError - where error == .unsupported(.actionUnsupported) && - allowAccessibilityValueFallback && -@@ -204,7 +212,52 @@ - } - } - -- /// Issues `AXShowMenu` without waiting on the menu's tracking runloop. -+ /// Keeps the operation lane owned until native completion while allowing a self-targeted app to service AX. -+ private func performPressAction( -+ on element: any AutomationElementRepresenting, -+ beforeMutation: @MainActor () throws -> Void) async throws -> UIInputExecutionResult.Action -+ { -+ guard let axElement = element.underlyingAXElement else { -+ return try self.performAction(AXActionNames.kAXPressAction, on: element, beforeMutation: beforeMutation) -+ } -+ try Task.checkCancellation() -+ guard element.supportsAction(AXActionNames.kAXPressAction) else { -+ throw ActionInputError.unsupported(.actionUnsupported) -+ } -+ // The action may remove this element; result metadata must not query it after dispatch. -+ let anchorPoint = element.anchorPoint -+ let elementRole = element.role -+ try beforeMutation() -+ try Task.checkCancellation() -+ do { -+ _ = try await DetachedAXActionRunner.perform( -+ action: AXActionNames.kAXPressAction, -+ on: axElement, -+ gracePeriod: nil) -+ return UIInputExecutionResult.Action( -+ outcome: .dispatchedUnverified( -+ delivery: Self.accessibilityActionDelivery, -+ evidence: .deliveryAccepted, -+ unitCount: .one), -+ actionName: AXActionNames.kAXPressAction, -+ anchorPoint: anchorPoint, -+ elementRole: elementRole) -+ } catch { -+ let classified = Self.classify(error) -+ if Self.nativeMutationFailureMayHaveDispatched(classified) { -+ throw DesktopActionFailure.indeterminate( -+ delivery: Self.accessibilityActionDelivery, -+ evidence: .completionUnknown, -+ unitCount: .one, -+ message: "Accessibility press returned without reliable completion evidence.", -+ hint: "Observe the exact target before retrying this click.", -+ causeDescription: error.localizedDescription) -+ } -+ throw classified -+ } -+ } -+ -+ /// Issues `AXShowMenu` without waiting on the menu's tracking runloop. - /// - /// A successful `AXShowMenu` opens an NSMenu whose tracking loop is a nested runloop on the - /// target app's main thread, so `AXUIElementPerformAction` does not return until the menu is -@@ -566,9 +619,23 @@ - let observationTarget = try await self.observationTarget(element) - try Self.validateBeforeMutation(beforeMutation) - do { -- try element.setAutomationFocused(true) -+ if let native = element.underlyingAXElement { -+ try await DetachedAXActionRunner.focus(on: native) -+ } else { -+ try element.setAutomationFocused(true) -+ } - } catch { -- throw Self.classify(error) -+ let classified = Self.classify(error) -+ if Self.nativeMutationFailureMayHaveDispatched(classified) { -+ throw DesktopActionFailure.indeterminate( -+ delivery: Self.accessibilityValueDelivery, -+ evidence: .completionUnknown, -+ unitCount: .one, -+ message: "Accessibility focus returned without reliable completion evidence.", -+ hint: "Observe the exact field before deciding whether to retry focus.", -+ causeDescription: error.localizedDescription) -+ } -+ throw classified - } - var confirmedIdentity: FocusedElementIdentity? - guard await self.observeMutation( -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/DetachedAXActionRunner.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/DetachedAXActionRunner.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/DetachedAXActionRunner.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/DetachedAXActionRunner.swift -@@ -24,10 +24,11 @@ - /// modal loop should still be reported as delivered. - static let pressGracePeriod: TimeInterval = 2.0 - -+ /// A nil grace period retains the awaiting operation and its lane until the native call actually returns. - static func perform( - action actionName: String, - on element: AXUIElement, -- gracePeriod: TimeInterval) async throws -> DesktopActionOutcome -+ gracePeriod: TimeInterval?) async throws -> DesktopActionOutcome - { - let box = UncheckedAXElementBox(element: element) - return try await self.run(gracePeriod: gracePeriod) { -@@ -35,13 +36,26 @@ - } - } - -+ /// Retains the caller's lane until the focus write returns, including after cancellation. -+ static func focus(on element: AXUIElement) async throws { -+ let box = UncheckedAXElementBox(element: element) -+ _ = try await self.run( -+ gracePeriod: nil, -+ delivery: .init(mechanism: .accessibilityValue, mode: .background)) -+ { -+ AXUIElementSetAttributeValue(box.element, kAXFocusedAttribute as CFString, kCFBooleanTrue) -+ } -+ } -+ - /// Runs `operation` on a detached thread, resolving with operation-still-running evidence if - /// it does not return within `gracePeriod`. Factored over a closure so tests can exercise the - /// race without a live accessibility element. - static func run( -- gracePeriod: TimeInterval, -+ gracePeriod: TimeInterval?, -+ delivery: DesktopActionOutcome.Delivery? = nil, - operation: @escaping @Sendable () -> AXError) async throws -> DesktopActionOutcome - { -+ let delivery = delivery ?? self.delivery - let gate = OneShotOutcomeGate() - let result = await withCheckedContinuation { (continuation: CheckedContinuation< - Result, -@@ -52,16 +66,18 @@ - let result = operation() - if result == .success { - gate.resume(with: .success(.dispatchedUnverified( -- delivery: self.delivery, -+ delivery: delivery, - evidence: .deliveryAccepted))) - } else { - gate.resume(with: .failure(AccessibilitySystemError(result))) - } - } -- DispatchQueue.global().asyncAfter(deadline: .now() + gracePeriod) { -- gate.resume(with: .success(.dispatchedUnverified( -- delivery: self.delivery, -- evidence: .operationStillRunning))) -+ if let gracePeriod { -+ DispatchQueue.global().asyncAfter(deadline: .now() + gracePeriod) { -+ gate.resume(with: .success(.dispatchedUnverified( -+ delivery: delivery, -+ evidence: .operationStillRunning))) -+ } - } - } - return try result.get() -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift -@@ -193,8 +193,9 @@ - validateProcessIdentity: validatesProcessIdentity) - switch clickType { - case .single: -- let valueBefore = element.intAttribute(AXAttributeNames.kAXValueAttribute) -- let originalIdentity = element.focusedElementIdentity -+ let valueBefore = element.intAttribute(AXAttributeNames.kAXValueAttribute) -+ let requiresTabVerification = element.subrole == "AXTabButton" && valueBefore == 0 -+ let originalIdentity = element.focusedElementIdentity - let originalRole = element.role - let result = try await self.actionInputDriver.tryClick( - element: element, -@@ -228,7 +229,11 @@ - } - let tabPressDidNotSelect: Bool - do { -- tabPressDidNotSelect = try await self.tabSelectionVerifier(element, valueBefore) -+ if requiresTabVerification { -+ tabPressDidNotSelect = try await self.tabSelectionVerifier(element, valueBefore) -+ } else { -+ tabPressDidNotSelect = false -+ } - } catch { - guard result.outcome.dispatchState.mutationDispatched else { throw error } - throw Self.unconfirmedTabPressFailure(result, cause: error) diff --git a/apps/desktop/native/patches/peekaboo-clipboard-files-write.patch b/apps/desktop/native/patches/peekaboo-clipboard-files-write.patch deleted file mode 100644 index b96299c..0000000 --- a/apps/desktop/native/patches/peekaboo-clipboard-files-write.patch +++ /dev/null @@ -1,298 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardFilesInput.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardFilesInput.swift -new file mode 100644 -index 0000000..89e574c ---- /dev/null -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardFilesInput.swift -@@ -0,0 +1,36 @@ -+import Foundation -+import PeekabooFoundation -+ -+public struct ClipboardFilesInput: Sendable { -+ public let files: [ClipboardFilesContents.File] -+ -+ public static func validate(_ paths: [String]) throws -> Self { -+ try Task.checkCancellation() -+ guard (1...32).contains(paths.count) else { -+ throw Self.refusal("Clipboard file writes require 1–32 absolute paths.") -+ } -+ let allowed = CharacterSet(charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~/") -+ let files = try paths.map { path in -+ guard path.hasPrefix("/"), path.utf16.count <= 4096, !path.utf8.contains(0), -+ let encoded = path.addingPercentEncoding(withAllowedCharacters: allowed) else { -+ throw Self.refusal("Clipboard file writes require absolute paths of at most 4096 UTF-16 units without NUL.") -+ } -+ // Encode literal paths without resolving links, standardizing components, or consulting the filesystem. -+ let raw = "file://" + encoded -+ guard let url = URL(string: raw, encodingInvalidCharacters: false), url.isFileURL, -+ url.host == nil || url.host == "", !(url as NSURL).isFileReferenceURL() else { -+ throw Self.refusal("Clipboard file writes require ordinary local file references.") -+ } -+ return ClipboardFilesContents.File(url: raw, path: path) -+ } -+ let largestRead = ClipboardFilesContents(present: true, files: files, change_count: Int.max) -+ guard try JSONEncoder().encode(largestRead).count <= 24_000 else { -+ throw Self.refusal("Clipboard file references exceed the complete 24 KB read-result limit.") -+ } -+ return .init(files: files) -+ } -+ -+ private static func refusal(_ message: String) -> DesktopActionFailure { -+ .preDispatchRefusal(reason: .invalidRequest, message: message) -+ } -+} -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -index aba68c5..6a40bc6 100644 ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -@@ -631,6 +631,42 @@ ClipboardReadAccessProviding { - verify: { _ in self.matches(request: request) }) - } - -+ public func setFilesActionResult(_ input: ClipboardFilesInput) throws -> DesktopActionResult { -+ var generation: Int? -+ return try ClipboardMutationResultOwner.perform( -+ operation: "Clipboard files write", -+ mutation: { didDispatch in -+ let items = try input.files.map { file in -+ let item = NSPasteboardItem() -+ guard item.setData(Data(file.url.utf8), forType: .fileURL) else { -+ throw ClipboardServiceError.writeFailed("Unable to prepare a file reference") -+ } -+ return item -+ } -+ try Task.checkCancellation() -+ let claimed = self.pasteboard.clearContents() -+ didDispatch = true -+ generation = claimed -+ try self.requireTemporaryOwnership(claimed) -+ guard self.pasteboard.writeObjects(items) else { -+ throw ClipboardServiceError.writeFailed("Unable to publish all file reference items") -+ } -+ try self.requireTemporaryOwnership(claimed) -+ return input.files.count -+ }, -+ verify: { _ in -+ guard let generation, self.readAccessStatus().readAdmitted, -+ self.pasteboard.changeCount == generation, -+ let items = self.pasteboard.pasteboardItems, -+ items.count == input.files.count else { return false } -+ for (item, file) in zip(items, input.files) { -+ guard item.types.contains(.fileURL), -+ item.data(forType: .fileURL) == Data(file.url.utf8) else { return false } -+ } -+ return self.pasteboard.changeCount == generation -+ }) -+ } -+ - private func set( - _ request: ClipboardWriteRequest, - didDispatch: inout Bool, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardFilesWrite.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardFilesWrite.swift -new file mode 100644 -index 0000000..7c1d708 ---- /dev/null -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardFilesWrite.swift -@@ -0,0 +1,51 @@ -+import Foundation -+import PeekabooAutomationKit -+import PeekabooFoundation -+ -+public struct PeekabooBridgeClipboardFilesWriteResult: Codable, Sendable { -+ public let outcome: String -+ public let native_outcome: DesktopActionOutcome -+ public let clipboard_changed: Bool -+ public let file_count: Int -+ public let error: LiteralInsertError? -+} -+ -+extension PeekabooBridgeClient { -+ public func clipboardFilesWrite(_ paths: [String]) async throws -> PeekabooBridgeClipboardFilesWriteResult { -+ switch try await self.send(.clipboardFilesWrite(paths)) { -+ case let .clipboardFilesWrite(result): return result -+ case let .error(error): throw error -+ default: throw PeekabooBridgeErrorEnvelope(code: .internalError, message: "Unexpected clipboard files write response") -+ } -+ } -+} -+ -+@MainActor -+extension PeekabooBridgeServer { -+ func handleClipboardFilesWrite(_ request: PeekabooBridgeRequest) async throws -> PeekabooBridgeHandledResponse { -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ guard case let .clipboardFilesWrite(paths) = request else { throw Self.invalidRequest(for: request) } -+ let files = try ClipboardFilesInput.validate(paths) -+ return try await ClipboardPasteTransactionGate.withExclusiveTransaction { -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ let result: PeekabooBridgeClipboardFilesWriteResult -+ do { -+ let written = try self.literalInsertClipboard.setFilesActionResult(files) -+ let native = try ClipboardMutationResultSemantics.requireSuccessfulOutcome( -+ written.outcome, operation: "Clipboard files write") -+ result = .init(outcome: "completed", native_outcome: native, -+ clipboard_changed: true, file_count: files.files.count, error: nil) -+ } catch let failure as DesktopActionFailure { -+ let changed = failure.outcome.dispatchState.mutationDispatched -+ result = .init(outcome: changed ? "unknown" : "refused", native_outcome: failure.outcome, -+ clipboard_changed: changed, file_count: files.files.count, -+ error: .init(code: failure.standardErrorCode?.rawValue ?? "CLIPBOARD_WRITE_FAILED", -+ message: failure.message, hint: failure.hint)) -+ } -+ // The signed global receipt binds result metadata, not a receiver's later use of the references. -+ return .init(response: .clipboardFilesWrite(result), mutation: .init( -+ outcome: .dispatchedUnverified(delivery: ClipboardMutationResultSemantics.delivery, -+ evidence: .deliveryAccepted, unitCount: .one), target: .global)) -+ } -+ } -+} -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -index fae31e8..88b5fe0 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -@@ -42,6 +42,7 @@ public enum PeekabooBridgeOperation: String, Codable, Sendable, CaseIterable, Ha - case clipboardFilesRead - case clipboardTextWrite - case clipboardImageWrite -+ case clipboardFilesWrite - case permissionsStatus - case requestPostEventPermission - case daemonStatus -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -index 0e8c83c..70552fb 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -@@ -29,7 +29,7 @@ extension PeekabooBridgeOperation { - - /// Operations enabled by default for remote helper hosts. - public static let remoteDefaultAllowlist: Set = [ -- .literalInsert, .clipboardTextRead, .clipboardImageRead, .clipboardFilesRead, .clipboardTextWrite, .clipboardImageWrite, -+ .literalInsert, .clipboardTextRead, .clipboardImageRead, .clipboardFilesRead, .clipboardTextWrite, .clipboardImageWrite, .clipboardFilesWrite, - .permissionsStatus, - .requestPostEventPermission, - .daemonStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -index 706717c..61df86e 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -@@ -66,6 +66,9 @@ extension PeekabooBridgeOperationResultSemantics { - descriptor(completion: .readOnly, targetPolicy: .notApplicable, responseFamilies: [.clipboardImage]) - case .clipboardTextRead: - descriptor(completion: .readOnly, targetPolicy: .notApplicable, responseFamilies: [.clipboardText]) -+ case .clipboardFilesWrite: -+ descriptor(ownership: .service, completion: .dispatchedUnverified(clipboardForeground), -+ targetPolicy: .global, responseFamilies: [.clipboardFilesWrite]) - case .clipboardImageWrite: - descriptor(ownership: .service, completion: .dispatchedUnverified(clipboardForeground), - targetPolicy: .global, responseFamilies: [.clipboardImageWrite]) -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -index fe7f936..9a95670 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -@@ -8,6 +8,7 @@ extension PeekabooBridgeOperationResultSemantics.ResponseFamily { - (.clipboardText, .clipboardText), - (.clipboardTextWrite, .clipboardTextWrite), - (.clipboardImageWrite, .clipboardImageWrite), -+ (.clipboardFilesWrite, .clipboardFilesWrite), - (.literalInsert, .literalInsert), - (.agentExecutionTrace, .agentExecutionTrace), - (.application, .application), -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -index 2ca63b6..438c981 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -@@ -123,7 +123,7 @@ extension PeekabooBridgeResponse { - bounds: termination.receipt.windowBounds)] - case let .error(envelope): - return envelope.actionTargetReceipt.map { [DesktopTargetEvidenceAdapter.evidence(receipt: $0)] } ?? [] -- case .clipboardFiles, .clipboardImage, .clipboardText, .clipboardTextWrite, .clipboardImageWrite, .literalInsert, -+ case .clipboardFiles, .clipboardImage, .clipboardText, .clipboardTextWrite, .clipboardImageWrite, .clipboardFilesWrite, .literalInsert, - .operationSessionRollover, - .handshake, - .permissionsStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -index 031dffa..f3cf34f 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -@@ -60,6 +60,7 @@ enum PeekabooBridgeOperationResultSemantics { - case clipboardFiles - case clipboardTextWrite - case clipboardImageWrite -+ case clipboardFilesWrite - case agentExecutionTrace - case application - case applicationMutationInventory -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -index 664dc3b..62a7070 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -@@ -446,7 +446,7 @@ extension PeekabooBridgeOperationResultSemantics { - .performAction(target: payload.target, actionName: payload.actionName) - case let .selectText(payload): - .selectText(target: payload.target, request: payload.request) -- case .clipboardFilesRead, .clipboardImageRead, .clipboardTextRead, .clipboardTextWrite, .clipboardImageWrite, .literalInsert, -+ case .clipboardFilesRead, .clipboardImageRead, .clipboardTextRead, .clipboardTextWrite, .clipboardImageWrite, .clipboardFilesWrite, .literalInsert, - .attestedOperation, - .projectedAction, - .handshake, -@@ -597,7 +597,7 @@ extension PeekabooBridgeOperationResultSemantics { - .suspectedNoop, - ] - switch request.operation { -- case .clipboardTextWrite, .clipboardImageWrite, .literalInsert, .agentExecutionTrace: -+ case .clipboardTextWrite, .clipboardImageWrite, .clipboardFilesWrite, .literalInsert, .agentExecutionTrace: - return [.dispatchedUnverified] - case .requestPostEventPermission, .browserExecute, .swipe, .drag, .moveMouse, - .menuCommand, .clickMenuItem, .clickMenuItemByName, .clickMenuExtra, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -index 312bdbe..1b6daa5 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -@@ -247,7 +247,7 @@ extension PeekabooBridgeRequest { - /// never own the desktop lane or mutation watermark: its child re-enters this same Bridge and - /// each nested tool call owns its own exact-target lane and signed receipt. - var bypassesOuterDesktopMutationLane: Bool { -- [.agentExecutionTrace, .literalInsert, .clipboardTextWrite, .clipboardImageWrite].contains(self.unwrappedOperationRequest.operation) -+ [.agentExecutionTrace, .literalInsert, .clipboardTextWrite, .clipboardImageWrite, .clipboardFilesWrite].contains(self.unwrappedOperationRequest.operation) - } - } - -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -index d7aaedc..d835672 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -@@ -12,6 +12,7 @@ public enum PeekabooBridgeRequest: Codable, Sendable { - case clipboardFilesRead - case clipboardTextWrite(String) - case clipboardImageWrite(Data) -+ case clipboardFilesWrite([String]) - case permissionsStatus - case requestPostEventPermission - case daemonStatus -@@ -146,6 +147,7 @@ extension PeekabooBridgeRequest { - case .clipboardFilesRead: .clipboardFilesRead - case .clipboardTextWrite: .clipboardTextWrite - case .clipboardImageWrite: .clipboardImageWrite -+ case .clipboardFilesWrite: .clipboardFilesWrite - case .permissionsStatus: .permissionsStatus - case .requestPostEventPermission: .requestPostEventPermission - case .daemonStatus: .daemonStatus -@@ -280,6 +282,7 @@ public enum PeekabooBridgeResponse: Codable, Sendable { - case clipboardFiles(ClipboardFilesContents) - case clipboardTextWrite(PeekabooBridgeClipboardTextWriteResult) - case clipboardImageWrite(PeekabooBridgeClipboardImageWriteResult) -+ case clipboardFilesWrite(PeekabooBridgeClipboardFilesWriteResult) - case permissionsStatus(PermissionsStatus) - case daemonStatus(PeekabooDaemonStatus) - case agentExecutionTrace(PeekabooBridgeAgentExecutionTraceResponse) -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -index 08c92e7..e88c5d1 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -@@ -14,6 +14,8 @@ extension PeekabooBridgeServer { - switch request.operation { - case .clipboardFilesRead: - return try await self.handleClipboardFilesRead() -+ case .clipboardFilesWrite: -+ return try await self.handleClipboardFilesWrite(request) - case .clipboardImageWrite: - return try await self.handleClipboardImageWrite(request) - case .clipboardImageRead: diff --git a/apps/desktop/native/patches/peekaboo-clipboard-files.patch b/apps/desktop/native/patches/peekaboo-clipboard-files.patch deleted file mode 100644 index e7a0b0d..0000000 --- a/apps/desktop/native/patches/peekaboo-clipboard-files.patch +++ /dev/null @@ -1,252 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardFilesContents.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardFilesContents.swift -new file mode 100644 -index 000000000..9ef2670a2 ---- /dev/null -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardFilesContents.swift -@@ -0,0 +1,13 @@ -+import Foundation -+ -+/// Advertised file references only; paths are decoded without accessing the filesystem. -+public struct ClipboardFilesContents: Codable, Sendable { -+ public let present: Bool -+ public let files: [File]? -+ public let change_count: Int -+ -+ public struct File: Codable, Sendable { -+ public let url: String -+ public let path: String -+ } -+} -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -index 93366f36b..aba68c503 100644 ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -@@ -383,6 +383,60 @@ ClipboardReadAccessProviding { - return result - } - -+ public func readFiles() throws -> ClipboardFilesContents { -+ try Self.requireSilentReadAccess(self.readAccessStatus()) -+ let generation = self.pasteboard.changeCount -+ guard let items = self.pasteboard.pasteboardItems, items.count <= 32 else { -+ throw PeekabooError.invalidInput("File reads require a complete clipboard with at most 32 items.") -+ } -+ var files: [ClipboardFilesContents.File] = [] -+ for item in items { -+ guard !item.types.isEmpty, -+ !item.types.contains(where: { $0.rawValue.localizedCaseInsensitiveContains("promise") }) else { -+ throw PeekabooError.invalidInput("Clipboard file promises or unavailable representations cannot be read as file references.") -+ } -+ guard item.types.contains(.fileURL) else { -+ guard !item.types.contains(where: { $0.rawValue == "NSFilenamesPboardType" }) else { -+ throw PeekabooError.invalidInput("Legacy filename lists are unsupported; copy advertised file URL items.") -+ } -+ continue -+ } -+ guard let data = item.data(forType: .fileURL), data.count <= 24_000, -+ let raw = String(data: data, encoding: .utf8) else { -+ throw PeekabooError.invalidInput("The advertised file URL could not be read completely within 24 KB.") -+ } -+ guard let url = URL(string: raw, encodingInvalidCharacters: false), url.isFileURL, -+ !(url as NSURL).isFileReferenceURL(), -+ url.host == nil || url.host == "" || url.host?.lowercased() == "localhost", -+ url.user == nil, url.password == nil, url.port == nil, -+ url.query == nil, url.fragment == nil else { -+ throw PeekabooError.invalidInput("Clipboard files require ordinary absolute local file URLs.") -+ } -+ let path = url.path(percentEncoded: false) -+ guard path.hasPrefix("/"), !path.utf8.contains(0) else { -+ throw PeekabooError.invalidInput("Clipboard files require ordinary absolute local file URLs.") -+ } -+ files.append(.init(url: raw, path: path)) -+ } -+ guard files.isEmpty || files.count == items.count else { -+ throw PeekabooError.invalidInput("Clipboard file reads do not omit mixed non-file items.") -+ } -+ if items.isEmpty { -+ guard let types = self.pasteboard.types, types.isEmpty else { -+ throw PeekabooError.invalidInput("The clipboard item inventory is unavailable.") -+ } -+ } -+ guard self.pasteboard.changeCount == generation else { -+ throw PeekabooError.invalidInput("The clipboard changed during the read; request a fresh read.") -+ } -+ let result = ClipboardFilesContents(present: !files.isEmpty, files: files.isEmpty ? nil : files, -+ change_count: generation) -+ guard try JSONEncoder().encode(result).count <= 24_000 else { -+ throw PeekabooError.invalidInput("Clipboard file references exceed the complete 24 KB result limit.") -+ } -+ return result -+ } -+ - private static func plainTextReadRefusal(_ message: String) -> PeekabooError { - .invalidInput(message) - } -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardFiles.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardFiles.swift -new file mode 100644 -index 000000000..77ceb76c0 ---- /dev/null -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardFiles.swift -@@ -0,0 +1,22 @@ -+import Foundation -+import PeekabooAutomationKit -+ -+extension PeekabooBridgeClient { -+ public func clipboardFilesRead() async throws -> ClipboardFilesContents { -+ switch try await self.send(.clipboardFilesRead) { -+ case let .clipboardFiles(result): return result -+ case let .error(error): throw error -+ default: throw PeekabooBridgeErrorEnvelope(code: .internalError, message: "Unexpected clipboard files response") -+ } -+ } -+} -+ -+@MainActor -+extension PeekabooBridgeServer { -+ func handleClipboardFilesRead() async throws -> PeekabooBridgeHandledResponse { -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ let result = try self.literalInsertClipboard.readFiles() -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ return .init(response: .clipboardFiles(result)) -+ } -+} -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -index c8ba8ea71..4cf85d94e 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -@@ -39,6 +39,7 @@ public enum PeekabooBridgeOperation: String, Codable, Sendable, CaseIterable, Ha - case literalInsert - case clipboardTextRead - case clipboardImageRead -+ case clipboardFilesRead - case clipboardTextWrite - case permissionsStatus - case requestPostEventPermission -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -index a2b3bacc8..fb5c29152 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -@@ -29,7 +29,7 @@ extension PeekabooBridgeOperation { - - /// Operations enabled by default for remote helper hosts. - public static let remoteDefaultAllowlist: Set = [ -- .literalInsert, .clipboardTextRead, .clipboardImageRead, .clipboardTextWrite, -+ .literalInsert, .clipboardTextRead, .clipboardImageRead, .clipboardFilesRead, .clipboardTextWrite, - .permissionsStatus, - .requestPostEventPermission, - .daemonStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -index d3c4c20f1..caaa3dd08 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -@@ -60,6 +60,8 @@ extension PeekabooBridgeOperationResultSemantics { - } - - return switch operation { -+ case .clipboardFilesRead: -+ descriptor(completion: .readOnly, targetPolicy: .notApplicable, responseFamilies: [.clipboardFiles]) - case .clipboardImageRead: - descriptor(completion: .readOnly, targetPolicy: .notApplicable, responseFamilies: [.clipboardImage]) - case .clipboardTextRead: -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -index 58fdd785a..62d831d65 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -@@ -3,7 +3,8 @@ import Foundation - extension PeekabooBridgeOperationResultSemantics.ResponseFamily { - func matches(_ response: PeekabooBridgeResponse) -> Bool { - switch (self, response) { -- case (.clipboardImage, .clipboardImage), -+ case (.clipboardFiles, .clipboardFiles), -+ (.clipboardImage, .clipboardImage), - (.clipboardText, .clipboardText), - (.clipboardTextWrite, .clipboardTextWrite), - (.literalInsert, .literalInsert), -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -index 3bf1af6bf..461d375c8 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -@@ -123,7 +123,7 @@ extension PeekabooBridgeResponse { - bounds: termination.receipt.windowBounds)] - case let .error(envelope): - return envelope.actionTargetReceipt.map { [DesktopTargetEvidenceAdapter.evidence(receipt: $0)] } ?? [] -- case .clipboardImage, .clipboardText, .clipboardTextWrite, .literalInsert, -+ case .clipboardFiles, .clipboardImage, .clipboardText, .clipboardTextWrite, .literalInsert, - .operationSessionRollover, - .handshake, - .permissionsStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -index 77ac27a06..c201fe53d 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -@@ -57,6 +57,7 @@ enum PeekabooBridgeOperationResultSemantics { - case literalInsert - case clipboardText - case clipboardImage -+ case clipboardFiles - case clipboardTextWrite - case agentExecutionTrace - case application -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -index b14ec6edf..e053a6ab6 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -@@ -446,7 +446,7 @@ extension PeekabooBridgeOperationResultSemantics { - .performAction(target: payload.target, actionName: payload.actionName) - case let .selectText(payload): - .selectText(target: payload.target, request: payload.request) -- case .clipboardImageRead, .clipboardTextRead, .clipboardTextWrite, .literalInsert, -+ case .clipboardFilesRead, .clipboardImageRead, .clipboardTextRead, .clipboardTextWrite, .literalInsert, - .attestedOperation, - .projectedAction, - .handshake, -@@ -649,7 +649,7 @@ extension PeekabooBridgeOperationResultSemantics { - return [.dispatchedUnverified] - case .browserConnect: - return [.confirmedNoChange, .dispatchedUnverified] -- case .clipboardImageRead, .clipboardTextRead, .permissionsStatus, .observeProcessGeneration, .certificationProducerAttestation, -+ case .clipboardFilesRead, .clipboardImageRead, .clipboardTextRead, .permissionsStatus, .observeProcessGeneration, .certificationProducerAttestation, - .createExactWindowHeldPointerOwner, - .daemonStatus, .daemonStop, .browserStatus, - .browserDisconnect, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -index 65f2fce18..1743b0a98 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -@@ -9,6 +9,7 @@ public enum PeekabooBridgeRequest: Codable, Sendable { - case literalInsert(PeekabooBridgeLiteralInsertRequest) - case clipboardTextRead - case clipboardImageRead -+ case clipboardFilesRead - case clipboardTextWrite(String) - case permissionsStatus - case requestPostEventPermission -@@ -141,6 +142,7 @@ extension PeekabooBridgeRequest { - case .literalInsert: .literalInsert - case .clipboardTextRead: .clipboardTextRead - case .clipboardImageRead: .clipboardImageRead -+ case .clipboardFilesRead: .clipboardFilesRead - case .clipboardTextWrite: .clipboardTextWrite - case .permissionsStatus: .permissionsStatus - case .requestPostEventPermission: .requestPostEventPermission -@@ -273,6 +275,7 @@ public enum PeekabooBridgeResponse: Codable, Sendable { - case literalInsert(PeekabooBridgeLiteralInsertResult) - case clipboardText(ClipboardTextContents) - case clipboardImage(ClipboardImageContents) -+ case clipboardFiles(ClipboardFilesContents) - case clipboardTextWrite(PeekabooBridgeClipboardTextWriteResult) - case permissionsStatus(PermissionsStatus) - case daemonStatus(PeekabooDaemonStatus) -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -index 5cad2d202..5151c95a8 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -@@ -12,6 +12,8 @@ extension PeekabooBridgeServer { - permissions: PermissionsStatus) async throws -> PeekabooBridgeHandledResponse - { - switch request.operation { -+ case .clipboardFilesRead: -+ return try await self.handleClipboardFilesRead() - case .clipboardImageRead: - return try await self.handleClipboardImageRead() - case .clipboardTextRead, .clipboardTextWrite: diff --git a/apps/desktop/native/patches/peekaboo-clipboard-image-write.patch b/apps/desktop/native/patches/peekaboo-clipboard-image-write.patch deleted file mode 100644 index 5a169bc..0000000 --- a/apps/desktop/native/patches/peekaboo-clipboard-image-write.patch +++ /dev/null @@ -1,282 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardImageInput.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardImageInput.swift -new file mode 100644 -index 0000000..7480846 ---- /dev/null -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardImageInput.swift -@@ -0,0 +1,59 @@ -+import Foundation -+import ImageIO -+import PeekabooFoundation -+import UniformTypeIdentifiers -+ -+public struct ClipboardImageInput: Sendable { -+ public let data: Data -+ public let source: ClipboardImageContents.Source -+ -+ public static func validate(_ data: Data) throws -> Self { -+ try Task.checkCancellation() -+ guard !data.isEmpty, data.count <= 10 * 1024 * 1024 else { -+ throw Self.refusal("Clipboard images must contain at most 10 MiB of complete image data.") -+ } -+ guard let image = CGImageSourceCreateWithData( -+ data as CFData, [kCGImageSourceShouldCache: false] as CFDictionary), -+ let detected = CGImageSourceGetType(image), -+ [UTType.png.identifier, UTType.jpeg.identifier, UTType.tiff.identifier].contains(detected as String), -+ let mime = UTType(detected as String)?.preferredMIMEType, -+ CGImageSourceGetCount(image) == 1, -+ CGImageSourceGetStatus(image) == .statusComplete, -+ CGImageSourceGetStatusAtIndex(image, 0) == .statusComplete, -+ let properties = CGImageSourceCopyPropertiesAtIndex(image, 0, nil) as? [CFString: Any], -+ let width = Self.dimension(properties[kCGImagePropertyPixelWidth]), -+ let height = Self.dimension(properties[kCGImagePropertyPixelHeight]), -+ width <= 64_000_000 / height -+ else { -+ throw Self.refusal("Clipboard images require one complete PNG, JPEG, or TIFF frame of at most 64 million pixels.") -+ } -+ let orientation = (properties[kCGImagePropertyOrientation] as? NSNumber)?.intValue ?? 1 -+ guard (1...8).contains(orientation) else { -+ throw Self.refusal("The image has an unsupported orientation.") -+ } -+ try Task.checkCancellation() -+ // Decode completely before admission; the original bytes, including orientation and alpha, are written. -+ guard let decoded = CGImageSourceCreateImageAtIndex(image, 0, [ -+ kCGImageSourceShouldCacheImmediately: true, -+ ] as CFDictionary), decoded.width == width, decoded.height == height, -+ CGImageSourceGetStatusAtIndex(image, 0) == .statusComplete -+ else { -+ throw Self.refusal("The image could not be decoded completely.") -+ } -+ try Task.checkCancellation() -+ return .init(data: data, source: .init( -+ uti: detected as String, mimeType: mime, bytes: data.count, -+ width: width, height: height, orientation: orientation)) -+ } -+ -+ private static func dimension(_ value: Any?) -> Int? { -+ guard let number = value as? NSNumber else { return nil } -+ let value = number.doubleValue -+ guard value.isFinite, value > 0, value <= 64_000_000, value.rounded() == value else { return nil } -+ return Int(value) -+ } -+ -+ private static func refusal(_ message: String) -> DesktopActionFailure { -+ .preDispatchRefusal(reason: .invalidRequest, message: message) -+ } -+} -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardImageWrite.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardImageWrite.swift -new file mode 100644 -index 0000000..bebfdf5 ---- /dev/null -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardImageWrite.swift -@@ -0,0 +1,59 @@ -+import Foundation -+import PeekabooAutomationKit -+import PeekabooFoundation -+ -+public struct PeekabooBridgeClipboardImageWriteResult: Codable, Sendable { -+ public let outcome: String -+ public let native_outcome: DesktopActionOutcome -+ public let clipboard_changed: Bool -+ public let source: ClipboardImageContents.Source -+ public let error: LiteralInsertError? -+} -+ -+extension PeekabooBridgeClient { -+ public func clipboardImageWrite(_ data: Data) async throws -> PeekabooBridgeClipboardImageWriteResult { -+ switch try await self.send(.clipboardImageWrite(data)) { -+ case let .clipboardImageWrite(result): return result -+ case let .error(error): throw error -+ default: throw PeekabooBridgeErrorEnvelope(code: .internalError, message: "Unexpected clipboard image write response") -+ } -+ } -+} -+ -+@MainActor -+extension PeekabooBridgeServer { -+ func handleClipboardImageWrite(_ request: PeekabooBridgeRequest) async throws -> PeekabooBridgeHandledResponse { -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ guard case let .clipboardImageWrite(data) = request else { throw Self.invalidRequest(for: request) } -+ let decoding = Task.detached(priority: .userInitiated) { try ClipboardImageInput.validate(data) } -+ let image = try await withTaskCancellationHandler { -+ try await decoding.value -+ } onCancel: { -+ decoding.cancel() -+ } -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ return try await ClipboardPasteTransactionGate.withExclusiveTransaction { -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ let result: PeekabooBridgeClipboardImageWriteResult -+ do { -+ let written = try self.literalInsertClipboard.setActionResult(.init(representations: [ -+ .init(utiIdentifier: image.source.uti, data: image.data), -+ ])) -+ let native = try ClipboardMutationResultSemantics.requireSuccessfulOutcome( -+ written.outcome, operation: "Clipboard image write") -+ result = .init(outcome: "completed", native_outcome: native, -+ clipboard_changed: true, source: image.source, error: nil) -+ } catch let failure as DesktopActionFailure { -+ let changed = failure.outcome.dispatchState.mutationDispatched -+ result = .init(outcome: changed ? "unknown" : "refused", native_outcome: failure.outcome, -+ clipboard_changed: changed, source: image.source, -+ error: .init(code: failure.standardErrorCode?.rawValue ?? "CLIPBOARD_WRITE_FAILED", -+ message: failure.message, hint: failure.hint)) -+ } -+ // The existing signed global receipt binds the content-free result, not image bytes. -+ return .init(response: .clipboardImageWrite(result), mutation: .init( -+ outcome: .dispatchedUnverified(delivery: ClipboardMutationResultSemantics.delivery, -+ evidence: .deliveryAccepted, unitCount: .one), target: .global)) -+ } -+ } -+} -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -index 4cf85d9..fae31e8 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -@@ -41,6 +41,7 @@ public enum PeekabooBridgeOperation: String, Codable, Sendable, CaseIterable, Ha - case clipboardImageRead - case clipboardFilesRead - case clipboardTextWrite -+ case clipboardImageWrite - case permissionsStatus - case requestPostEventPermission - case daemonStatus -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -index fb5c291..0e8c83c 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -@@ -29,7 +29,7 @@ extension PeekabooBridgeOperation { - - /// Operations enabled by default for remote helper hosts. - public static let remoteDefaultAllowlist: Set = [ -- .literalInsert, .clipboardTextRead, .clipboardImageRead, .clipboardFilesRead, .clipboardTextWrite, -+ .literalInsert, .clipboardTextRead, .clipboardImageRead, .clipboardFilesRead, .clipboardTextWrite, .clipboardImageWrite, - .permissionsStatus, - .requestPostEventPermission, - .daemonStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -index caaa3dd..706717c 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -@@ -66,6 +66,9 @@ extension PeekabooBridgeOperationResultSemantics { - descriptor(completion: .readOnly, targetPolicy: .notApplicable, responseFamilies: [.clipboardImage]) - case .clipboardTextRead: - descriptor(completion: .readOnly, targetPolicy: .notApplicable, responseFamilies: [.clipboardText]) -+ case .clipboardImageWrite: -+ descriptor(ownership: .service, completion: .dispatchedUnverified(clipboardForeground), -+ targetPolicy: .global, responseFamilies: [.clipboardImageWrite]) - case .clipboardTextWrite: - descriptor(ownership: .service, completion: .dispatchedUnverified(clipboardForeground), - targetPolicy: .global, responseFamilies: [.clipboardTextWrite]) -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -index 62d831d..fe7f936 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -@@ -7,6 +7,7 @@ extension PeekabooBridgeOperationResultSemantics.ResponseFamily { - (.clipboardImage, .clipboardImage), - (.clipboardText, .clipboardText), - (.clipboardTextWrite, .clipboardTextWrite), -+ (.clipboardImageWrite, .clipboardImageWrite), - (.literalInsert, .literalInsert), - (.agentExecutionTrace, .agentExecutionTrace), - (.application, .application), -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -index 461d375..2ca63b6 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -@@ -123,7 +123,7 @@ extension PeekabooBridgeResponse { - bounds: termination.receipt.windowBounds)] - case let .error(envelope): - return envelope.actionTargetReceipt.map { [DesktopTargetEvidenceAdapter.evidence(receipt: $0)] } ?? [] -- case .clipboardFiles, .clipboardImage, .clipboardText, .clipboardTextWrite, .literalInsert, -+ case .clipboardFiles, .clipboardImage, .clipboardText, .clipboardTextWrite, .clipboardImageWrite, .literalInsert, - .operationSessionRollover, - .handshake, - .permissionsStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -index c201fe5..031dffa 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -@@ -59,6 +59,7 @@ enum PeekabooBridgeOperationResultSemantics { - case clipboardImage - case clipboardFiles - case clipboardTextWrite -+ case clipboardImageWrite - case agentExecutionTrace - case application - case applicationMutationInventory -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -index e053a6a..664dc3b 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -@@ -446,7 +446,7 @@ extension PeekabooBridgeOperationResultSemantics { - .performAction(target: payload.target, actionName: payload.actionName) - case let .selectText(payload): - .selectText(target: payload.target, request: payload.request) -- case .clipboardFilesRead, .clipboardImageRead, .clipboardTextRead, .clipboardTextWrite, .literalInsert, -+ case .clipboardFilesRead, .clipboardImageRead, .clipboardTextRead, .clipboardTextWrite, .clipboardImageWrite, .literalInsert, - .attestedOperation, - .projectedAction, - .handshake, -@@ -597,7 +597,7 @@ extension PeekabooBridgeOperationResultSemantics { - .suspectedNoop, - ] - switch request.operation { -- case .clipboardTextWrite, .literalInsert, .agentExecutionTrace: -+ case .clipboardTextWrite, .clipboardImageWrite, .literalInsert, .agentExecutionTrace: - return [.dispatchedUnverified] - case .requestPostEventPermission, .browserExecute, .swipe, .drag, .moveMouse, - .clickMenuItem, .clickMenuItemByName, .clickMenuExtra, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -index 4886efd..312bdbe 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -@@ -247,7 +247,7 @@ extension PeekabooBridgeRequest { - /// never own the desktop lane or mutation watermark: its child re-enters this same Bridge and - /// each nested tool call owns its own exact-target lane and signed receipt. - var bypassesOuterDesktopMutationLane: Bool { -- [.agentExecutionTrace, .literalInsert, .clipboardTextWrite].contains(self.unwrappedOperationRequest.operation) -+ [.agentExecutionTrace, .literalInsert, .clipboardTextWrite, .clipboardImageWrite].contains(self.unwrappedOperationRequest.operation) - } - } - -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -index 1743b0a..d7aaedc 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -@@ -11,6 +11,7 @@ public enum PeekabooBridgeRequest: Codable, Sendable { - case clipboardImageRead - case clipboardFilesRead - case clipboardTextWrite(String) -+ case clipboardImageWrite(Data) - case permissionsStatus - case requestPostEventPermission - case daemonStatus -@@ -144,6 +145,7 @@ extension PeekabooBridgeRequest { - case .clipboardImageRead: .clipboardImageRead - case .clipboardFilesRead: .clipboardFilesRead - case .clipboardTextWrite: .clipboardTextWrite -+ case .clipboardImageWrite: .clipboardImageWrite - case .permissionsStatus: .permissionsStatus - case .requestPostEventPermission: .requestPostEventPermission - case .daemonStatus: .daemonStatus -@@ -277,6 +279,7 @@ public enum PeekabooBridgeResponse: Codable, Sendable { - case clipboardImage(ClipboardImageContents) - case clipboardFiles(ClipboardFilesContents) - case clipboardTextWrite(PeekabooBridgeClipboardTextWriteResult) -+ case clipboardImageWrite(PeekabooBridgeClipboardImageWriteResult) - case permissionsStatus(PermissionsStatus) - case daemonStatus(PeekabooDaemonStatus) - case agentExecutionTrace(PeekabooBridgeAgentExecutionTraceResponse) -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -index 5151c95..08c92e7 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -@@ -14,6 +14,8 @@ extension PeekabooBridgeServer { - switch request.operation { - case .clipboardFilesRead: - return try await self.handleClipboardFilesRead() -+ case .clipboardImageWrite: -+ return try await self.handleClipboardImageWrite(request) - case .clipboardImageRead: - return try await self.handleClipboardImageRead() - case .clipboardTextRead, .clipboardTextWrite: diff --git a/apps/desktop/native/patches/peekaboo-clipboard-image.patch b/apps/desktop/native/patches/peekaboo-clipboard-image.patch deleted file mode 100644 index 81da66c..0000000 --- a/apps/desktop/native/patches/peekaboo-clipboard-image.patch +++ /dev/null @@ -1,376 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardImageContents.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardImageContents.swift -new file mode 100644 -index 000000000..c7ba2d8ca ---- /dev/null -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardImageContents.swift -@@ -0,0 +1,141 @@ -+import CoreGraphics -+import Foundation -+import ImageIO -+import PeekabooFoundation -+import UniformTypeIdentifiers -+ -+public struct ClipboardImageContents: Codable, Sendable { -+ public let present: Bool -+ public let change_count: Int -+ public let source: Source? -+ public let image: Preview? -+ -+ public struct Source: Codable, Sendable { -+ public let uti: String -+ public let mimeType: String -+ public let bytes: Int -+ public let width: Int -+ public let height: Int -+ public let orientation: Int -+ } -+ -+ public struct Preview: Codable, Sendable { -+ public let data: Data -+ public let mimeType: String -+ public let bytes: Int -+ public let width: Int -+ public let height: Int -+ public let resized: Bool -+ public let reencoded: Bool -+ public let background: String? -+ } -+} -+ -+enum ClipboardImageRenderer { -+ private static let maximumSourceBytes = 10 * 1024 * 1024 -+ private static let maximumPixels = 64_000_000 -+ private static let maximumPreviewBytes = 900_000 -+ -+ static func render(_ data: Data, uti: String, generation: Int) throws -> ClipboardImageContents { -+ try Task.checkCancellation() -+ guard !data.isEmpty, data.count <= self.maximumSourceBytes else { -+ throw PeekabooError.invalidInput("Clipboard image data must fit the complete 10 MiB source limit.") -+ } -+ guard let source = CGImageSourceCreateWithData( -+ data as CFData, [kCGImageSourceShouldCache: false] as CFDictionary), -+ CGImageSourceGetCount(source) == 1, -+ CGImageSourceGetStatus(source) == .statusComplete, -+ CGImageSourceGetStatusAtIndex(source, 0) == .statusComplete, -+ let detected = CGImageSourceGetType(source), -+ detected as String == uti, -+ let mimeType = UTType(uti)?.preferredMIMEType, -+ let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any], -+ let width = self.dimension(properties[kCGImagePropertyPixelWidth]), -+ let height = self.dimension(properties[kCGImagePropertyPixelHeight]), -+ width <= self.maximumPixels / height -+ else { -+ throw PeekabooError.invalidInput("Clipboard images require one complete PNG, JPEG, or TIFF frame of at most 64 million pixels.") -+ } -+ let orientation = (properties[kCGImagePropertyOrientation] as? NSNumber)?.intValue ?? 1 -+ guard (1...8).contains(orientation) else { -+ throw PeekabooError.invalidInput("The clipboard image has an unsupported orientation.") -+ } -+ let metadata = ClipboardImageContents.Source( -+ uti: uti, mimeType: mimeType, bytes: data.count, width: width, height: height, -+ orientation: orientation) -+ // Prefer a lossless transparent preview at each bounded size before flattening for JPEG. -+ for format in [UTType.png, UTType.jpeg] { -+ var previousSize = 0 -+ for bound in [1600, 1200, 800] { -+ try Task.checkCancellation() -+ let size = min(bound, max(width, height)) -+ if size == previousSize { continue } -+ previousSize = size -+ guard let thumbnail = CGImageSourceCreateThumbnailAtIndex(source, 0, [ -+ kCGImageSourceCreateThumbnailFromImageAlways: true, -+ kCGImageSourceCreateThumbnailWithTransform: true, -+ kCGImageSourceThumbnailMaxPixelSize: size, -+ kCGImageSourceShouldCacheImmediately: true, -+ ] as CFDictionary), -+ thumbnail.width > 0, thumbnail.height > 0, -+ thumbnail.width <= 1600, thumbnail.height <= 1600, -+ CGImageSourceGetStatusAtIndex(source, 0) == .statusComplete -+ else { -+ throw PeekabooError.invalidInput("The clipboard image could not be decoded completely.") -+ } -+ let jpeg = format == .jpeg -+ let image = try jpeg ? self.onWhite(thumbnail) : thumbnail -+ let encoded = try self.encode(image, format: format) -+ if encoded.count > self.maximumPreviewBytes { continue } -+ return ClipboardImageContents( -+ present: true, change_count: generation, source: metadata, -+ image: .init(data: encoded, mimeType: jpeg ? "image/jpeg" : "image/png", -+ bytes: encoded.count, width: image.width, height: image.height, -+ resized: max(width, height) > max(image.width, image.height), -+ reencoded: true, background: jpeg ? "white" : nil)) -+ } -+ } -+ throw PeekabooError.invalidInput("The clipboard image preview exceeds the complete 900 KB result limit.") -+ } -+ -+ private static func dimension(_ value: Any?) -> Int? { -+ guard let number = value as? NSNumber else { return nil } -+ let value = number.doubleValue -+ guard value.isFinite, value > 0, value <= Double(self.maximumPixels), value.rounded() == value else { -+ return nil -+ } -+ return Int(value) -+ } -+ -+ private static func onWhite(_ image: CGImage) throws -> CGImage { -+ guard let color = CGColorSpace(name: CGColorSpace.sRGB), -+ let context = CGContext(data: nil, width: image.width, height: image.height, -+ bitsPerComponent: 8, bytesPerRow: 0, space: color, -+ bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { -+ throw PeekabooError.invalidInput("The clipboard image preview could not be rendered.") -+ } -+ let bounds = CGRect(x: 0, y: 0, width: image.width, height: image.height) -+ context.setFillColor(CGColor(gray: 1, alpha: 1)) -+ context.fill(bounds) -+ context.draw(image, in: bounds) -+ guard let result = context.makeImage() else { -+ throw PeekabooError.invalidInput("The clipboard image preview could not be rendered.") -+ } -+ return result -+ } -+ -+ private static func encode(_ image: CGImage, format: UTType) throws -> Data { -+ let data = NSMutableData() -+ guard let destination = CGImageDestinationCreateWithData(data, format.identifier as CFString, 1, nil) else { -+ throw PeekabooError.invalidInput("The clipboard image preview could not be encoded.") -+ } -+ let options = format == .jpeg -+ ? [kCGImageDestinationLossyCompressionQuality: 0.7] as CFDictionary -+ : nil -+ CGImageDestinationAddImage(destination, image, options) -+ guard CGImageDestinationFinalize(destination) else { -+ throw PeekabooError.invalidInput("The clipboard image preview could not be encoded.") -+ } -+ return data as Data -+ } -+} -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -index 117a940f6..93366f36b 100644 ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -@@ -333,6 +333,56 @@ ClipboardReadAccessProviding { - return result - } - -+ public func readImage() async throws -> ClipboardImageContents { -+ try Self.requireSilentReadAccess(self.readAccessStatus()) -+ let generation = self.pasteboard.changeCount -+ guard let items = self.pasteboard.pasteboardItems, items.count <= 1 else { -+ throw PeekabooError.invalidInput("Image reads require a complete clipboard with at most one item.") -+ } -+ var imageData: Data? -+ var imageType: String? -+ if let item = items.first { -+ guard !item.types.isEmpty, -+ !item.types.contains(where: { $0.rawValue.localizedCaseInsensitiveContains("promise") }) else { -+ throw PeekabooError.invalidInput("Clipboard file promises or unavailable representations cannot be read as an image.") -+ } -+ let supported = [UTType.png.identifier, UTType.jpeg.identifier, UTType.tiff.identifier] -+ if let type = supported.first(where: { item.types.contains(.init($0)) }) { -+ guard let data = item.data(forType: .init(type)) else { -+ throw PeekabooError.invalidInput("The advertised image representation could not be read completely.") -+ } -+ imageData = data -+ imageType = type -+ } else if item.types.contains(where: { UTType($0.rawValue)?.conforms(to: .image) == true }) { -+ throw PeekabooError.invalidInput("Clipboard image reads support PNG, JPEG, or TIFF.") -+ } -+ } else { -+ guard let types = self.pasteboard.types, types.isEmpty else { -+ throw PeekabooError.invalidInput("The clipboard item inventory is unavailable.") -+ } -+ } -+ guard self.pasteboard.changeCount == generation else { -+ throw PeekabooError.invalidInput("The clipboard changed during the read; request a fresh read.") -+ } -+ guard let imageData, let imageType else { -+ return ClipboardImageContents(present: false, change_count: generation, source: nil, image: nil) -+ } -+ // Image decoding stays off the GUI actor; only its immutable copy leaves the pasteboard owner. -+ let rendering = Task.detached(priority: .userInitiated) { -+ try ClipboardImageRenderer.render(imageData, uti: imageType, generation: generation) -+ } -+ let result = try await withTaskCancellationHandler { -+ try await rendering.value -+ } onCancel: { -+ rendering.cancel() -+ } -+ try Task.checkCancellation() -+ guard self.pasteboard.changeCount == generation else { -+ throw PeekabooError.invalidInput("The clipboard changed during the read; request a fresh read.") -+ } -+ return result -+ } -+ - private static func plainTextReadRefusal(_ message: String) -> PeekabooError { - .invalidInput(message) - } -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardImage.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardImage.swift -new file mode 100644 -index 000000000..ce84c9045 ---- /dev/null -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardImage.swift -@@ -0,0 +1,22 @@ -+import Foundation -+import PeekabooAutomationKit -+ -+extension PeekabooBridgeClient { -+ public func clipboardImageRead() async throws -> ClipboardImageContents { -+ switch try await self.send(.clipboardImageRead) { -+ case let .clipboardImage(result): return result -+ case let .error(error): throw error -+ default: throw PeekabooBridgeErrorEnvelope(code: .internalError, message: "Unexpected clipboard image response") -+ } -+ } -+} -+ -+@MainActor -+extension PeekabooBridgeServer { -+ func handleClipboardImageRead() async throws -> PeekabooBridgeHandledResponse { -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ let result = try await self.literalInsertClipboard.readImage() -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ return .init(response: .clipboardImage(result)) -+ } -+} -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -index b82ffbfa0..c8ba8ea71 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -@@ -38,6 +38,7 @@ public enum PeekabooBridgeOperation: String, Codable, Sendable, CaseIterable, Ha - // Core - case literalInsert - case clipboardTextRead -+ case clipboardImageRead - case clipboardTextWrite - case permissionsStatus - case requestPostEventPermission -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -index c0b245449..a2b3bacc8 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -@@ -29,7 +29,7 @@ extension PeekabooBridgeOperation { - - /// Operations enabled by default for remote helper hosts. - public static let remoteDefaultAllowlist: Set = [ -- .literalInsert, .clipboardTextRead, .clipboardTextWrite, -+ .literalInsert, .clipboardTextRead, .clipboardImageRead, .clipboardTextWrite, - .permissionsStatus, - .requestPostEventPermission, - .daemonStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -index e6bb4ada5..d3c4c20f1 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -@@ -60,6 +60,8 @@ extension PeekabooBridgeOperationResultSemantics { - } - - return switch operation { -+ case .clipboardImageRead: -+ descriptor(completion: .readOnly, targetPolicy: .notApplicable, responseFamilies: [.clipboardImage]) - case .clipboardTextRead: - descriptor(completion: .readOnly, targetPolicy: .notApplicable, responseFamilies: [.clipboardText]) - case .clipboardTextWrite: -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -index bfc7c94ef..58fdd785a 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -@@ -3,7 +3,8 @@ import Foundation - extension PeekabooBridgeOperationResultSemantics.ResponseFamily { - func matches(_ response: PeekabooBridgeResponse) -> Bool { - switch (self, response) { -- case (.clipboardText, .clipboardText), -+ case (.clipboardImage, .clipboardImage), -+ (.clipboardText, .clipboardText), - (.clipboardTextWrite, .clipboardTextWrite), - (.literalInsert, .literalInsert), - (.agentExecutionTrace, .agentExecutionTrace), -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -index 997624588..3bf1af6bf 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -@@ -123,7 +123,7 @@ extension PeekabooBridgeResponse { - bounds: termination.receipt.windowBounds)] - case let .error(envelope): - return envelope.actionTargetReceipt.map { [DesktopTargetEvidenceAdapter.evidence(receipt: $0)] } ?? [] -- case .clipboardText, .clipboardTextWrite, .literalInsert, -+ case .clipboardImage, .clipboardText, .clipboardTextWrite, .literalInsert, - .operationSessionRollover, - .handshake, - .permissionsStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -index 555cd5819..77ac27a06 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -@@ -56,6 +56,7 @@ enum PeekabooBridgeOperationResultSemantics { - enum ResponseFamily: Hashable, Sendable { - case literalInsert - case clipboardText -+ case clipboardImage - case clipboardTextWrite - case agentExecutionTrace - case application -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -index 26d05b0c0..b14ec6edf 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -@@ -446,7 +446,7 @@ extension PeekabooBridgeOperationResultSemantics { - .performAction(target: payload.target, actionName: payload.actionName) - case let .selectText(payload): - .selectText(target: payload.target, request: payload.request) -- case .clipboardTextRead, .clipboardTextWrite, .literalInsert, -+ case .clipboardImageRead, .clipboardTextRead, .clipboardTextWrite, .literalInsert, - .attestedOperation, - .projectedAction, - .handshake, -@@ -649,7 +649,7 @@ extension PeekabooBridgeOperationResultSemantics { - return [.dispatchedUnverified] - case .browserConnect: - return [.confirmedNoChange, .dispatchedUnverified] -- case .clipboardTextRead, .permissionsStatus, .observeProcessGeneration, .certificationProducerAttestation, -+ case .clipboardImageRead, .clipboardTextRead, .permissionsStatus, .observeProcessGeneration, .certificationProducerAttestation, - .createExactWindowHeldPointerOwner, - .daemonStatus, .daemonStop, .browserStatus, - .browserDisconnect, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -index cb14f0a10..65f2fce18 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -@@ -8,6 +8,7 @@ public enum PeekabooBridgeRequest: Codable, Sendable { - case handshake(PeekabooBridgeHandshake) - case literalInsert(PeekabooBridgeLiteralInsertRequest) - case clipboardTextRead -+ case clipboardImageRead - case clipboardTextWrite(String) - case permissionsStatus - case requestPostEventPermission -@@ -139,6 +140,7 @@ extension PeekabooBridgeRequest { - case .handshake: .permissionsStatus - case .literalInsert: .literalInsert - case .clipboardTextRead: .clipboardTextRead -+ case .clipboardImageRead: .clipboardImageRead - case .clipboardTextWrite: .clipboardTextWrite - case .permissionsStatus: .permissionsStatus - case .requestPostEventPermission: .requestPostEventPermission -@@ -270,6 +272,7 @@ public enum PeekabooBridgeResponse: Codable, Sendable { - case handshake(PeekabooBridgeHandshakeResponse) - case literalInsert(PeekabooBridgeLiteralInsertResult) - case clipboardText(ClipboardTextContents) -+ case clipboardImage(ClipboardImageContents) - case clipboardTextWrite(PeekabooBridgeClipboardTextWriteResult) - case permissionsStatus(PermissionsStatus) - case daemonStatus(PeekabooDaemonStatus) -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -index e7e94b5a8..5cad2d202 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -@@ -12,6 +12,8 @@ extension PeekabooBridgeServer { - permissions: PermissionsStatus) async throws -> PeekabooBridgeHandledResponse - { - switch request.operation { -+ case .clipboardImageRead: -+ return try await self.handleClipboardImageRead() - case .clipboardTextRead, .clipboardTextWrite: - return try await self.handleClipboardTextRequest(request) - case .literalInsert: diff --git a/apps/desktop/native/patches/peekaboo-clipboard-text.patch b/apps/desktop/native/patches/peekaboo-clipboard-text.patch deleted file mode 100644 index 334eac6..0000000 --- a/apps/desktop/native/patches/peekaboo-clipboard-text.patch +++ /dev/null @@ -1,310 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -index 8f09447..117a940 100644 ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -@@ -53,6 +53,13 @@ public struct ClipboardReadResult: Sendable { - } - } - -+/// A complete, generation-bound plain-text read; absence is distinct from an empty string. -+public struct ClipboardTextContents: Codable, Sendable { -+ public let present: Bool -+ public let text: String? -+ public let change_count: Int -+} -+ - /// Possible errors thrown by the clipboard service. - public enum ClipboardServiceError: LocalizedError, Sendable { - case empty -@@ -290,6 +297,46 @@ ClipboardReadAccessProviding { - - var literalInsertChangeCount: Int { self.pasteboard.changeCount } - -+ public func readPlainText() throws -> ClipboardTextContents { -+ try Self.requireSilentReadAccess(self.readAccessStatus()) -+ let generation = self.pasteboard.changeCount -+ guard let items = self.pasteboard.pasteboardItems, items.count <= 1 else { -+ throw Self.plainTextReadRefusal("Plain-text reads require a complete clipboard with at most one item.") -+ } -+ var text: String? -+ if let item = items.first { -+ guard !item.types.isEmpty, -+ !item.types.contains(where: { $0.rawValue.localizedCaseInsensitiveContains("promise") }) else { -+ throw Self.plainTextReadRefusal("Clipboard file promises or unavailable representations cannot be read as plain text.") -+ } -+ if item.types.contains(.string) { -+ guard let value = item.string(forType: .string) else { -+ throw Self.plainTextReadRefusal("The advertised plain-text representation could not be read completely.") -+ } -+ guard value.utf16.count <= 24_000 else { -+ throw Self.plainTextReadRefusal("Clipboard text exceeds the complete 24 KB result limit.") -+ } -+ text = value -+ } -+ } else { -+ guard let types = self.pasteboard.types, types.isEmpty else { -+ throw Self.plainTextReadRefusal("The clipboard item inventory is unavailable.") -+ } -+ } -+ guard self.pasteboard.changeCount == generation else { -+ throw Self.plainTextReadRefusal("The clipboard changed during the read; request a fresh read.") -+ } -+ let result = ClipboardTextContents(present: text != nil, text: text, change_count: generation) -+ guard try JSONEncoder().encode(result).count <= 24_000 else { -+ throw Self.plainTextReadRefusal("Clipboard text exceeds the complete 24 KB result limit.") -+ } -+ return result -+ } -+ -+ private static func plainTextReadRefusal(_ message: String) -> PeekabooError { -+ .invalidInput(message) -+ } -+ - public func prepareTemporaryWrite() throws -> any ClipboardTemporaryWriteTransaction { - try self.prepareTemporaryWrite(maximumBytes: Int.max, maximumItems: Int.max, maximumRepresentations: Int.max) - } -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardText.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardText.swift -new file mode 100644 -index 0000000..2f41f07 ---- /dev/null -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClipboardText.swift -@@ -0,0 +1,68 @@ -+import Foundation -+import PeekabooAutomationKit -+import PeekabooFoundation -+ -+public struct PeekabooBridgeClipboardTextWriteResult: Codable, Sendable { -+ public let outcome: String -+ public let native_outcome: DesktopActionOutcome -+ public let clipboard_changed: Bool -+ public let error: LiteralInsertError? -+} -+ -+extension PeekabooBridgeClient { -+ public func clipboardTextRead() async throws -> ClipboardTextContents { -+ switch try await self.send(.clipboardTextRead) { -+ case let .clipboardText(result): return result -+ case let .error(error): throw error -+ default: throw PeekabooBridgeErrorEnvelope(code: .internalError, message: "Unexpected clipboard read response") -+ } -+ } -+ -+ public func clipboardTextWrite(_ text: String) async throws -> PeekabooBridgeClipboardTextWriteResult { -+ switch try await self.send(.clipboardTextWrite(text)) { -+ case let .clipboardTextWrite(result): return result -+ case let .error(error): throw error -+ default: throw PeekabooBridgeErrorEnvelope(code: .internalError, message: "Unexpected clipboard write response") -+ } -+ } -+} -+ -+@MainActor -+extension PeekabooBridgeServer { -+ func handleClipboardTextRequest(_ request: PeekabooBridgeRequest) async throws -> PeekabooBridgeHandledResponse { -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ switch request { -+ case .clipboardTextRead: -+ // Reading never consumes or clears an unresolved paste reservation. -+ return try .init(response: .clipboardText(self.literalInsertClipboard.readPlainText())) -+ case let .clipboardTextWrite(text): -+ guard text.utf16.count <= 8192 else { -+ throw DesktopActionFailure.preDispatchRefusal(reason: .invalidRequest, -+ message: "Clipboard text must contain at most 8192 UTF-16 code units.") -+ } -+ return try await ClipboardPasteTransactionGate.withExclusiveTransaction { -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ let result: PeekabooBridgeClipboardTextWriteResult -+ do { -+ let written = try self.literalInsertClipboard.setActionResult(.init( -+ representations: ClipboardWriteRequest.textRepresentations(from: Data(text.utf8)))) -+ let native = try ClipboardMutationResultSemantics.requireSuccessfulOutcome( -+ written.outcome, operation: "Clipboard text write") -+ result = .init(outcome: "completed", native_outcome: native, clipboard_changed: true, error: nil) -+ } catch let failure as DesktopActionFailure { -+ let changed = failure.outcome.dispatchState.mutationDispatched -+ result = .init(outcome: changed ? "unknown" : "refused", native_outcome: failure.outcome, -+ clipboard_changed: changed, -+ error: .init(code: failure.standardErrorCode?.rawValue ?? "CLIPBOARD_WRITE_FAILED", -+ message: failure.message, hint: failure.hint)) -+ } -+ // The signed global receipt binds the content-free native result, not the clipboard payload. -+ return .init(response: .clipboardTextWrite(result), mutation: .init( -+ outcome: .dispatchedUnverified(delivery: ClipboardMutationResultSemantics.delivery, -+ evidence: .deliveryAccepted, unitCount: .one), target: .global)) -+ } -+ default: -+ throw Self.invalidRequest(for: request) -+ } -+ } -+} -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -index d854fc8..b82ffbf 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -@@ -37,6 +37,8 @@ public enum PeekabooBridgePermissionKind: String, Codable, Sendable { - public enum PeekabooBridgeOperation: String, Codable, Sendable, CaseIterable, Hashable { - // Core - case literalInsert -+ case clipboardTextRead -+ case clipboardTextWrite - case permissionsStatus - case requestPostEventPermission - case daemonStatus -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -index edf9431..c0b2454 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -@@ -29,7 +29,7 @@ extension PeekabooBridgeOperation { - - /// Operations enabled by default for remote helper hosts. - public static let remoteDefaultAllowlist: Set = [ -- .literalInsert, -+ .literalInsert, .clipboardTextRead, .clipboardTextWrite, - .permissionsStatus, - .requestPostEventPermission, - .daemonStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -index 2b83093..e6bb4ad 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -@@ -60,6 +60,11 @@ extension PeekabooBridgeOperationResultSemantics { - } - - return switch operation { -+ case .clipboardTextRead: -+ descriptor(completion: .readOnly, targetPolicy: .notApplicable, responseFamilies: [.clipboardText]) -+ case .clipboardTextWrite: -+ descriptor(ownership: .service, completion: .dispatchedUnverified(clipboardForeground), -+ targetPolicy: .global, responseFamilies: [.clipboardTextWrite]) - case .literalInsert: - descriptor(ownership: .service, requiredPermissions: [.accessibility, .postEvent], - completion: .dispatchedUnverified(clipboardForeground), -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -index 2a3bce0..bfc7c94 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -@@ -3,7 +3,9 @@ import Foundation - extension PeekabooBridgeOperationResultSemantics.ResponseFamily { - func matches(_ response: PeekabooBridgeResponse) -> Bool { - switch (self, response) { -- case (.literalInsert, .literalInsert), -+ case (.clipboardText, .clipboardText), -+ (.clipboardTextWrite, .clipboardTextWrite), -+ (.literalInsert, .literalInsert), - (.agentExecutionTrace, .agentExecutionTrace), - (.application, .application), - (.applicationMutationInventory, .applicationMutationInventory), -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -index 654c77a..9976245 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -@@ -123,7 +123,7 @@ extension PeekabooBridgeResponse { - bounds: termination.receipt.windowBounds)] - case let .error(envelope): - return envelope.actionTargetReceipt.map { [DesktopTargetEvidenceAdapter.evidence(receipt: $0)] } ?? [] -- case .literalInsert, -+ case .clipboardText, .clipboardTextWrite, .literalInsert, - .operationSessionRollover, - .handshake, - .permissionsStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -index 3b4734f..555cd58 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -@@ -55,6 +55,8 @@ enum PeekabooBridgeOperationResultSemantics { - - enum ResponseFamily: Hashable, Sendable { - case literalInsert -+ case clipboardText -+ case clipboardTextWrite - case agentExecutionTrace - case application - case applicationMutationInventory -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -index 36daeb5..26d05b0 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -@@ -446,7 +446,7 @@ extension PeekabooBridgeOperationResultSemantics { - .performAction(target: payload.target, actionName: payload.actionName) - case let .selectText(payload): - .selectText(target: payload.target, request: payload.request) -- case .literalInsert, -+ case .clipboardTextRead, .clipboardTextWrite, .literalInsert, - .attestedOperation, - .projectedAction, - .handshake, -@@ -597,7 +597,7 @@ extension PeekabooBridgeOperationResultSemantics { - .suspectedNoop, - ] - switch request.operation { -- case .literalInsert, .agentExecutionTrace: -+ case .clipboardTextWrite, .literalInsert, .agentExecutionTrace: - return [.dispatchedUnverified] - case .requestPostEventPermission, .browserExecute, .swipe, .drag, .moveMouse, - .clickMenuItem, .clickMenuItemByName, .clickMenuExtra, -@@ -649,7 +649,7 @@ extension PeekabooBridgeOperationResultSemantics { - return [.dispatchedUnverified] - case .browserConnect: - return [.confirmedNoChange, .dispatchedUnverified] -- case .permissionsStatus, .observeProcessGeneration, .certificationProducerAttestation, -+ case .clipboardTextRead, .permissionsStatus, .observeProcessGeneration, .certificationProducerAttestation, - .createExactWindowHeldPointerOwner, - .daemonStatus, .daemonStop, .browserStatus, - .browserDisconnect, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -index 21f7862..4886efd 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -@@ -247,7 +247,7 @@ extension PeekabooBridgeRequest { - /// never own the desktop lane or mutation watermark: its child re-enters this same Bridge and - /// each nested tool call owns its own exact-target lane and signed receipt. - var bypassesOuterDesktopMutationLane: Bool { -- [.agentExecutionTrace, .literalInsert].contains(self.unwrappedOperationRequest.operation) -+ [.agentExecutionTrace, .literalInsert, .clipboardTextWrite].contains(self.unwrappedOperationRequest.operation) - } - } - -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -index 8d3e670..cb14f0a 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -@@ -7,6 +7,8 @@ public enum PeekabooBridgeRequest: Codable, Sendable { - indirect case projectedAction(PeekabooBridgeProjectedActionRequest) - case handshake(PeekabooBridgeHandshake) - case literalInsert(PeekabooBridgeLiteralInsertRequest) -+ case clipboardTextRead -+ case clipboardTextWrite(String) - case permissionsStatus - case requestPostEventPermission - case daemonStatus -@@ -136,6 +138,8 @@ extension PeekabooBridgeRequest { - case let .projectedAction(payload): payload.request.operation - case .handshake: .permissionsStatus - case .literalInsert: .literalInsert -+ case .clipboardTextRead: .clipboardTextRead -+ case .clipboardTextWrite: .clipboardTextWrite - case .permissionsStatus: .permissionsStatus - case .requestPostEventPermission: .requestPostEventPermission - case .daemonStatus: .daemonStatus -@@ -265,6 +269,8 @@ public enum PeekabooBridgeResponse: Codable, Sendable { - indirect case projectedAction(PeekabooBridgeProjectedActionResponse) - case handshake(PeekabooBridgeHandshakeResponse) - case literalInsert(PeekabooBridgeLiteralInsertResult) -+ case clipboardText(ClipboardTextContents) -+ case clipboardTextWrite(PeekabooBridgeClipboardTextWriteResult) - case permissionsStatus(PermissionsStatus) - case daemonStatus(PeekabooDaemonStatus) - case agentExecutionTrace(PeekabooBridgeAgentExecutionTraceResponse) -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -index d68c1b5..e7e94b5 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -@@ -12,6 +12,8 @@ extension PeekabooBridgeServer { - permissions: PermissionsStatus) async throws -> PeekabooBridgeHandledResponse - { - switch request.operation { -+ case .clipboardTextRead, .clipboardTextWrite: -+ return try await self.handleClipboardTextRequest(request) - case .literalInsert: - return try await self.handleLiteralInsert(request) - case .permissionsStatus, .daemonStatus, .daemonStop: diff --git a/apps/desktop/native/patches/peekaboo-close.patch b/apps/desktop/native/patches/peekaboo-close.patch deleted file mode 100644 index c98fff4..0000000 --- a/apps/desktop/native/patches/peekaboo-close.patch +++ /dev/null @@ -1,156 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/WindowManagementService+StateOperations.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/WindowManagementService+StateOperations.swift -index cc8ea70e4..d0c57fa06 100644 ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/WindowManagementService+StateOperations.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/WindowManagementService+StateOperations.swift -@@ -3,6 +3,7 @@ import AppKit - import AXorcist - import CoreGraphics - import Foundation -+import os - import PeekabooFoundation - - @MainActor -@@ -70,21 +71,25 @@ extension WindowManagementService { - hint: "Restore the same exact window first, or retry with explicit foreground consent.") - } - -- let attempt = try await self.attemptPinnedBackgroundClose(expectedIdentity) -- guard attempt.dispatched else { -- throw OperationError.interactionFailed( -- action: "close window", -- reason: "Window close operation failed") -- } -- guard attempt.disappeared else { -- throw WindowManagementActionOutcome.suspectedNoop( -+ try await BoundedBackgroundWindowAX.dispatchOneClose(expectedIdentity: expectedIdentity) -+ do { -+ guard try await self.verifyPinnedWindowClose(expectedIdentity) == .succeeded else { -+ // An accepted close may be waiting on unsaved work; it is not safe to repeat. -+ return .dispatchedUnverified( -+ delivery: WindowManagementActionOutcome.backgroundActionDelivery, -+ evidence: .operationStillRunning, -+ unitCount: .one) -+ } -+ } catch { -+ throw WindowManagementActionOutcome.dispatchedUnverified( - action: "close window", - delivery: WindowManagementActionOutcome.backgroundActionDelivery, -- dispatchCount: attempt.dispatchCount) -+ dispatchCount: 1, -+ cause: error) - } -- return WindowManagementActionOutcome.confirmedChange( -+ return .confirmedChange( - delivery: WindowManagementActionOutcome.backgroundActionDelivery, -- dispatchCount: attempt.dispatchCount) -+ unitCount: .one) - } - } - return DesktopActionResult(outcome: outcome) -@@ -1087,6 +1092,106 @@ private enum BoundedBackgroundWindowAX { - } - } - -+ static func dispatchOneClose(expectedIdentity: WindowMutationIdentity) async throws { -+ let cancelled = OSAllocatedUnfairLock(initialState: false) -+ try await withTaskCancellationHandler { -+ let result: Result = await self.perform(expectedIdentity: expectedIdentity) { -+ guard SystemIdentityResolver.validateWindowMutationIdentity(expectedIdentity), -+ let capturedBounds = expectedIdentity.capturedBounds, -+ let windowID = CGWindowID(exactly: expectedIdentity.windowID), -+ let rawWindow = self.exactWindow( -+ windowID: windowID, -+ ownerPID: expectedIdentity.ownerProcessIdentifier) -+ else { -+ return .failure(.preDispatchRefusal( -+ reason: .targetUnavailable, -+ message: "The exact window could not be verified before close.")) -+ } -+ return AXChildWindowMessagingTimeout.perform(on: rawWindow, timeout: self.messagingTimeout) { window in -+ var actionsValue: CFArray? -+ guard AXUIElementCopyActionNames(window, &actionsValue) == .success, -+ let actions = actionsValue as? [String] -+ else { -+ return .failure(.preDispatchRefusal( -+ reason: .operationUnsupported, -+ message: "The exact window's supported close actions could not be read.")) -+ } -+ // Choose one route from read-only evidence. No native failure selects another route. -+ let receiver: AXUIElement -+ let action: CFString -+ if actions.contains("AXClose") { -+ receiver = window -+ action = "AXClose" as CFString -+ } else { -+ var buttonValue: CFTypeRef? -+ guard AXUIElementCopyAttributeValue( -+ window, kAXCloseButtonAttribute as CFString, &buttonValue) == .success, -+ let buttonValue, CFGetTypeID(buttonValue) == AXUIElementGetTypeID() -+ else { -+ return .failure(.preDispatchRefusal( -+ reason: .operationUnsupported, -+ message: "The exact window exposes no supported background close route.")) -+ } -+ receiver = unsafeDowncast(buttonValue, to: AXUIElement.self) -+ action = kAXPressAction as CFString -+ } -+ return AXChildWindowMessagingTimeout.perform(on: receiver, timeout: self.messagingTimeout) { target in -+ if !CFEqual(target, window) { -+ var buttonActionsValue: CFArray? -+ var currentButton: CFTypeRef? -+ guard AXUIElementCopyActionNames(target, &buttonActionsValue) == .success, -+ (buttonActionsValue as? [String])?.contains(kAXPressAction as String) == true, -+ AXUIElementCopyAttributeValue( -+ window, kAXCloseButtonAttribute as CFString, ¤tButton) == .success, -+ let currentButton, CFEqual(currentButton, target) -+ else { -+ return .failure(.preDispatchRefusal( -+ reason: .operationUnsupported, -+ message: "The exact window's close button could not be verified.")) -+ } -+ } -+ var ownerPID: pid_t = 0 -+ var currentWindowID: CGWindowID = 0 -+ guard AXUIElementGetPid(target, &ownerPID) == .success, -+ ownerPID == expectedIdentity.ownerProcessIdentifier, -+ AXWindowIDResolver.copyWindowID(window, into: ¤tWindowID) == .success, -+ currentWindowID == windowID, -+ self.bounds(of: window) == capturedBounds, -+ self.boolAttribute(kAXMinimizedAttribute as String, of: window) == false, -+ SystemIdentityResolver.validateWindowMutationIdentity(expectedIdentity) -+ else { -+ return .failure(.preDispatchRefusal( -+ reason: .targetUnavailable, -+ message: "The exact window changed before close.")) -+ } -+ guard !cancelled.withLock({ $0 }) else { -+ return .failure(.preDispatchRefusal( -+ reason: .requestCancelled, -+ message: "Close was cancelled before native dispatch.")) -+ } -+ let nativeResult = AXUIElementPerformAction(target, action) -+ if nativeResult == .success { return .success(()) } -+ let error = ActionInputDriver.classify(nativeResult) -+ if ActionInputDriver.nativeMutationFailureMayHaveDispatched(error) { -+ return .failure(.indeterminate( -+ delivery: WindowManagementActionOutcome.backgroundActionDelivery, -+ evidence: .completionUnknown, -+ unitCount: .one, -+ message: "The close request may have been delivered; native completion is unknown.", -+ hint: "Refresh the window inventory; do not blindly retry close.", -+ causeDescription: String(describing: error))) -+ } -+ return .failure(WindowManagementActionOutcome.refused(action: "close window", error: error)) -+ } -+ } -+ } -+ // Keep the coordinator lane until the actual AX call settles, including after cancellation. -+ try result.get() -+ } onCancel: { -+ cancelled.withLock { $0 = true } -+ } -+ } -+ - static func dispatchClose( - expectedIdentity: WindowMutationIdentity, - action: BoundedBackgroundWindowCloseAction) async -> Bool diff --git a/apps/desktop/native/patches/peekaboo-insert.patch b/apps/desktop/native/patches/peekaboo-insert.patch deleted file mode 100644 index 3d8241e..0000000 --- a/apps/desktop/native/patches/peekaboo-insert.patch +++ /dev/null @@ -1,899 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardPasteTransactionGate.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardPasteTransactionGate.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardPasteTransactionGate.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardPasteTransactionGate.swift -@@ -79,6 +79,8 @@ - case unsafeDirectory(path: String) - case unsafeLockFile(path: String) - case lockTimeout(path: String) -+ case invalidReservation(path: String) -+ case transactionUnavailable - - var errorDescription: String? { - switch self { -@@ -93,6 +95,10 @@ - return "Clipboard paste transaction lock is not a regular file owned by the current user: \(path)" - case let .lockTimeout(path): - return "Timed out waiting for the exclusive clipboard paste transaction lock at \(path)." -+ case let .invalidReservation(path): -+ return "Clipboard paste reservation at \(path) is unreadable or unsupported. No new clipboard mutation was admitted; recover it only after the original receiver has terminated." -+ case .transactionUnavailable: -+ return "Clipboard paste reservation requires the active transaction gate." - } - } - } -@@ -102,6 +108,134 @@ - - /// Serializes callers in a shared host before they enter the file-lock wait loop. - @MainActor private static var isActive = false -+ @TaskLocal private static var transaction: Transaction? -+ @MainActor private static var pendingProof: PendingProof? -+ -+ private struct Reservation: Codable, Sendable { -+ let version: Int -+ let id: UUID -+ let pid: Int32 -+ let processStartIdentity: UInt64 -+ let windowID: Int -+ } -+ -+ private struct PendingProof: Sendable { -+ let id: UUID -+ let path: String -+ let canRelease: @MainActor @Sendable () async -> Bool -+ } -+ -+ @MainActor -+ private final class Transaction { -+ let descriptor: Int32 -+ let path: String -+ var active = true -+ var reservation: Reservation? -+ -+ init(descriptor: Int32, path: String) { -+ self.descriptor = descriptor; self.path = path -+ } -+ -+ func read() throws -> Reservation? { -+ var info = stat() -+ guard fstat(self.descriptor, &info) == 0 else { throw self.failure("inspect reservation") } -+ if info.st_size == 0 { return nil } -+ guard info.st_size > 0, info.st_size <= 2048 else { throw GateError.invalidReservation(path: self.path) } -+ var data = Data(count: Int(info.st_size)) -+ let count = data.withUnsafeMutableBytes { pread(self.descriptor, $0.baseAddress!, $0.count, 0) } -+ guard count == data.count, -+ let value = try? JSONDecoder().decode(Reservation.self, from: data), -+ value.version == 1, value.pid > 0, value.processStartIdentity > 0, value.windowID > 0 -+ else { throw GateError.invalidReservation(path: self.path) } -+ return value -+ } -+ -+ func write(_ value: Reservation?) throws { -+ let data = try value.map { try JSONEncoder().encode($0) } ?? Data() -+ guard ftruncate(self.descriptor, 0) == 0 else { throw self.failure("truncate reservation") } -+ var offset = 0 -+ while offset < data.count { -+ let written = data.withUnsafeBytes { -+ pwrite(self.descriptor, $0.baseAddress!.advanced(by: offset), $0.count - offset, off_t(offset)) -+ } -+ if written < 0, errno == EINTR { continue } -+ guard written > 0 else { throw self.failure("write reservation") } -+ offset += written -+ } -+ guard fsync(self.descriptor) == 0 else { throw self.failure("sync reservation") } -+ self.reservation = value -+ } -+ -+ private func failure(_ operation: String) -> GateError { -+ .systemCall(operation: operation, path: self.path, code: errno) -+ } -+ } -+ -+ /// Called synchronously immediately before primary paste-key delivery, never after it. -+ /// The deferred proof is bounded and read-only; it must not retain the prior clipboard backup. -+ @MainActor -+ public static func reservePaste( -+ process: ApplicationProcessIdentity, windowID: Int, -+ canRelease: @escaping @MainActor @Sendable () async -> Bool) throws -> UUID -+ { -+ guard let transaction, transaction.active, transaction.reservation == nil else { -+ throw GateError.transactionUnavailable -+ } -+ let value = Reservation(version: 1, id: UUID(), pid: process.processIdentifier, -+ processStartIdentity: process.processStartIdentity, windowID: windowID) -+ // A failed metadata write cannot be followed by V delivery. Preserve uncertain disk state on cleanup failure. -+ transaction.reservation = value -+ do { try transaction.write(value) } -+ catch { -+ try? transaction.write(nil) -+ throw error -+ } -+ self.pendingProof = PendingProof(id: value.id, path: transaction.path, canRelease: canRelease) -+ return value.id -+ } -+ -+ @MainActor -+ public static func releasePaste(_ id: UUID) throws { -+ guard let transaction, transaction.active, transaction.reservation?.id == id else { -+ throw GateError.transactionUnavailable -+ } -+ try transaction.write(nil) -+ if self.pendingProof?.id == id { self.pendingProof = nil } -+ } -+ -+ @MainActor -+ public static var hasUnresolvedPaste: Bool { self.transaction?.reservation != nil } -+ -+ @MainActor -+ private static func admit(_ transaction: Transaction) async throws { -+ transaction.reservation = try transaction.read() -+ guard let reservation = transaction.reservation else { -+ if self.pendingProof?.path == transaction.path { self.pendingProof = nil } -+ return -+ } -+ var resolved = self.processGenerationTerminated(reservation) -+ if !resolved, let proof = self.pendingProof, -+ proof.path == transaction.path, proof.id == reservation.id -+ { -+ resolved = await proof.canRelease() -+ } -+ if resolved { -+ try self.releasePaste(reservation.id) -+ return -+ } -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .targetUnavailable, -+ message: "A previous clipboard paste to process \(reservation.pid), window \(reservation.windowID) remains unresolved.", -+ hint: "No new clipboard mutation was sent. Inspect the original receiver; its observed consumption or original process termination must resolve ownership first.") -+ } -+ -+ private static func processGenerationTerminated(_ reservation: Reservation) -> Bool { -+ switch SystemIdentityResolver.processStartIdentityObservation(reservation.pid) { -+ case let .identity(current): current != reservation.processStartIdentity -+ case .absent: true -+ case .permissionDenied, .unavailable: false -+ } -+ } - - @MainActor - public static func withExclusiveTransaction( -@@ -164,7 +298,21 @@ - defer { flock(fd, LOCK_UN) } - - try self.checkLockDeadline(deadline, now: now(), path: standardizedLockPath) -- return try await operation() -+ let transaction = Transaction(descriptor: fd, path: standardizedLockPath) -+ defer { transaction.active = false } -+ return try await Self.$transaction.withValue(transaction) { -+ do { try await self.admit(transaction) } -+ catch is CancellationError { throw CancellationError() } -+ catch let failure as DesktopActionFailure { throw failure } -+ catch { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .targetUnavailable, -+ message: "Shared clipboard ownership could not be verified before this operation.", -+ hint: error.localizedDescription) -+ } -+ try Task.checkCancellation() -+ return try await operation() -+ } - } - - @MainActor -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ClipboardService.swift -@@ -288,10 +288,19 @@ - return .init(policy: policy) - } - -+ var literalInsertChangeCount: Int { self.pasteboard.changeCount } -+ - public func prepareTemporaryWrite() throws -> any ClipboardTemporaryWriteTransaction { -+ try self.prepareTemporaryWrite(maximumBytes: Int.max, maximumItems: Int.max, maximumRepresentations: Int.max) -+ } -+ -+ public func prepareTemporaryWrite( -+ maximumBytes: Int, maximumItems: Int, maximumRepresentations: Int) throws -> any ClipboardTemporaryWriteTransaction -+ { - try Self.requireSilentReadAccess(self.readAccessStatus()) - let originalChangeCount = self.pasteboard.changeCount -- let items = try self.temporarySnapshotItems() -+ let items = try self.temporarySnapshotItems( -+ maximumBytes: maximumBytes, maximumItems: maximumItems, maximumRepresentations: maximumRepresentations) - guard self.pasteboard.changeCount == originalChangeCount else { - throw ClipboardTemporaryWriteError.snapshotChanged - } -@@ -326,7 +335,9 @@ - } - } - -- private func temporarySnapshotItems() throws -> [[ClipboardRepresentation]] { -+ private func temporarySnapshotItems( -+ maximumBytes: Int, maximumItems: Int, maximumRepresentations: Int) throws -> [[ClipboardRepresentation]] -+ { - guard let items = self.pasteboard.pasteboardItems else { - throw ClipboardTemporaryWriteError.snapshotUnavailable - } -@@ -336,14 +347,24 @@ - } - return [] - } -+ guard items.count <= maximumItems else { throw ClipboardTemporaryWriteError.snapshotUnavailable } -+ var bytes = 0 -+ var representations = 0 - return try items.map { item in - guard !item.types.isEmpty else { - throw ClipboardTemporaryWriteError.snapshotUnavailable - } -+ representations += item.types.count -+ guard representations <= maximumRepresentations else { throw ClipboardTemporaryWriteError.snapshotUnavailable } - return try item.types.map { type in -+ guard !type.rawValue.localizedCaseInsensitiveContains("promise") else { -+ throw ClipboardTemporaryWriteError.snapshotUnavailable -+ } - guard let data = item.data(forType: type) else { - throw ClipboardTemporaryWriteError.snapshotUnavailable - } -+ guard data.count <= maximumBytes - bytes else { throw ClipboardTemporaryWriteError.snapshotUnavailable } -+ bytes += data.count - return ClipboardRepresentation(utiIdentifier: type.rawValue, data: data) - } - } -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/HotkeyService.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/HotkeyService.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/HotkeyService.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/HotkeyService.swift -@@ -197,6 +197,8 @@ - automationTarget: UIAutomationTarget, - clipboardClaim: GeneralPasteboardWriteClaim? = nil, - prepareBackgroundWindow: Bool = false, -+ laneIsOwned: Bool = false, -+ beforePrimaryKeyDown: (@MainActor @Sendable () throws -> Void)? = nil, - deliveryValidator: (@MainActor @Sendable () async throws -> Void)? = nil) async throws - -> UIAutomationActionResult - { -@@ -209,8 +211,9 @@ - try BackgroundHotkeyPolicy.validate(keys: keys) - let parsedKeys = try Self.parsedKeys(keys) - let plannedChord = try? self.makeHotkeyPlan(parsedKeys) -- if clipboardClaim != nil || prepareBackgroundWindow { -+ if clipboardClaim != nil || prepareBackgroundWindow || beforePrimaryKeyDown != nil { - guard automationTarget.exactWindow != nil, holdDuration > 0, -+ beforePrimaryKeyDown == nil || clipboardClaim != nil, - !prepareBackgroundWindow || - (clipboardClaim != nil && automationTarget.exactWindow?.focusedElement != nil), - let plannedChord, -@@ -329,7 +332,8 @@ - let outcome = try await self.performTargetedHotkey( - parsedKeys: parsedKeys, plannedChord: plannedChord, holdDuration: holdDuration, - holdNanoseconds: holdNanoseconds, -- automationTarget: automationTarget, targetValidator: targetValidator) -+ automationTarget: automationTarget, targetValidator: targetValidator, -+ beforePrimaryKeyDown: beforePrimaryKeyDown) - guard prepareBackgroundWindow else { return outcome } - preparation.record(.outcome(outcome)) - guard let result = preparation.successResolution().outcome else { -@@ -347,7 +351,12 @@ - finalize: self.operationFinalizer) - let result: UIAutomationActionResult - do { -- result = try await self.desktopOperationExecutor.executeWithTargetIdentity(plan) -+ if laneIsOwned { -+ let owned = try await self.desktopOperationExecutor.executeOwned(plan) -+ result = UIAutomationActionResult(payload: owned, outcome: owned.outcome, targetIdentity: try plan.targetIdentity()) -+ } else { -+ result = try await self.desktopOperationExecutor.executeWithTargetIdentity(plan) -+ } - } catch let error as ActionInputError where error == .unsupported(.actionUnsupported) && - automationTarget.exactWindow != nil && - strategy.strategy == .actionOnly -@@ -368,7 +377,8 @@ - holdDuration: Int, - holdNanoseconds: UInt64, - automationTarget: UIAutomationTarget, -- targetValidator: @escaping @MainActor @Sendable () async throws -> Void) async throws -> DesktopActionOutcome -+ targetValidator: @escaping @MainActor @Sendable () async throws -> Void, -+ beforePrimaryKeyDown: (@MainActor @Sendable () throws -> Void)?) async throws -> DesktopActionOutcome - { - guard let targetProcessIdentifier = automationTarget.processIdentifier else { - throw PeekabooError.invalidInput("Targeted hotkey requires a process target") -@@ -383,7 +393,8 @@ - holdNanoseconds: holdNanoseconds, - targetProcessIdentifier: targetProcessIdentifier, - deliveryValidator: targetValidator, -- cleanupProcessIdentity: automationTarget.exactWindow?.identity.processIdentity) -+ cleanupProcessIdentity: automationTarget.exactWindow?.identity.processIdentity, -+ beforePrimaryKeyDown: beforePrimaryKeyDown) - - do { - if holdDuration <= 0 { -@@ -458,7 +469,8 @@ - holdNanoseconds: UInt64, - targetProcessIdentifier: pid_t, - deliveryValidator: (@MainActor @Sendable () async throws -> Void)? = nil, -- cleanupProcessIdentity: ApplicationProcessIdentity? = nil) async throws -+ cleanupProcessIdentity: ApplicationProcessIdentity? = nil, -+ beforePrimaryKeyDown: (@MainActor @Sendable () throws -> Void)? = nil) async throws - -> Int - { - guard self.postEventAccessEvaluator() else { -@@ -476,7 +488,8 @@ - holdNanoseconds: holdNanoseconds, - targetProcessIdentifier: targetProcessIdentifier, - deliveryValidator: deliveryValidator, -- cleanupProcessIdentity: cleanupProcessIdentity) -+ cleanupProcessIdentity: cleanupProcessIdentity, -+ beforePrimaryKeyDown: beforePrimaryKeyDown) - } - - var pressedModifierKeyCodes: Set = [] -@@ -544,7 +557,8 @@ - holdNanoseconds: UInt64, - targetProcessIdentifier: pid_t, - deliveryValidator: (@MainActor @Sendable () async throws -> Void)?, -- cleanupProcessIdentity: ApplicationProcessIdentity) async throws -> Int -+ cleanupProcessIdentity: ApplicationProcessIdentity, -+ beforePrimaryKeyDown: (@MainActor @Sendable () throws -> Void)?) async throws -> Int - { - var state = HeldHotkeyState() - -@@ -567,6 +581,7 @@ - try await self.validateDelivery( - deliveryValidator, - emittedUnitCount: state.emittedUnitCount) -+ try beforePrimaryKeyDown?() - state.primaryKeyIsDown = true - self.eventPoster(eventPlan.primaryKeyDownEvent, targetProcessIdentifier) - state.emittedUnitCount += 1 -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/LiteralInsert.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/LiteralInsert.swift -new file mode 100644 ---- /dev/null -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/LiteralInsert.swift -@@ -0,0 +1,315 @@ -+import AppKit -+import ApplicationServices -+import Foundation -+import PeekabooFoundation -+ -+public struct LiteralInsertError: Codable, Sendable { -+ public let code: String -+ public let message: String -+ public var hint: String? -+ public var cause: String? -+ -+ public init(code: String, message: String, hint: String? = nil, cause: String? = nil) { -+ self.code = code; self.message = message; self.hint = hint; self.cause = cause -+ } -+} -+ -+public struct LiteralInsertReceipt: Codable, Sendable { -+ public let pid: Int32 -+ public let window_id: Int -+ public let process_start_identity_decimal: String -+} -+ -+/// Clipboard contents and the private text comparison never enter the response. -+public struct LiteralInsertResult: Codable, Sendable { -+ public var outcome: String -+ public var native_outcome: DesktopActionOutcome? -+ public let target_receipt: LiteralInsertReceipt? -+ public var clipboard_changed = false -+ public var clipboard_cleanup = "not_needed" -+ public var clipboard_ownership = "released" -+ public var consumption = "unverified" -+ public var requires_fresh_observation = false -+ public var error: LiteralInsertError? -+} -+ -+@MainActor -+public protocol LiteralInsertServiceProtocol: UIAutomationServiceProtocol { -+ func literalInsert( -+ text: String, target: UIAutomationTarget.ExactWindow, clipboard: ClipboardService, -+ checkActive: @escaping @MainActor @Sendable () throws -> Void) async throws -> LiteralInsertResult -+} -+ -+@MainActor -+private final class LiteralInsertReservation { -+ var token: UUID? -+ var resolved = false -+} -+ -+extension UIAutomationService: LiteralInsertServiceProtocol { -+ public func literalInsert( -+ text: String, target: UIAutomationTarget.ExactWindow, clipboard: ClipboardService, -+ checkActive: @escaping @MainActor @Sendable () throws -> Void) async throws -> LiteralInsertResult -+ { -+ guard !text.isEmpty, text.utf16.count <= 8192, target.focusedElement != nil else { -+ throw Self.literalInsertRefusal("Literal insertion requires 1–8192 UTF-16 units and an exact focused receiver.") -+ } -+ do { -+ return try await self.operationLaneCoordinator.run(scope: .process(target.identity.processIdentity), access: .write) { -+ try await self.insertLiteralText(text, target: target, clipboard: clipboard, checkActive: checkActive) -+ } -+ } catch is CancellationError { throw CancellationError() } -+ catch let failure as DesktopActionFailure { throw failure } -+ catch { throw Self.literalInsertRefusal("Literal insertion could not establish its read-only preflight: \(error.localizedDescription)") } -+ } -+ -+ private func insertLiteralText( -+ _ text: String, target: UIAutomationTarget.ExactWindow, clipboard: ClipboardService, -+ checkActive: @escaping @MainActor @Sendable () throws -> Void) async throws -> LiteralInsertResult -+ { -+ try checkActive() -+ let focused = try await self.requireExactWindowKeyboardFocus( -+ expectedWindowIdentity: target.identity, expectedWindowBounds: target.bounds, -+ expectedFocusedElement: target.focusedElement) -+ guard let retained = focused.nativeElement, -+ let baseline = try await self.literalInsertState(target, retained: retained), -+ let range = Range(NSRange(location: baseline.location, length: baseline.length), in: baseline.text) -+ else { throw Self.literalInsertRefusal("The exact receiver's complete text and selection cannot be read safely.") } -+ let expectedText = baseline.text.replacingCharacters(in: range, with: text) -+ guard expectedText.utf16.count <= 65_536 else { -+ throw Self.literalInsertRefusal("The intended edit exceeds the complete text verification limit.") -+ } -+ let expectedLocation = baseline.location + text.utf16.count -+ let meaningful = !baseline.matches(text: expectedText, location: expectedLocation, length: 0) -+ guard let transaction = try clipboard.prepareTemporaryWrite( -+ maximumBytes: 32 * 1024 * 1024, maximumItems: 128, maximumRepresentations: 512 -+ ) as? any ClipboardTemporaryWriteClaimProviding else { -+ throw Self.literalInsertRefusal("The native clipboard owner cannot retain a generation claim.") -+ } -+ var result = LiteralInsertResult(outcome: "refused", target_receipt: .init( -+ pid: target.identity.ownerProcessIdentifier, window_id: target.identity.windowID, -+ process_start_identity_decimal: String(target.identity.ownerProcessStartIdentity))) -+ var possibleInput = false -+ var claim: GeneralPasteboardWriteClaim? -+ let reservation = LiteralInsertReservation() -+ do { -+ try checkActive() -+ try await self.validateLiteralInsertBaseline(baseline, target: target, retained: retained) -+ let prepareBackgroundWindow = !Self.literalInsertTargetIsActive(target) -+ try checkActive() -+ let write = ClipboardWriteRequest(representations: ClipboardWriteRequest.textRepresentations(from: Data(text.utf8))) -+ let written = try transaction.writeWithClaim(write) -+ claim = written.claim -+ let writeClaim = written.claim -+ try checkActive() -+ possibleInput = true -+ let input = try await self.hotkeyService.hotkey( -+ keys: "cmd,v", holdDuration: 50, automationTarget: .exactWindow(target), -+ clipboardClaim: written.claim, prepareBackgroundWindow: prepareBackgroundWindow, laneIsOwned: true, -+ beforePrimaryKeyDown: { -+ let token = try ClipboardPasteTransactionGate.reservePaste( -+ process: target.identity.processIdentity, windowID: target.identity.windowID) -+ { -+ if reservation.resolved { return true } -+ guard meaningful, clipboard.literalInsertChangeCount == writeClaim.changeCount else { return false } -+ do { -+ let observed = try await self.literalInsertState(target, retained: retained) -+ return observed?.matches(text: expectedText, location: expectedLocation, length: 0) == true -+ && clipboard.literalInsertChangeCount == writeClaim.changeCount -+ } catch { return false } -+ } -+ reservation.token = token -+ do { -+ // Persistence can block; cancellation and ownership must still hold before V is posted. -+ try checkActive() -+ guard clipboard.literalInsertChangeCount == writeClaim.changeCount, -+ SystemIdentityResolver.processStartIdentity(target.identity.ownerProcessIdentifier) -+ == target.identity.ownerProcessStartIdentity else { -+ throw Self.literalInsertRefusal("Clipboard ownership or the original receiving process changed before paste delivery.") -+ } -+ guard prepareBackgroundWindow || Self.literalInsertTargetIsActive(target) else { -+ throw Self.literalInsertRefusal("The exact receiving app is no longer active; observe it before another insertion.") -+ } -+ } catch { -+ reservation.resolved = true -+ try? ClipboardPasteTransactionGate.releasePaste(token) -+ reservation.token = nil -+ throw error -+ } -+ }, -+ deliveryValidator: { -+ try checkActive() -+ try await self.validateLiteralInsertBaseline(baseline, target: target, retained: retained) -+ guard prepareBackgroundWindow || Self.literalInsertTargetIsActive(target) else { -+ throw Self.literalInsertRefusal("The exact receiving app is no longer active; observe it before another insertion.") -+ } -+ }) -+ result.native_outcome = input.outcome -+ guard let outcome = input.outcome, let actual = input.targetIdentity?.exactWindow, -+ actual.identity.hasSameStableReceipt(as: target.identity), actual.bounds == target.bounds -+ else { -+ throw DesktopActionFailure.indeterminate(evidence: .completionUnknown, -+ message: "Literal insertion returned without its expected exact-window delivery receipt.") -+ } -+ possibleInput = outcome.dispatchState.mutationDispatched -+ result.outcome = outcome.state == .refused ? "refused" : "unknown" -+ } catch let failure as DesktopActionFailure { -+ result.native_outcome = failure.outcome -+ possibleInput = failure.outcome.dispatchState.mutationDispatched -+ result.outcome = possibleInput ? "unknown" : "refused" -+ result.error = .init(code: failure.standardErrorCode?.rawValue ?? "LITERAL_INSERT_FAILED", -+ message: failure.message, hint: failure.hint, cause: failure.causeDescription) -+ } catch let error as InputDeliveryIndeterminateError { -+ let failure = error.desktopActionFailure(delivery: .init(mechanism: .windowTargetedEvents, mode: .background)) -+ result.native_outcome = failure.outcome -+ possibleInput = true -+ result.outcome = "unknown" -+ result.error = .init(code: "LITERAL_INSERT_FAILED", message: failure.message, hint: failure.hint, cause: failure.causeDescription) -+ } catch { -+ result.outcome = possibleInput ? "unknown" : "refused" -+ result.error = .init(code: "LITERAL_INSERT_FAILED", message: error.localizedDescription) -+ } -+ result.clipboard_changed = transaction.didMutate -+ result.requires_fresh_observation = possibleInput -+ if possibleInput, meaningful, let claim { -+ do { -+ let deadline = ContinuousClock.now.advanced(by: .seconds(2)) -+ while ContinuousClock.now < deadline { -+ try checkActive() -+ guard clipboard.literalInsertChangeCount == claim.changeCount, -+ SystemIdentityResolver.processStartIdentity(target.identity.ownerProcessIdentifier) -+ == target.identity.ownerProcessStartIdentity else { break } -+ let observed: LiteralInsertState? -+ do { -+ observed = try await self.literalInsertState(target, retained: retained) -+ } catch is CancellationError { throw CancellationError() } -+ catch { observed = nil } -+ if observed?.matches(text: expectedText, location: expectedLocation, length: 0) == true { -+ result.consumption = "observed" -+ reservation.resolved = true -+ if let token = reservation.token { -+ do { try ClipboardPasteTransactionGate.releasePaste(token) } -+ catch { -+ result.error = .init(code: "CLIPBOARD_OWNERSHIP_RELEASE_FAILED", -+ message: "The edit was observed, but shared clipboard ownership could not be released.") -+ } -+ } -+ if result.native_outcome?.state == .dispatchedUnverified || result.native_outcome?.state == .confirmedChange { -+ result.outcome = "completed" -+ } -+ break -+ } -+ try await Task.sleep(for: .milliseconds(20)) -+ } -+ } catch { -+ // Read-only verification cannot erase delivered input or authorize a retry. -+ if result.error == nil { -+ result.error = .init(code: "INSERTION_UNVERIFIED", message: "The receiver's text change was not confirmed.") -+ } -+ } -+ } -+ if reservation.token == nil || result.consumption == "observed" { -+ do { result.clipboard_cleanup = try transaction.cleanup().status.rawValue } -+ catch { -+ result.clipboard_cleanup = "failed" -+ if result.error == nil { -+ result.error = .init(code: "CLIPBOARD_CLEANUP_FAILED", message: "The prior clipboard could not be restored; input delivery is reported separately.") -+ } -+ } -+ } else { -+ // Cleanup is explicit, never a deinit action. Dropping the private backup cannot expose it to a delayed Cmd+V. -+ result.clipboard_cleanup = claim.map { clipboard.literalInsertChangeCount == $0.changeCount } -+ == true ? "retained_unverified" : "preserved_newer_contents" -+ if result.error == nil { -+ result.error = .init(code: "INSERTION_UNVERIFIED", -+ message: "Cmd+V may still be pending. The previous clipboard was not restored.", -+ hint: "Inspect the exact receiver before another input; do not repeat the insertion blindly.") -+ } -+ } -+ result.clipboard_ownership = ClipboardPasteTransactionGate.hasUnresolvedPaste ? "reserved" : "released" -+ guard result.clipboard_changed else { -+ throw Self.literalInsertRefusal(result.error?.message ?? "Literal insertion was refused before clipboard mutation.") -+ } -+ return result -+ } -+ -+ private func validateLiteralInsertBaseline( -+ _ baseline: LiteralInsertState, target: UIAutomationTarget.ExactWindow, retained: RetainedFocusElement) async throws -+ { -+ do { -+ let focus = try await self.requireExactWindowKeyboardFocus( -+ expectedWindowIdentity: target.identity, expectedWindowBounds: target.bounds, -+ expectedFocusedElement: target.focusedElement) -+ guard focus.nativeElement == retained, -+ let current = try await self.literalInsertState(target, retained: retained), -+ current.matches(text: baseline.text, location: baseline.location, length: baseline.length) -+ else { throw Self.literalInsertRefusal("The exact text receiver or its selection changed before insertion.") } -+ } catch is CancellationError { throw CancellationError() } -+ catch let failure as DesktopActionFailure { throw failure } -+ catch { throw Self.literalInsertRefusal("The exact focused receiver could not be revalidated before insertion.") } -+ } -+ -+ private func literalInsertState( -+ _ target: UIAutomationTarget.ExactWindow, retained: RetainedFocusElement) async throws -> LiteralInsertState? -+ { -+ guard let expected = target.focusedElement else { return nil } -+ let identity = target.identity -+ return try await ElementDetectionTimeoutRunner.runDetached( -+ targetProcessIdentifier: identity.ownerProcessIdentifier, -+ targetProcessStartIdentity: identity.ownerProcessStartIdentity, seconds: 0.2) -+ { -+ LiteralInsertState.read(expected: expected, retained: retained, target: target) -+ } -+ } -+ -+ private static func literalInsertTargetIsActive(_ target: UIAutomationTarget.ExactWindow) -> Bool { -+ guard let application = NSWorkspace.shared.frontmostApplication, -+ application.processIdentifier == target.identity.ownerProcessIdentifier, -+ application.isActive, -+ SystemIdentityResolver.processStartIdentity(application.processIdentifier) -+ == target.identity.ownerProcessStartIdentity else { return false } -+ return true -+ } -+ -+ private static func literalInsertRefusal(_ message: String) -> DesktopActionFailure { -+ .preDispatchRefusal(reason: .targetUnavailable, message: message) -+ } -+} -+ -+private struct LiteralInsertState: Sendable { -+ let text: String -+ let location: Int -+ let length: Int -+ -+ func matches(text: String, location: Int, length: Int) -> Bool { -+ self.text.utf16.elementsEqual(text.utf16) && self.location == location && self.length == length -+ } -+ -+ static func read( -+ expected: FocusedElementIdentity, retained: RetainedFocusElement, -+ target: UIAutomationTarget.ExactWindow) -> Self? -+ { -+ guard SystemIdentityResolver.processStartIdentity(target.identity.ownerProcessIdentifier) == target.identity.ownerProcessStartIdentity, -+ case let .success(before) = DetachedExactWindowFocusReader.readValue(expected: expected, retainedElement: retained), -+ let text = before.value, text.utf16.count <= 65_536, -+ before.nativeElement == retained, target.bounds.contains(CGPoint(x: before.frame.midX, y: before.frame.midY)) -+ else { return nil } -+ let element = retained.element -+ AXUIElementSetMessagingTimeout(element, 0.05) -+ defer { AXUIElementSetMessagingTimeout(element, 0) } -+ var value: CFTypeRef? -+ guard AXUIElementCopyAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, &value) == .success, -+ let value, CFGetTypeID(value) == AXValueGetTypeID() else { return nil } -+ let selection = unsafeDowncast(value, to: AXValue.self) -+ var range = CFRange() -+ guard AXValueGetType(selection) == .cfRange, AXValueGetValue(selection, .cfRange, &range), -+ range.location >= 0, range.length >= 0, range.location <= text.utf16.count, -+ range.length <= text.utf16.count - range.location, -+ case let .success(after) = DetachedExactWindowFocusReader.readValue(expected: expected, retainedElement: retained), -+ after.nativeElement == retained, let current = after.value, current.utf16.elementsEqual(text.utf16), -+ target.bounds.contains(CGPoint(x: after.frame.midX, y: after.frame.midY)), -+ SystemIdentityResolver.processStartIdentity(target.identity.ownerProcessIdentifier) == target.identity.ownerProcessStartIdentity -+ else { return nil } -+ return Self(text: text, location: range.location, length: range.length) -+ } -+} -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeLiteralInsert.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeLiteralInsert.swift -new file mode 100644 ---- /dev/null -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeLiteralInsert.swift -@@ -0,0 +1,71 @@ -+import Foundation -+import PeekabooAutomationKit -+import PeekabooFoundation -+ -+public typealias PeekabooBridgeLiteralInsertResult = LiteralInsertResult -+ -+public struct PeekabooBridgeLiteralInsertRequest: Codable, Sendable { -+ public let snapshot: String -+ public let text: String -+ -+ public init(snapshot: String, text: String) { self.snapshot = snapshot; self.text = text } -+} -+ -+extension PeekabooBridgeClient { -+ public func literalInsert(snapshot: String, text: String) async throws -> PeekabooBridgeLiteralInsertResult { -+ guard case let .literalInsert(result) = try await self.send(.literalInsert(.init(snapshot: snapshot, text: text))) else { -+ throw PeekabooBridgeErrorEnvelope(code: .internalError, message: "Unexpected literal insertion response") -+ } -+ return result -+ } -+} -+ -+@MainActor -+extension PeekabooBridgeServer { -+ func handleLiteralInsert(_ request: PeekabooBridgeRequest) async throws -> PeekabooBridgeHandledResponse { -+ try await ClipboardPasteTransactionGate.withExclusiveTransaction { -+ try PeekabooBridgeRequestContext.checkRequestIsActive() -+ guard case let .literalInsert(payload) = request, -+ !payload.snapshot.isEmpty, payload.snapshot.utf16.count <= 256, -+ !payload.text.isEmpty, payload.text.utf16.count <= 8192, -+ try await self.services.snapshots.ownsSnapshot(snapshotId: payload.snapshot), -+ let detection = try await self.services.snapshots.getDetectionResult(snapshotId: payload.snapshot), -+ let context = detection.metadata.windowContext, -+ let identity = context.windowMutationIdentity, -+ let bounds = context.windowBounds, -+ context.windowID == identity.windowID, -+ context.applicationProcessId == identity.ownerProcessIdentifier, -+ let focused = context.focusedElement, -+ let service = self.services.automation as? any LiteralInsertServiceProtocol -+ else { -+ throw DesktopActionFailure.preDispatchRefusal(reason: .targetUnavailable, -+ message: "Inspect an exact window with a readable focused text control before inserting text.") -+ } -+ let lease = try await self.services.snapshots.beginSnapshotMutation(snapshotId: payload.snapshot) -+ var result: LiteralInsertResult -+ do { -+ result = try await service.literalInsert( -+ text: payload.text, target: .init(identity: identity, bounds: bounds, focusedElement: focused), -+ clipboard: self.literalInsertClipboard, -+ checkActive: { try PeekabooBridgeRequestContext.checkRequestIsActive() }) -+ } catch { -+ try? await self.services.snapshots.finishSnapshotMutation(lease, requiresFreshObservation: true) -+ throw error -+ } -+ do { -+ try await self.services.snapshots.finishSnapshotMutation(lease, -+ requiresFreshObservation: result.requires_fresh_observation) -+ } catch { -+ result.requires_fresh_observation = true -+ if result.error == nil { -+ result.error = .init(code: "SNAPSHOT_FINISH_FAILED", -+ message: "The insertion result is retained; inspect again before another input.") -+ } -+ } -+ // The signed global receipt binds this GUI-owned clipboard transaction and its nested exact target/result. -+ return .init(response: .literalInsert(result), mutation: .init( -+ outcome: .dispatchedUnverified(delivery: ClipboardMutationResultSemantics.delivery, -+ evidence: .deliveryAccepted, unitCount: .one), target: .global)) -+ } -+ } -+} -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -@@ -36,6 +36,7 @@ - - public enum PeekabooBridgeOperation: String, Codable, Sendable, CaseIterable, Hashable { - // Core -+ case literalInsert - case permissionsStatus - case requestPostEventPermission - case daemonStatus -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -@@ -29,6 +29,7 @@ - - /// Operations enabled by default for remote helper hosts. - public static let remoteDefaultAllowlist: Set = [ -+ .literalInsert, - .permissionsStatus, - .requestPostEventPermission, - .daemonStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -@@ -60,6 +60,10 @@ - } - - return switch operation { -+ case .literalInsert: -+ descriptor(ownership: .service, requiredPermissions: [.accessibility, .postEvent], -+ completion: .dispatchedUnverified(clipboardForeground), -+ targetPolicy: .global, responseFamilies: [.literalInsert]) - case .permissionsStatus: - descriptor( - completion: .readOnly, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseFamily.swift -@@ -3,7 +3,8 @@ - extension PeekabooBridgeOperationResultSemantics.ResponseFamily { - func matches(_ response: PeekabooBridgeResponse) -> Bool { - switch (self, response) { -- case (.agentExecutionTrace, .agentExecutionTrace), -+ case (.literalInsert, .literalInsert), -+ (.agentExecutionTrace, .agentExecutionTrace), - (.application, .application), - (.applicationMutationInventory, .applicationMutationInventory), - (.applications, .applications), -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResponseTargetEvidence.swift -@@ -123,7 +123,8 @@ - bounds: termination.receipt.windowBounds)] - case let .error(envelope): - return envelope.actionTargetReceipt.map { [DesktopTargetEvidenceAdapter.evidence(receipt: $0)] } ?? [] -- case .operationSessionRollover, -+ case .literalInsert, -+ .operationSessionRollover, - .handshake, - .permissionsStatus, - .daemonStatus, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultContracts.swift -@@ -54,6 +54,7 @@ - } - - enum ResponseFamily: Hashable, Sendable { -+ case literalInsert - case agentExecutionTrace - case application - case applicationMutationInventory -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -@@ -446,7 +446,8 @@ - .performAction(target: payload.target, actionName: payload.actionName) - case let .selectText(payload): - .selectText(target: payload.target, request: payload.request) -- case .attestedOperation, -+ case .literalInsert, -+ .attestedOperation, - .projectedAction, - .handshake, - .permissionsStatus, -@@ -596,7 +597,7 @@ - .suspectedNoop, - ] - switch request.operation { -- case .agentExecutionTrace: -+ case .literalInsert, .agentExecutionTrace: - return [.dispatchedUnverified] - case .requestPostEventPermission, .browserExecute, .swipe, .drag, .moveMouse, - .clickMenuItem, .clickMenuItemByName, .clickMenuExtra, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequest+DesktopMutation.swift -@@ -247,7 +247,7 @@ - /// never own the desktop lane or mutation watermark: its child re-enters this same Bridge and - /// each nested tool call owns its own exact-target lane and signed receipt. - var bypassesOuterDesktopMutationLane: Bool { -- self.unwrappedOperationRequest.operation == .agentExecutionTrace -+ [.agentExecutionTrace, .literalInsert].contains(self.unwrappedOperationRequest.operation) - } - } - -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -@@ -6,6 +6,7 @@ - indirect case attestedOperation(PeekabooBridgeAttestedOperationRequest) - indirect case projectedAction(PeekabooBridgeProjectedActionRequest) - case handshake(PeekabooBridgeHandshake) -+ case literalInsert(PeekabooBridgeLiteralInsertRequest) - case permissionsStatus - case requestPostEventPermission - case daemonStatus -@@ -134,6 +135,7 @@ - case let .attestedOperation(payload): payload.request.operation - case let .projectedAction(payload): payload.request.operation - case .handshake: .permissionsStatus -+ case .literalInsert: .literalInsert - case .permissionsStatus: .permissionsStatus - case .requestPostEventPermission: .requestPostEventPermission - case .daemonStatus: .daemonStatus -@@ -262,6 +264,7 @@ - case operationSessionRollover(PeekabooBridgeOperationSessionRefusal) - indirect case projectedAction(PeekabooBridgeProjectedActionResponse) - case handshake(PeekabooBridgeHandshakeResponse) -+ case literalInsert(PeekabooBridgeLiteralInsertResult) - case permissionsStatus(PermissionsStatus) - case daemonStatus(PeekabooDaemonStatus) - case agentExecutionTrace(PeekabooBridgeAgentExecutionTraceResponse) -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -@@ -12,6 +12,8 @@ - permissions: PermissionsStatus) async throws -> PeekabooBridgeHandledResponse - { - switch request.operation { -+ case .literalInsert: -+ return try await self.handleLiteralInsert(request) - case .permissionsStatus, .daemonStatus, .daemonStop: - return try await .init( - response: self.handleCoreRequest(request, peer: peer, permissions: permissions)) -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer.swift ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer.swift -@@ -105,6 +105,7 @@ - } - } - -+ let literalInsertClipboard = ClipboardService() - let services: any PeekabooBridgeServiceProviding - let hostKind: PeekabooBridgeHostKind - let allowlistedTeams: Set diff --git a/apps/desktop/native/patches/peekaboo-launch.patch b/apps/desktop/native/patches/peekaboo-launch.patch deleted file mode 100644 index 0d50cf8..0000000 --- a/apps/desktop/native/patches/peekaboo-launch.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+ActionOutcomes.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+ActionOutcomes.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+ActionOutcomes.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+ActionOutcomes.swift -@@ -10,7 +10,22 @@ - { - let access: DesktopOperationAccess = request.activates ? .write : .read - return try await self.operationLaneCoordinator.run(scope: .global, access: access) { -- let preparedLaunch = try self.prepareApplicationLaunch(request) -+ let preparedLaunch: PreparedApplicationLaunch -+ do { -+ preparedLaunch = try self.prepareApplicationLaunch(request) -+ } catch { -+ // Selector preparation is synchronous and has not opened or activated an application. -+ let reason: DesktopActionOutcome.RefusalReason -+ if let nativeError = error as? PeekabooError, case .invalidInput = nativeError { -+ reason = .invalidRequest -+ } else { -+ reason = .targetUnavailable -+ } -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: reason, -+ message: error.localizedDescription, -+ hint: "Verify the application selector or launch options before choosing a new action.") -+ } - return try await self.performApplicationLaunchWithOutcomeOwnedLane(preparedLaunch) - } - } diff --git a/apps/desktop/native/patches/peekaboo-menu.patch b/apps/desktop/native/patches/peekaboo-menu.patch deleted file mode 100644 index c1e9188..0000000 --- a/apps/desktop/native/patches/peekaboo-menu.patch +++ /dev/null @@ -1,454 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/MenuCommand.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/MenuCommand.swift -new file mode 100644 -index 000000000..094ee51ee ---- /dev/null -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/MenuCommand.swift -@@ -0,0 +1,285 @@ -+import ApplicationServices -+import Foundation -+import os -+import PeekabooFoundation -+ -+public struct MenuCommandRequest: Codable, Sendable { -+ public let expectedIdentity: ApplicationProcessIdentity -+ public let path: [String] -+ -+ public init(expectedIdentity: ApplicationProcessIdentity, path: [String]) throws { -+ guard !path.isEmpty, path.count <= 8, -+ path.allSatisfy({ !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && $0.utf16.count <= 512 }) -+ else { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .invalidRequest, -+ message: "Menu commands require 1 to 8 nonblank literal titles of at most 512 UTF-16 units each.") -+ } -+ self.expectedIdentity = expectedIdentity -+ self.path = path -+ } -+ -+ private enum CodingKeys: String, CodingKey { case expectedIdentity, path } -+ -+ public init(from decoder: any Decoder) throws { -+ let container = try decoder.container(keyedBy: CodingKeys.self) -+ try self.init(expectedIdentity: container.decode(ApplicationProcessIdentity.self, forKey: .expectedIdentity), -+ path: container.decode([String].self, forKey: .path)) -+ } -+} -+ -+@MainActor -+public protocol MenuCommandProviding: Sendable { -+ func menuCommand(_ request: MenuCommandRequest) async throws -> UIAutomationActionResult -+} -+ -+extension MenuService: MenuCommandProviding { -+ public func menuCommand(_ request: MenuCommandRequest) async throws -> UIAutomationActionResult { -+ try await self.withPinnedMenuFailureAttribution(processIdentity: request.expectedIdentity) { -+ try await self.operationLaneCoordinator.run(scope: .process(request.expectedIdentity), access: .write) { -+ // This command never reads the inventory cache; a later inventory should re-read changed menus. -+ defer { self.menuCache.removeValue(forKey: "PID:\(request.expectedIdentity.processIdentifier)") } -+ guard request.expectedIdentity.processIdentifier != ProcessInfo.processInfo.processIdentifier else { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .operationUnsupported, -+ message: "Menu commands targeting this Ace process are not supported. No menu command was dispatched.") -+ } -+ let outcome = try await MenuCommandAX.run(request) -+ return try UIAutomationActionResult( -+ payload: (), outcome: outcome, -+ targetIdentity: DesktopTargetIdentity(processIdentity: request.expectedIdentity)) -+ } -+ } -+ } -+} -+ -+private enum MenuCommandAX { -+ static let delivery = DesktopActionOutcome.Delivery(mechanism: .accessibilityAction, mode: .background) -+ -+ static func run(_ request: MenuCommandRequest) async throws -> DesktopActionOutcome { -+ let cancelled = OSAllocatedUnfairLock(initialState: false) -+ return try await withTaskCancellationHandler { -+ let result: Result = await withCheckedContinuation { continuation in -+ Thread.detachNewThread { -+ do { -+ continuation.resume(returning: .success(try self.perform(request, cancelled: cancelled))) -+ } catch let failure as DesktopActionFailure { -+ continuation.resume(returning: .failure(failure)) -+ } catch { -+ continuation.resume(returning: .failure(.preDispatchRefusal( -+ reason: .targetUnavailable, message: "The menu path could not be verified before input.", -+ causeDescription: error.localizedDescription))) -+ } -+ } -+ } -+ // Cancellation never abandons a native AX call or releases its process lane early. -+ return try result.get() -+ } onCancel: { -+ cancelled.withLock { $0 = true } -+ } -+ } -+ -+ private static func perform( -+ _ request: MenuCommandRequest, -+ cancelled: OSAllocatedUnfairLock) throws -> DesktopActionOutcome -+ { -+ let read = MenuCommandRead(request: request, cancelled: cancelled) -+ let original = try read.resolve() -+ let current = try read.resolve() -+ guard original.count == current.count, -+ zip(original, current).allSatisfy({ CFEqual($0.0, $0.1) }), -+ let leaf = original.last -+ else { throw read.refusal("The observed menu ancestry changed before input.") } -+ guard try read.role(leaf) == kAXMenuItemRole as String else { -+ throw read.refusal("Choose a menu command item, not a menu-bar entry.") -+ } -+ let children = try read.children(leaf, requireAvailable: true) -+ for child in children { -+ guard try read.role(child) != kAXMenuRole as String else { -+ throw read.refusal("Choose a menu command leaf, not a menu with a submenu.") -+ } -+ } -+ guard let enabled = try read.attribute(leaf, kAXEnabledAttribute as String), -+ CFGetTypeID(enabled) == CFBooleanGetTypeID(), CFEqual(enabled, kCFBooleanTrue) -+ else { throw read.refusal("The exact menu command is disabled or its enabled state is unavailable.") } -+ let actions: [String] = try read.withElement(leaf) { -+ var value: CFArray? -+ guard AXUIElementCopyActionNames(leaf, &value) == .success, -+ let names = value as? [String], names.count <= 32 -+ else { throw read.refusal("The menu command's supported actions are unavailable.") } -+ return names -+ } -+ guard actions.contains(kAXPressAction as String) else { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .operationUnsupported, message: "The exact menu command does not support AXPress.") -+ } -+ try read.check() -+ guard AXUIElementSetMessagingTimeout(leaf, 2) == .success else { -+ throw read.refusal("The native menu command timeout could not be set before input.") -+ } -+ defer { AXUIElementSetMessagingTimeout(leaf, 0) } -+ try read.checkOwner(leaf) -+ try read.check() -+ let result = AXUIElementPerformAction(leaf, kAXPressAction as CFString) -+ if result == .success { -+ guard !cancelled.withLock({ $0 }) else { -+ throw DesktopActionFailure.indeterminate( -+ delivery: delivery, evidence: .completionUnknown, unitCount: .one, -+ message: "The menu command was cancelled after AXPress was accepted.", -+ hint: "Observe the application; do not repeat the menu command automatically.") -+ } -+ return .dispatchedUnverified(delivery: delivery, evidence: .deliveryAccepted, unitCount: .one) -+ } -+ let error = ActionInputDriver.classify(result) -+ if ActionInputDriver.nativeMutationFailureMayHaveDispatched(error) { -+ throw DesktopActionFailure.indeterminate( -+ delivery: delivery, evidence: .completionUnknown, unitCount: .one, -+ message: "The menu command may have been delivered; native completion is unknown.", -+ hint: "A modal command may still be running. Observe the application before deciding what to do; do not blindly repeat it.", -+ causeDescription: String(describing: error)) -+ } -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: result == .actionUnsupported ? .operationUnsupported : .targetUnavailable, -+ message: "The exact menu command was refused by Accessibility.", -+ causeDescription: String(describing: error)) -+ } -+} -+ -+/// Worker-local raw AX state; no AXorcist or GUI-actor state crosses this boundary. -+private final class MenuCommandRead { -+ let request: MenuCommandRequest -+ let cancelled: OSAllocatedUnfairLock -+ let deadline = ContinuousClock.now.advanced(by: .seconds(5)) -+ let application: AXUIElement -+ var remaining = 500 -+ -+ init(request: MenuCommandRequest, cancelled: OSAllocatedUnfairLock) { -+ self.request = request -+ self.cancelled = cancelled -+ self.application = AXUIElementCreateApplication(request.expectedIdentity.processIdentifier) -+ } -+ -+ func refusal(_ message: String) -> DesktopActionFailure { -+ .preDispatchRefusal(reason: .targetUnavailable, message: message, -+ hint: "Read the current literal menu path; no menu command was dispatched.") -+ } -+ -+ func check() throws { -+ guard !self.cancelled.withLock({ $0 }) else { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .requestCancelled, message: "The menu command was cancelled before AXPress.") -+ } -+ guard ContinuousClock.now < self.deadline else { -+ throw self.refusal("The menu path could not be verified within its native read budget.") -+ } -+ guard SystemIdentityResolver.processStartIdentity(self.request.expectedIdentity.processIdentifier) == -+ self.request.expectedIdentity.processStartIdentity else { -+ throw self.refusal("The application process generation changed before the menu command.") -+ } -+ } -+ -+ func checkOwner(_ element: AXUIElement) throws { -+ var pid: pid_t = 0 -+ guard AXUIElementGetPid(element, &pid) == .success, -+ pid == self.request.expectedIdentity.processIdentifier else { -+ throw self.refusal("The menu element does not belong to the exact application.") -+ } -+ } -+ -+ func withElement(_ element: AXUIElement, _ operation: () throws -> T) throws -> T { -+ try self.check() -+ try self.checkOwner(element) -+ guard AXUIElementSetMessagingTimeout(element, 1) == .success else { -+ throw self.refusal("The menu element's native read timeout could not be set.") -+ } -+ defer { AXUIElementSetMessagingTimeout(element, 0) } -+ let result = try operation() -+ try self.check() -+ return result -+ } -+ -+ func attribute(_ element: AXUIElement, _ name: String, optional: Bool = false) throws -> CFTypeRef? { -+ try self.withElement(element) { -+ var value: CFTypeRef? -+ let result = AXUIElementCopyAttributeValue(element, name as CFString, &value) -+ if optional && (result == .noValue || result == .attributeUnsupported) { return nil } -+ guard result == .success, let value else { -+ throw self.refusal("A required menu attribute could not be read completely.") -+ } -+ return value -+ } -+ } -+ -+ func role(_ element: AXUIElement) throws -> String { -+ guard let role = try self.attribute(element, kAXRoleAttribute as String) as? String else { -+ throw self.refusal("A menu element's role could not be verified.") -+ } -+ return role -+ } -+ -+ func children(_ element: AXUIElement, requireAvailable: Bool = false) throws -> [AXUIElement] { -+ try self.withElement(element) { -+ var count: CFIndex = 0 -+ let status = AXUIElementGetAttributeValueCount(element, kAXChildrenAttribute as CFString, &count) -+ if !requireAvailable && (status == .noValue || status == .attributeUnsupported) { return [] } -+ guard status == .success, count >= 0, count <= self.remaining else { -+ throw self.refusal("The complete menu sibling set is unavailable or exceeds its budget " + -+ "(AX \(status.rawValue), count \(count), remaining \(self.remaining), leaf check \(requireAvailable)).") -+ } -+ if count == 0 { return [] } -+ var values: CFArray? -+ guard AXUIElementCopyAttributeValues(element, kAXChildrenAttribute as CFString, 0, count, &values) == .success, -+ let values, CFArrayGetCount(values) == count, -+ let children = values as? [AXUIElement] else { -+ throw self.refusal("The complete menu sibling set could not be read.") -+ } -+ var currentCount: CFIndex = 0 -+ guard AXUIElementGetAttributeValueCount(element, kAXChildrenAttribute as CFString, ¤tCount) == .success, -+ currentCount == count else { -+ throw self.refusal("The menu sibling set changed while it was being read.") -+ } -+ self.remaining -= count -+ return children -+ } -+ } -+ -+ func resolve() throws -> [AXUIElement] { -+ guard let value = try self.attribute(self.application, kAXMenuBarAttribute as String), -+ CFGetTypeID(value) == AXUIElementGetTypeID() else { -+ throw self.refusal("The exact application's menu bar is unavailable.") -+ } -+ var menu = unsafeDowncast(value, to: AXUIElement.self) -+ var chain: [AXUIElement] = [menu] -+ for (index, title) in self.request.path.enumerated() { -+ var matches: [AXUIElement] = [] -+ for child in try self.children(menu) { -+ let role = try self.role(child) -+ // Separators and known non-item roles cannot compete for this literal title. -+ guard role == (index == 0 ? kAXMenuBarItemRole as String : kAXMenuItemRole as String) else { continue } -+ let rawTitle = try self.attribute(child, kAXTitleAttribute as String, optional: true) -+ let rawValue = try rawTitle ?? self.attribute(child, kAXValueAttribute as String, optional: true) -+ guard let candidate = rawValue as? String else { -+ throw self.refusal("A candidate menu title is unavailable, so its uniqueness cannot be established.") -+ } -+ if candidate.utf16.elementsEqual(title.utf16) { matches.append(child) } -+ } -+ guard matches.count == 1, let item = matches.first else { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .invalidRequest, -+ message: matches.isEmpty ? "The literal menu path is unavailable without opening ancestor menus." : "The literal menu path is ambiguous.") -+ } -+ chain.append(item) -+ if index == self.request.path.count - 1 { return chain } -+ var submenus: [AXUIElement] = [] -+ for child in try self.children(item) where try self.role(child) == kAXMenuRole as String { -+ submenus.append(child) -+ } -+ guard submenus.count == 1, let submenu = submenus.first else { -+ throw self.refusal("The literal submenu is missing, ambiguous, or unavailable without opening it.") -+ } -+ chain.append(submenu) -+ menu = submenu -+ } -+ throw self.refusal("The literal menu path contains no command.") -+ } -+} -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeMenuCommand.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeMenuCommand.swift -new file mode 100644 -index 000000000..e1b60e98a ---- /dev/null -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeMenuCommand.swift -@@ -0,0 +1,12 @@ -+import Foundation -+import PeekabooAutomationKit -+ -+extension PeekabooBridgeClient { -+ public func menuCommand(_ request: MenuCommandRequest) async throws -> UIAutomationActionResult { -+ try await self.actionResult(for: .menuCommand(request), expectedResponse: "literal menu command", -+ requiresTargetIdentity: true) { response in -+ guard case .ok = response else { return nil } -+ return () -+ } -+ } -+} -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -index fae31e82f..d584091b6 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift -@@ -122,6 +122,7 @@ public enum PeekabooBridgeOperation: String, Codable, Sendable, CaseIterable, Ha - // Menus - case listMenus - case listFrontmostMenus -+ case menuCommand - case clickMenuItem - case clickMenuItemByName - // Menu bar extras -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -index 0e8c83c45..5ef927366 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift -@@ -103,6 +103,7 @@ extension PeekabooBridgeOperation { - .showAllApplications, - .listMenus, - .listFrontmostMenus, -+ .menuCommand, - .clickMenuItem, - .clickMenuItemByName, - .listMenuExtras, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -index 706717cc4..5aaf74809 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift -@@ -511,7 +511,7 @@ extension PeekabooBridgeOperationResultSemantics { - completion: .readOnly, - targetPolicy: .notApplicable, - responseFamilies: [.menuStructure]) -- case .clickMenuItem, .clickMenuItemByName: -+ case .menuCommand, .clickMenuItem, .clickMenuItemByName: - descriptor( - ownership: .service, - requiredPermissions: [.accessibility], -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationReceipts.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationReceipts.swift -index ba6bfecd0..cde5d92da 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationReceipts.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationReceipts.swift -@@ -1344,6 +1344,9 @@ extension PeekabooBridgeRequest { - } else { - [] - } -+ case let .menuCommand(payload): -+ [.init(processIdentifier: payload.expectedIdentity.processIdentifier, -+ processIdentity: payload.expectedIdentity)] - case let .clickMenuItem(payload): - payload.expectedIdentity.map { - [.init(processIdentifier: $0.processIdentifier, processIdentity: $0)] -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -index 664dc3ba0..967470f30 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift -@@ -320,6 +320,8 @@ extension PeekabooBridgeOperationResultSemantics { - } else { - .global - } -+ case let .menuCommand(payload): -+ .process(payload.expectedIdentity) - case let .clickMenuItem(payload): - payload.expectedIdentity.map(DesktopOperationScope.process) ?? .global - case let .clickMenuItemByName(payload): -@@ -516,6 +518,7 @@ extension PeekabooBridgeOperationResultSemantics { - .showAllApplications, - .listMenus, - .listFrontmostMenus, -+ .menuCommand, - .clickMenuItem, - .clickMenuItemByName, - .listMenuExtras, -@@ -600,7 +603,7 @@ extension PeekabooBridgeOperationResultSemantics { - case .clipboardTextWrite, .clipboardImageWrite, .literalInsert, .agentExecutionTrace: - return [.dispatchedUnverified] - case .requestPostEventPermission, .browserExecute, .swipe, .drag, .moveMouse, -- .clickMenuItem, .clickMenuItemByName, .clickMenuExtra, -+ .menuCommand, .clickMenuItem, .clickMenuItemByName, .clickMenuExtra, - .clickMenuBarItemNamed, .clickMenuBarItemIndex, - .launchDockItem, .rightClickDockItem, - .detectElements, .inspectAccessibilityTree, -@@ -927,6 +930,8 @@ extension PeekabooBridgeOperationResultSemantics { - return [rule(nativeBackground, .exact(1)), rule(axBackground, .exact(1))] - case .hideOtherApplications, .showAllApplications: - return [rule(axBackground, .variable), rule(nativeBackground, .variable)] -+ case .menuCommand: -+ return [rule(axBackground, .exact(1))] - case let .clickMenuItem(payload): - return [rule( - payload.deliveryMode == .background ? axBackground : axForeground, -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -index d7aaedc83..69ceb0de8 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift -@@ -88,6 +88,7 @@ public enum PeekabooBridgeRequest: Codable, Sendable { - case showAllApplications - case listMenus(PeekabooBridgeMenuListRequest) - case listFrontmostMenus -+ case menuCommand(MenuCommandRequest) - case clickMenuItem(PeekabooBridgeMenuClickRequest) - case clickMenuItemByName(PeekabooBridgeMenuClickByNameRequest) - case listMenuExtras -@@ -221,6 +222,7 @@ extension PeekabooBridgeRequest { - case .showAllApplications: .showAllApplications - case .listMenus: .listMenus - case .listFrontmostMenus: .listFrontmostMenus -+ case .menuCommand: .menuCommand - case .clickMenuItem: .clickMenuItem - case .clickMenuItemByName: .clickMenuItemByName - case .listMenuExtras: .listMenuExtras -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -index 08c92e719..105c9f43b 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift -@@ -115,7 +115,7 @@ extension PeekabooBridgeServer { - .activateApplication, .quitApplication, - .hideApplication, .unhideApplication, .hideOtherApplications, .showAllApplications: - return try await self.handleApplicationRequest(request) -- case .listMenus, .listFrontmostMenus, .clickMenuItem, .clickMenuItemByName, .listMenuExtras, -+ case .listMenus, .listFrontmostMenus, .menuCommand, .clickMenuItem, .clickMenuItemByName, .listMenuExtras, - .clickMenuExtra, .menuExtraOpenMenuFrame, .listMenuBarItems, .clickMenuBarItemNamed, - .clickMenuBarItemIndex: - return try await self.handleMenuRequest(request) -diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+ServiceHandlers.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+ServiceHandlers.swift -index f894bfa7f..c56f4d521 100644 ---- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+ServiceHandlers.swift -+++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+ServiceHandlers.swift -@@ -380,6 +380,17 @@ extension PeekabooBridgeServer { - case .listFrontmostMenus: - let menus = try await self.services.menu.listFrontmostMenus() - return .init(response: .menuStructure(menus)) -+ case let .menuCommand(payload): -+ guard PeekabooBridgeRequestContext.usesAttestedOperationResultSemantics, -+ let service = self.services.menu as? any MenuCommandProviding else { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .runtimeIncompatible, -+ message: "The native runtime cannot attest exact literal menu commands.") -+ } -+ let result = try await service.menuCommand(payload) -+ return try Self.pinnedMenuMutationResponse( -+ .ok, result: result, expectedIdentity: payload.expectedIdentity, -+ expectedDeliveryMode: .background, operation: "literal menu command") - case let .clickMenuItem(payload): - guard PeekabooBridgeRequestContext.usesAttestedOperationResultSemantics else { - try await self.services.menu.clickMenuItem( diff --git a/apps/desktop/native/patches/peekaboo-open.patch b/apps/desktop/native/patches/peekaboo-open.patch deleted file mode 100644 index 336e715..0000000 --- a/apps/desktop/native/patches/peekaboo-open.patch +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+Lifecycle.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+Lifecycle.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+Lifecycle.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+Lifecycle.swift -@@ -174,6 +174,15 @@ - "Launched application process generation changed before its receipt could be returned") - } - let boundApplication = try await self.bindSelectorResolution(application, launch: launch) -+ if !launch.openURLs.isEmpty { -+ // LaunchServices acceptance does not establish the receiving app's document or URL effect. -+ return DesktopActionResult( -+ payload: boundApplication, -+ outcome: .dispatchedUnverified( -+ delivery: Self.applicationDelivery(mode: .foreground), -+ evidence: .deliveryAccepted, -+ unitCount: DesktopActionOutcome.DispatchUnitCount(acceptedNativeDispatchCount) ?? .one)) -+ } - return DesktopActionResult( - payload: boundApplication, - outcome: .confirmedChange( diff --git a/apps/desktop/native/patches/peekaboo-point-focus.patch b/apps/desktop/native/patches/peekaboo-point-focus.patch deleted file mode 100644 index d02dcdd..0000000 --- a/apps/desktop/native/patches/peekaboo-point-focus.patch +++ /dev/null @@ -1,255 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/BackgroundInputDriver.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/BackgroundInputDriver.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/BackgroundInputDriver.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/BackgroundInputDriver.swift -@@ -81,8 +81,9 @@ - /// `candidates` is ordered: the hit-tested element first, then its descendants, then its - /// ancestors (see `hitTestCandidates`). The hit-test element is authoritative — macOS returned - /// it for this exact point — so it is never rejected on frame grounds; every other candidate -- /// must still contain the point. The first enabled candidate that supports the required action -- /// wins. SwiftUI hit-tests can land on a non-pressable container whose pressable target is a -+ /// must still contain the point. Editable text fields prefer verified keyboard focus even when -+ /// they advertise AXPress; otherwise the first enabled actionable candidate wins. -+ /// SwiftUI hit-tests can land on a non-pressable container whose pressable target is a - /// descendant, so descendants are searched before ancestors. Left clicks on text inputs (which - /// have no `AXPress`) fall back to focusing the element, mirroring `ActionInputDriver`. - @MainActor -@@ -93,12 +94,18 @@ - { - let spatiallyValid = self.spatiallyValidCandidates(candidates, at: point) - -+ // Preserve hit-test ordering: a button inside a field still owns its own point. -+ let prefersFocus: (any AutomationElementRepresenting) -> Bool = { -+ button == .left && ($0.role == "AXTextField" || $0.role == "AXTextArea") && $0.isFocusedSettable -+ } - let requiredAction = button == .right ? AXActionNames.kAXShowMenuAction : AXActionNames.kAXPressAction - if let actionable = spatiallyValid.first(where: { -- $0.isEnabled && -- $0.supportsAction(requiredAction) && -- (button == .right || !self.nonPressableContainerRoles.contains($0.role ?? "")) -+ $0.isEnabled && (prefersFocus($0) || -+ ($0.supportsAction(requiredAction) && -+ (button == .right || !self.nonPressableContainerRoles.contains($0.role ?? "")))) - }) { -+ // WebKit text fields may advertise AXPress without giving it keyboard focus. -+ if prefersFocus(actionable) { return (actionable, .focus) } - let role = actionable.role ?? "" - let frame = String(describing: actionable.frame) - self.logger.debug( -@@ -197,15 +204,62 @@ - } - } - -+ // The worker returns only an immutable retained CF handle; AXorcist remains on MainActor. -+ private struct HitTestElement: @unchecked Sendable { -+ let element: AXUIElement -+ } -+ - @MainActor - private static func hitTestCandidates( - at point: CGPoint, -- targetProcessIdentifier: pid_t) -> [any AutomationElementRepresenting] -- { -- guard let hit = Element.elementAtPoint(point, pid: targetProcessIdentifier) else { -- return [] -- } -- -+ targetProcessIdentifier: pid_t, -+ targetProcessStartIdentity: UInt64?) async throws -> [any AutomationElementRepresenting] -+ { -+ let retained: HitTestElement -+ do { -+ // A self-targeted AX hit test needs the app's main actor free to answer it. -+ // Only the read runs here; cancellation never permits later input from this lane. -+ retained = try await ElementDetectionTimeoutRunner.runDetached( -+ targetProcessIdentifier: targetProcessIdentifier, -+ targetProcessStartIdentity: targetProcessStartIdentity, -+ seconds: 2, -+ maximumPendingOperationCount: 1) -+ { -+ let application = AXUIElementCreateApplication(targetProcessIdentifier) -+ let timeoutError = AXUIElementSetMessagingTimeout(application, 1) -+ guard timeoutError == .success else { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: timeoutError == .apiDisabled ? .permissionDenied : .targetUnavailable, -+ message: "The background hit-test timeout could not be set (AX error \(timeoutError.rawValue)).") -+ } -+ var element: AXUIElement? -+ let error = AXUIElementCopyElementAtPosition( -+ application, Float(point.x), Float(point.y), &element) -+ guard error == .success else { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: error == .apiDisabled ? .permissionDenied : .targetUnavailable, -+ message: "The background Accessibility hit test failed before input (AX error \(error.rawValue)).", -+ hint: "Inspect the exact window again before choosing another action.") -+ } -+ guard let element else { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .targetUnavailable, -+ message: "The background Accessibility hit test returned no target before input.") -+ } -+ return HitTestElement(element: element) -+ } -+ } catch let failure as DesktopActionFailure { -+ throw failure -+ } catch { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .targetUnavailable, -+ message: "The background click point could not be read before input.", -+ hint: "Inspect the exact window again before choosing another action.", -+ causeDescription: error.localizedDescription) -+ } -+ try Task.checkCancellation() -+ -+ let hit = Element(retained.element) - var candidates: [any AutomationElementRepresenting] = [AutomationElement(hit)] - candidates.append(contentsOf: self.descendantsBreadthFirst(of: hit, maxVisited: 256, maxDepth: 8)) - -@@ -243,8 +297,12 @@ - private static func performDetachedAction( - _ actionName: String, - on element: any AutomationElementRepresenting, -- gracePeriod: TimeInterval) async throws -> DesktopActionOutcome -- { -+ gracePeriod: TimeInterval?, -+ beforeMutation: @MainActor () throws -> Void = {}) async throws -> DesktopActionOutcome -+ { -+ try Task.checkCancellation() -+ try beforeMutation() -+ try Task.checkCancellation() - guard let axElement = element.underlyingAXElement else { - try element.performAutomationAction(actionName) - return .dispatchedUnverified( -@@ -261,7 +319,19 @@ - } catch let error as DesktopActionFailure { - throw error - } catch { -- throw ActionInputDriver.classify(error) -+ let classified = ActionInputDriver.classify(error) -+ if actionName == AXActionNames.kAXPressAction, -+ ActionInputDriver.nativeMutationFailureMayHaveDispatched(classified) -+ { -+ throw DesktopActionFailure.indeterminate( -+ delivery: .init(mechanism: .accessibilityAction, mode: .background), -+ evidence: .completionUnknown, -+ unitCount: .one, -+ message: "Accessibility click returned without reliable completion evidence.", -+ hint: "Observe the exact target before deciding whether to retry this click.", -+ causeDescription: error.localizedDescription) -+ } -+ throw classified - } - } - -@@ -1398,7 +1468,8 @@ - static func performPositionalClickAction( - _ action: PositionalClickAction, - on element: any AutomationElementRepresenting, -- allowsAccessibilityValueDelivery: Bool = true) async throws -> DesktopActionOutcome -+ allowsAccessibilityValueDelivery: Bool = true, -+ beforeMutation: @MainActor () throws -> Void = {}) async throws -> DesktopActionOutcome - { - if !allowsAccessibilityValueDelivery, action == .select || action == .focus { - throw DesktopActionFailure.preDispatchRefusal( -@@ -1411,23 +1482,25 @@ - return try await self.performDetachedAction( - AXActionNames.kAXPressAction, - on: element, -- gracePeriod: DetachedAXActionRunner.pressGracePeriod) -+ gracePeriod: nil, -+ beforeMutation: beforeMutation) - case .showMenu: - return try await self.performDetachedAction( - AXActionNames.kAXShowMenuAction, - on: element, -- gracePeriod: DetachedAXActionRunner.showMenuGracePeriod) -+ gracePeriod: DetachedAXActionRunner.showMenuGracePeriod, -+ beforeMutation: beforeMutation) - case .select: -+ try Task.checkCancellation() -+ try beforeMutation() - try element.setAutomationSelected(true) - return .dispatchedUnverified( - delivery: .init(mechanism: .accessibilityValue, mode: .background), - evidence: .deliveryAccepted) - case .focus: -- try Task.checkCancellation() -- try element.setAutomationFocused(true) -- return .dispatchedUnverified( -- delivery: .init(mechanism: .accessibilityValue, mode: .background), -- evidence: .deliveryAccepted) -+ return try await ActionInputDriver().tryFocus( -+ element: element, -+ beforeMutation: beforeMutation).outcome - } - } - -@@ -1457,7 +1530,10 @@ - exactWindowID: targetWindowID, - candidates: self.mouseWindowRouteCandidates(exactWindowID: targetWindowID)) - -- let candidates = self.hitTestCandidates(at: point, targetProcessIdentifier: targetProcessIdentifier) -+ let candidates = try await self.hitTestCandidates( -+ at: point, -+ targetProcessIdentifier: targetProcessIdentifier, -+ targetProcessStartIdentity: exactWindow.identity.ownerProcessStartIdentity) - guard let element = self.positionalFocusTarget(inCandidates: candidates, at: point) else { - throw DesktopActionFailure.preDispatchRefusal( - reason: .operationUnsupported, -@@ -1488,7 +1564,19 @@ - "Exact-window pixel-focus receipt changed before the Accessibility focus write") - } - -- let outcome = try await self.performPositionalClickAction(.focus, on: element) -+ let outcome = try await self.performPositionalClickAction(.focus, on: element) { -+ guard exactWindowIdentityValidator(exactWindow.identity, exactWindow.bounds) else { -+ throw PeekabooError.snapshotStale( -+ "Exact-window pixel-focus receipt changed before the Accessibility focus write") -+ } -+ if let axElement = element.underlyingAXElement { -+ guard let windowID = CGWindowID(exactly: exactWindow.identity.windowID), -+ AXWindowIDResolver.owningWindowID(of: axElement) == windowID else { -+ throw PeekabooError.snapshotStale( -+ "Exact-window pixel-focus element changed windows before the Accessibility focus write") -+ } -+ } -+ } - guard element.focusedState == true, - let focusedElement = element.focusedElementIdentity - else { -@@ -1624,13 +1712,18 @@ - throw PeekabooError.permissionDeniedAccessibility - } - -- let candidates = self.hitTestCandidates(at: point, targetProcessIdentifier: targetProcessIdentifier) -+ let candidates = try await self.hitTestCandidates( -+ at: point, -+ targetProcessIdentifier: targetProcessIdentifier, -+ targetProcessStartIdentity: expectedWindowIdentity?.ownerProcessStartIdentity) - guard let resolved = Self.positionalClickTarget(inCandidates: candidates, at: point, button: button) else { - throw PeekabooError.serviceUnavailable( - Self.noActionableElementMessage(at: point, targetProcessIdentifier: targetProcessIdentifier)) - } -- if let targetWindowID { -- guard let expectedWindowIdentity, -+ let beforeMutation: @MainActor () throws -> Void = { -+ try Task.checkCancellation() -+ guard let targetWindowID, -+ let expectedWindowIdentity, - let expectedWindowBounds, - SystemIdentityResolver.validateWindowMutationIdentity( - expectedWindowIdentity, -@@ -1641,11 +1734,13 @@ - } - try self.assertBelongsToTargetWindow(resolved.element, targetWindowID: targetWindowID, at: point) - } -+ try beforeMutation() - - return try await self.performPositionalClickAction( - resolved.action, - on: resolved.element, -- allowsAccessibilityValueDelivery: allowsAccessibilityValueDelivery) -+ allowsAccessibilityValueDelivery: allowsAccessibilityValueDelivery, -+ beforeMutation: beforeMutation) - } - } - diff --git a/apps/desktop/native/patches/peekaboo-pointer-window.patch b/apps/desktop/native/patches/peekaboo-pointer-window.patch deleted file mode 100644 index 896bfc2..0000000 --- a/apps/desktop/native/patches/peekaboo-pointer-window.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/BackgroundInputDriver.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/BackgroundInputDriver.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/BackgroundInputDriver.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/BackgroundInputDriver.swift -@@ -178,10 +178,9 @@ - /// the click. - /// Rejects a resolved element that does not belong to the pinned `targetWindowID`. - /// -- /// `_AXUIElementGetWindow` (via `AXWindowResolver`) returns the element's containing window in -- /// the same CGWindowID namespace as the pinning selector. A `nil` window id means the element -- /// exposes no window (or the lookup failed); for a pinned exact-window click that is treated as -- /// a mismatch so the click is never delivered to an unverified window. -+ /// WebKit leaves may only identify their containing window through AXWindow. Resolve that -+ /// native link before comparing the same CGWindowID used by the pinning selector. Missing or -+ /// different window identities are still refused; bounds alone never authorize an AX action. - @MainActor - private static func assertBelongsToTargetWindow( - _ element: any AutomationElementRepresenting, -@@ -192,7 +191,7 @@ - // In-memory elements (tests) carry no AX identity and cannot be window-verified. - return - } -- guard AXWindowResolver().windowID(from: axElement) == targetWindowID else { -+ guard AXWindowIDResolver.owningWindowID(of: axElement) == targetWindowID else { - throw PeekabooError.serviceUnavailable( - Self.occludedWindowMessage(at: point, targetWindowID: targetWindowID)) - } -@@ -857,7 +856,7 @@ - let element = Element.elementAtPoint(point) - else { return nil } - let axElement = element.underlyingElement -- guard let windowID = AXWindowResolver().windowID(from: axElement) else { return nil } -+ guard let windowID = AXWindowIDResolver.owningWindowID(of: axElement) else { return nil } - var processIdentifier: pid_t = 0 - guard AXUIElementGetPid(axElement, &processIdentifier) == .success else { return nil } - return self.pointerReceiverIdentity( diff --git a/apps/desktop/native/patches/peekaboo-quit.patch b/apps/desktop/native/patches/peekaboo-quit.patch deleted file mode 100644 index 6208674..0000000 --- a/apps/desktop/native/patches/peekaboo-quit.patch +++ /dev/null @@ -1,41 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/Core/Protocols/ApplicationServiceProtocol.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/Core/Protocols/ApplicationServiceProtocol.swift -index a1c9852e9..a72ab102f 100644 ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/Core/Protocols/ApplicationServiceProtocol.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/Core/Protocols/ApplicationServiceProtocol.swift -@@ -242,6 +242,12 @@ public enum ApplicationActionResultSemantics { - hint: "Observe the pinned application before retrying and update the runtime host.") - .attributed(to: expectedIdentity.actionTargetReceipt) - } -+ // The bool records termination, not whether the quit request was accepted. -+ if !result.payload, outcome.state == .dispatchedUnverified, -+ outcome.evidence == .operationStillRunning -+ { -+ return -+ } - if !result.payload, outcome.isAccepted(by: .confirmedOrDispatched) { - throw DesktopActionFailure.indeterminate( - route: outcome.route, -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+ActionOutcomes.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+ActionOutcomes.swift -index de8b7d9b2..448f67d7e 100644 ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+ActionOutcomes.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+ActionOutcomes.swift -@@ -101,9 +101,16 @@ extension ApplicationService { - hint: "Refresh the application inventory before retrying.") - } - let delivery = Self.applicationDelivery(mode: .background) -- let outcome: DesktopActionOutcome = attempt.terminated -- ? .confirmedChange(delivery: delivery, unitCount: .one) -- : .suspectedNoop(delivery: delivery, unitCount: .one) -+ let outcome: DesktopActionOutcome -+ if !attempt.terminated, !request.force { -+ // An accepted quit can be waiting on unsaved work; repeating it is not safe. -+ outcome = .dispatchedUnverified( -+ delivery: delivery, evidence: .operationStillRunning, unitCount: .one) -+ } else { -+ outcome = attempt.terminated -+ ? .confirmedChange(delivery: delivery, unitCount: .one) -+ : .suspectedNoop(delivery: delivery, unitCount: .one) -+ } - return DesktopActionResult(payload: attempt.terminated, outcome: outcome) - } - } catch let failure as DesktopActionFailure { diff --git a/apps/desktop/native/patches/peekaboo-stale-click.patch b/apps/desktop/native/patches/peekaboo-stale-click.patch deleted file mode 100644 index dec878d..0000000 --- a/apps/desktop/native/patches/peekaboo-stale-click.patch +++ /dev/null @@ -1,23 +0,0 @@ -diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift ---- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift -+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift -@@ -1436,6 +1436,19 @@ - afterDispatch: false, - validateProcessIdentity: validatesProcessIdentity, - validateExactWindow: false) -+ // Preparation precedes every strategy route, so only this first exact-window check -+ // proves no input was sent; later route checks stay conservative. -+ if clickType != .longPress, -+ let exactWindowReceipt, -+ !self.exactWindowIdentityValidator(exactWindowReceipt.identity, exactWindowReceipt.bounds) -+ { -+ throw DesktopActionFailure.preDispatchRefusal( -+ reason: .targetUnavailable, -+ message: "Exact-window click identity changed before dispatch; capture a fresh snapshot", -+ hint: "Inspect the exact window again and use its fresh snapshot.", -+ standardErrorCode: .snapshotStale) -+ .attributed(to: exactWindowReceipt.actionTargetReceipt) -+ } - mutationReceipt = preparedReceipt - syntheticDestination = SyntheticClickDestination( - captureReceipt: preparedReceipt, diff --git a/apps/desktop/native/sources/desktop/project.swift b/apps/desktop/native/project.swift similarity index 91% rename from apps/desktop/native/sources/desktop/project.swift rename to apps/desktop/native/project.swift index f4994b8..d055e3b 100644 --- a/apps/desktop/native/sources/desktop/project.swift +++ b/apps/desktop/native/project.swift @@ -100,24 +100,31 @@ private final class ProjectPicker { } } -func openProjectPicker() { +@_cdecl("ace_project_open") +public func projectOpen() { project.open() } -func closeProjectPicker() { +@_cdecl("ace_project_close") +public func projectClose() { project.close() DispatchQueue.main.async { ProjectPicker.shared.cancel() } } -@_cdecl("ace_desktop_project_start") +@_cdecl("ace_project_start") public func projectStart(_ path: UnsafePointer) { let path = String(cString: path) guard project.begin() else { return } DispatchQueue.main.async { ProjectPicker.shared.start(path: path) } } -@_cdecl("ace_desktop_project_status") +@_cdecl("ace_project_status") public func projectStatus() -> UnsafeMutablePointer? { let data = try! JSONEncoder().encode(project.read()) return strdup(String(decoding: data, as: UTF8.self)) } + +@_cdecl("ace_project_free") +public func projectFree(_ value: UnsafeMutableRawPointer?) { + free(value) +} diff --git a/apps/desktop/native/sources/client/actions.swift b/apps/desktop/native/sources/client/actions.swift deleted file mode 100644 index ea05db4..0000000 --- a/apps/desktop/native/sources/client/actions.swift +++ /dev/null @@ -1,409 +0,0 @@ -import CoreGraphics -import Foundation -import PeekabooAutomationKit -import PeekabooBridge -import PeekabooFoundation - -private struct ActionRequest: Decodable { - enum Operation: String, Decodable { - case click, type, key, insert, select, scroll, drag - } - - let op: Operation - let snapshot: String - let element: String? - let text: String? - let key: String? - let modifiers: [String]? - let prefix: String? - let suffix: String? - let selection: String? - let point: PointerPoint? - let kind: String? - let direction: String? - let amount: Int? - let from: PointerPoint? - let to: PointerPoint? - let button: String? - let duration_ms: Int? - - func validate() throws { - guard !snapshot.isEmpty, snapshot.utf16.count <= 256 else { - throw ActionError("Use a snapshot ID returned by desktop_inspect.") - } - if op != .key, modifiers != nil { - throw ActionError("Only key presses accept modifiers.") - } - if op != .select, prefix != nil || suffix != nil || selection != nil { - throw ActionError("Only text selection accepts prefix, suffix, or selection.") - } - if op != .click, kind != nil { - throw ActionError("Only clicks accept a click kind.") - } - if op != .scroll, direction != nil || amount != nil { - throw ActionError("Only scrolling accepts direction or amount.") - } - if op != .drag, from != nil || to != nil || button != nil || duration_ms != nil { - throw ActionError("Only dragging accepts endpoints, button, or duration.") - } - if op != .click, op != .scroll, point != nil { - throw ActionError("Only clicks or scrolling accept a target point.") - } - switch op { - case .click, .scroll: - guard (element == nil) != (point == nil), text == nil, key == nil else { - throw ActionError("Choose exactly one observed element ID or normalized screenshot point.") - } - if let element, element.isEmpty || element.utf16.count > 256 { - throw ActionError("Choose a literal element ID from the inspected snapshot.") - } - try point?.validate() - if op == .click { - guard kind == nil || pointerClicks.contains(kind!) else { - throw ActionError("Choose single, double, right, middle, or triple click.") - } - } else { - guard let direction, ScrollDirection(rawValue: direction) != nil, - let amount, (1...20).contains(amount) - else { throw ActionError("Choose up, down, left, or right and 1 to 20 native scroll units.") } - } - case .drag: - guard element == nil, text == nil, key == nil, let from, let to, from != to else { - throw ActionError("Choose distinct normalized screenshot points for the drag endpoints.") - } - try from.validate() - try to.validate() - guard button == nil || ExactWindowHeldPointerButton(rawValue: button!) != nil, - ExactWindowDragRequest.durationMillisecondsRange.contains(duration_ms ?? 500) - else { throw ActionError("Choose a left or right drag lasting 1 to 10000 milliseconds.") } - case .type, .select: - guard let element, !element.isEmpty, element.utf16.count <= 256, key == nil else { - throw ActionError("Choose a literal element ID from the inspected snapshot.") - } - if op == .type { - guard let text, text.utf16.count <= 8192 else { - throw ActionError("Replacement text must contain at most 8,192 UTF-16 code units.") - } - } else if op == .select { - guard let text, !text.isEmpty, text.utf16.count <= 4096, - (prefix?.utf16.count ?? 0) <= 2048, (suffix?.utf16.count ?? 0) <= 2048 - else { - throw ActionError("Select nonempty text of at most 4,096 UTF-16 code units, with prefix and suffix of at most 2,048 each.") - } - if let selection, TextSelectionType(rawValue: selection) == nil { - throw ActionError("Choose text, cursor_before, or cursor_after selection.") - } - } - case .insert: - guard element == nil, key == nil, let text, !text.isEmpty, text.utf16.count <= 8192 else { - throw ActionError("Insert nonempty text of at most 8,192 UTF-16 code units into the observed focused control.") - } - case .key: - guard element == nil, text == nil, let key, keyboardKeys.contains(key) else { - throw ActionError("Choose a supported navigation key, letter, digit, or f1 through f12.") - } - let modifiers = modifiers ?? [] - guard modifiers.count <= 4, Set(modifiers).count == modifiers.count, - modifiers.allSatisfy({ keyboardModifiers.contains($0) }) - else { - throw ActionError("Use each of command, control, option, and shift at most once.") - } - } - } -} - -private struct PointerPoint: Decodable, Equatable { - let x: Double - let y: Double - - func validate() throws { - guard x.isFinite, y.isFinite, (0..<1).contains(x), (0..<1).contains(y) else { - throw ActionError("Screenshot point coordinates must be at least 0 and less than 1.") - } - } - - func mapped(in authority: SnapshotTargetReceipt.CoordinateAuthority) throws -> CGPoint { - let point = try CaptureCoordinateMapper.globalPoint( - for: CGPoint(x: x, y: y), in: .normalized, context: authority.context - ) - guard authority.target.bounds.contains(point) else { - throw ActionError("The screenshot point is outside its exact captured window.") - } - return point - } -} - -private let pointerClicks: Set = ["single", "double", "right", "middle", "triple"] -private let keyboardModifiers: Set = ["command", "control", "option", "shift"] -private let keyboardKeys: Set = { - var keys: Set = [ - "enter", "tab", "escape", "backspace", "delete", "up", "down", "left", "right", - "space", "home", "end", "pageup", "pagedown", - ] - keys.formUnion("abcdefghijklmnopqrstuvwxyz0123456789".map(String.init)) - keys.formUnion((1...12).map { "f\($0)" }) - return keys -}() - -private struct ActionError: LocalizedError { - let message: String - init(_ message: String) { self.message = message } - var errorDescription: String? { message } -} - -private struct ActionEvidence { - let outcome: DesktopActionOutcome? - let target: DesktopTargetIdentity? - let selected: [DesktopSelectedLeafEvidence]? - - init(_ result: UIAutomationActionResult) { - outcome = result.outcome - target = result.targetIdentity - selected = result.selectedLeafEvidence - } -} - -private struct ActionResult: Encodable { - var outcome: String - let action: String - let snapshot_id: String - var native_outcome: DesktopActionOutcome? - var selected_leaf_evidence: [DesktopSelectedLeafEvidence]? - var selection: TextSelectionResult? - var clipboard_changed: Bool? - var clipboard_cleanup: String? - var clipboard_ownership: String? - var consumption: String? - var requires_fresh_observation = false - var error: ActionMessage? -} - -private struct ActionMessage: Encodable { - let code: String - let message: String - var hint: String? - var cause: String? -} - -private struct ActionReply: Encodable { - let success = true - let data: ActionResult - let target_receipt: Receipt? -} - -func nativeAction(_ client: PeekabooBridgeClient) async throws -> Data { - let request = try readAction() - var result = ActionResult(outcome: "refused", action: request.op.rawValue, snapshot_id: request.snapshot) - var receipt: Receipt? - var lease: SnapshotMutationLease? - var invoked = false - do { - try request.validate() - let chord = try request.key.map { key in - // Ace keeps delete as forward delete; Peekaboo's unqualified delete means backspace. - let primary = key == "delete" ? "forwarddelete" : key - return try KeyboardChord(parsing: ((request.modifiers ?? []) + [primary]).joined(separator: "+")) - } - result.requires_fresh_observation = true - guard try await client.ownsSnapshot(snapshotId: request.snapshot) else { - throw ActionError("This observation no longer belongs to the running desktop. Inspect the window again.") - } - let detection = try await client.getDetectionResult(snapshotId: request.snapshot) - let (context, identity, bounds) = try snapshotWindow(detection) - receipt = Receipt( - pid: identity.ownerProcessIdentifier, - window_id: identity.windowID, - process_start_identity_decimal: String(identity.ownerProcessStartIdentity) - ) - if let element = request.element { - // Native set-value accepts text queries too; Ace accepts only a literal observed ID. - guard let observed = detection.elements.findById(element) else { - throw ActionError("The element ID is not in this snapshot. Inspect the window again.") - } - guard observed.knownIsEnabled != false else { throw ActionError("The observed element is disabled.") } - } - if request.op == .key || request.op == .insert, context.focusedElement == nil { - throw ActionError("The snapshot has no exact focused control. Click a control, then inspect the window again.") - } - if request.op == .insert { - // The GUI owns the snapshot lease and clipboard transaction through consumption and cleanup. - invoked = true - let insertion = try await client.literalInsert(snapshot: request.snapshot, text: request.text!) - result.outcome = insertion.outcome - result.native_outcome = insertion.native_outcome - result.clipboard_changed = insertion.clipboard_changed - result.clipboard_cleanup = insertion.clipboard_cleanup - result.clipboard_ownership = insertion.clipboard_ownership - result.consumption = insertion.consumption - result.requires_fresh_observation = insertion.requires_fresh_observation - if let error = insertion.error { - result.error = ActionMessage(code: error.code, message: error.message, hint: error.hint, cause: error.cause) - } - guard let target = insertion.target_receipt, - target.pid == identity.ownerProcessIdentifier, target.window_id == identity.windowID, - target.process_start_identity_decimal == String(identity.ownerProcessStartIdentity) - else { - throw ActionError("Literal insertion returned without its expected exact-window receipt. Inspect before retrying.") - } - return try JSONEncoder().encode(ActionReply(data: result, target_receipt: receipt)) - } - let window = try UIAutomationTarget.ExactWindow(identity: identity, bounds: bounds) - var scrollWindow: UIAutomationTarget.ExactWindow? - if request.op == .scroll { - // Request-pinned scroll receipts retain the snapshot's focus evidence as well as its geometry. - scrollWindow = try .init(identity: identity, bounds: bounds, focusedElement: context.focusedElement) - } - var point: CGPoint? - var drag: ExactWindowDragRequest? - if request.point != nil || request.op == .drag { - // Normalized coordinates survive host image resizing; authority stays in the bridge's capture. - let authority = try coordinateAuthority(request.snapshot, detection, window: window) - point = try request.point?.mapped(in: authority) - if request.op == .drag { - // Drag validates the full capture receipt; coordinate authority intentionally omits focus. - let dragWindow = try UIAutomationTarget.ExactWindow( - identity: identity, bounds: bounds, focusedElement: context.focusedElement - ) - drag = try ExactWindowDragRequest( - snapshotID: request.snapshot, target: dragWindow, - from: request.from!.mapped(in: authority), to: request.to!.mapped(in: authority), - durationMilliseconds: request.duration_ms ?? 500, - button: request.button.flatMap(ExactWindowHeldPointerButton.init(rawValue:)) ?? .left - ) - try drag!.validate() - } - } - lease = try await client.beginSnapshotMutation(snapshotId: request.snapshot) - invoked = true - let evidence: ActionEvidence - switch request.op { - case .click: - evidence = try await ActionEvidence(client.clickWithOutcome( - target: point.map(ClickTarget.coordinates) ?? .elementId(request.element!), - clickType: request.kind.flatMap(ClickType.init(rawValue:)) ?? .single, - snapshotId: request.snapshot, - windowEvidence: .init(identity: identity, bounds: bounds), - allowsAccessibilityValueDelivery: true - )) - case .scroll: - evidence = try await ActionEvidence(client.scrollWithOutcome(.init( - direction: ScrollDirection(rawValue: request.direction!)!, amount: request.amount!, - target: request.element, point: point, snapshotId: request.snapshot, - expectedWindow: scrollWindow!, foreground: false - ))) - case .drag: - evidence = try await ActionEvidence(client.dragExactWindow(drag!)) - case .type: - evidence = try await ActionEvidence(client.setValueWithOutcome( - target: request.element!, value: .string(request.text!), snapshotId: request.snapshot - )) - case .key: - // Text-action emulation rejects WKWebView's focused receiver; deliver the exact-window key instead. - evidence = try await ActionEvidence(client.hotkeyWithOutcome( - keys: chord!.serviceKeys, - holdDuration: 0, - target: .init( - windowIdentity: identity, - windowBounds: bounds, - focusedElement: context.focusedElement! - ) - )) - case .insert: - throw ActionError("Literal insertion must use its GUI-owned transaction.") - case .select: - let selected = try await client.selectText( - target: request.element!, - request: .init( - text: request.text!, prefix: request.prefix, suffix: request.suffix, - selectionType: request.selection.flatMap(TextSelectionType.init(rawValue:)) ?? .text - ), - snapshotId: request.snapshot - ) - evidence = ActionEvidence(selected) - result.selection = selected.payload.textSelection - } - result.native_outcome = evidence.outcome - result.selected_leaf_evidence = evidence.selected - guard let outcome = evidence.outcome, - let target = evidence.target?.exactWindow, - target.identity.hasSameStableReceipt(as: identity), target.bounds == bounds - else { - throw ActionError("The action returned without its expected outcome and exact-window receipt. Inspect before retrying.") - } - // Every dispatched action consumes this snapshot, including confirmed changes and partial cleanup. - result.requires_fresh_observation = outcome.dispatchState.mutationDispatched - try await client.finishSnapshotMutation(lease!, requiresFreshObservation: result.requires_fresh_observation) - lease = nil - switch outcome.state { - case .refused: - result.outcome = "refused" - case .indeterminate, .partial: - result.outcome = "unknown" - default: - result.outcome = outcome.evidence == .operationStillRunning ? "unknown" : "completed" - } - } catch let failure as DesktopActionFailure { - result.outcome = failure.outcome.dispatchState.mutationDispatched ? "unknown" : "refused" - result.native_outcome = failure.outcome - result.selected_leaf_evidence = failure.selectedLeafEvidence - let dispatched = failure.outcome.dispatchState.mutationDispatched - result.requires_fresh_observation = dispatched || failure.outcome.escalation == .refreshTarget - result.error = ActionMessage( - code: failure.standardErrorCode?.rawValue ?? "DESKTOP_ACTION_FAILED", - message: failure.message, hint: failure.hint - ) - if let lease { - do { - try await client.finishSnapshotMutation(lease, requiresFreshObservation: dispatched) - } catch { - result.requires_fresh_observation = true - } - } - } catch { - result.outcome = invoked ? "unknown" : "refused" - if invoked { result.requires_fresh_observation = true } - let envelope = error as? PeekabooBridgeErrorEnvelope - result.error = ActionMessage( - code: envelope?.code.rawValue ?? "DESKTOP_ACTION_FAILED", - message: error.localizedDescription, - hint: invoked ? "Inspect the target before retrying; the action may already have happened." : nil - ) - // Unknown completion leaves the host's pending lease in place, including client death or response loss. - } - return try JSONEncoder().encode(ActionReply(data: result, target_receipt: receipt)) -} - -func snapshotWindow( - _ detection: ElementDetectionResult -) throws -> (WindowContext, WindowMutationIdentity, CGRect) { - guard let context = detection.metadata.windowContext, - let identity = context.windowMutationIdentity, - let bounds = context.windowBounds, - context.windowID == identity.windowID, - context.applicationProcessId == identity.ownerProcessIdentifier - else { - throw ActionError("This observation has no exact-window action target. Inspect the window again.") - } - return (context, identity, bounds) -} - -func coordinateAuthority( - _ snapshot: String, _ detection: ElementDetectionResult, window: UIAutomationTarget.ExactWindow -) throws -> SnapshotTargetReceipt.CoordinateAuthority { - let authority = try SnapshotTargetReceiptPlanner.assemble( - snapshotID: snapshot, detectionResult: detection - ).receipt.requireCoordinateAuthority() - guard authority.target == window, let captured = authority.context.logicalBounds, - window.bounds.contains(captured), !detection.screenshotPath.isEmpty - else { throw ActionError("This observation has no pixel-backed coordinate authority for its exact window.") } - return authority -} - -private func readAction() throws -> ActionRequest { - var data = Data() - while let chunk = try FileHandle.standardInput.read(upToCount: 4096), !chunk.isEmpty { - data.append(chunk) - guard data.count <= 65_536 else { throw ActionError("The desktop action request exceeds 64 KiB.") } - } - return try JSONDecoder().decode(ActionRequest.self, from: data) -} diff --git a/apps/desktop/native/sources/client/client.swift b/apps/desktop/native/sources/client/client.swift deleted file mode 100644 index 3df2a86..0000000 --- a/apps/desktop/native/sources/client/client.swift +++ /dev/null @@ -1,368 +0,0 @@ -import AceSigning -import Darwin -import Foundation -import PeekabooAutomationKit -import PeekabooBridge - -@main -private enum Client { - static func main() async { - do { - write(try await execute(Array(CommandLine.arguments.dropFirst()))) - } catch let error as PeekabooBridgeErrorEnvelope { - write(try! JSONEncoder().encode(Failure(error: error))) - exit(1) - } catch { - write(try! JSONEncoder().encode(Failure(error: Message( - code: "DESKTOP_ERROR", message: error.localizedDescription - )))) - exit(1) - } - } - - private static func execute(_ args: [String]) async throws -> Data { - guard args.count >= 2 else { throw ClientError.usage } - let operation = args[1] - guard (operation == "apps" && args.count == 2) - || (operation == "windows" && args.count == 3) - || (operation == "inspect" && (args.count == 5 || args.count == 6)) - || (operation == "action" && args.count == 2) - || (operation == "management" && args.count == 2) - || (operation == "launch" && args.count == 2) - || (operation == "open" && args.count == 2) - || (operation == "clipboard" && args.count == 2) - || (operation == "menus" && args.count == 2) - || (operation == "menu" && args.count == 2) - else { throw ClientError.usage } - let identity = try SigningIdentity.current() - let client = PeekabooBridgeClient( - socketPath: args[0], - maxResponseBytes: 48 * 1024 * 1024, - requestTimeoutSec: 25, - trustedHostTeamIDs: [identity.team] - ) - let handshake = try await client.handshake( - client: .init( - bundleIdentifier: identity.identifier, - teamIdentifier: identity.team, - processIdentifier: getpid() - ), - requestedHost: .gui, - overallTimeoutSec: 5 - ) - switch operation { - case "action": - return try await nativeAction(client) - case "clipboard": - return try await nativeClipboard(client) - case "menu": - return try await nativeMenuCommand(client, handshake: handshake) - case "menus": - return try await nativeMenus(client) - case "management": - return try await nativeManagement(client, handshake: handshake) - case "launch": - return try await nativeLaunch(client, handshake: handshake) - case "open": - return try await nativeLaunch(client, handshake: handshake, opensItem: true) - case "apps": - let inventory = try await client.listApplicationMutationInventory() - var metadata: [ServiceApplicationInfo] = [] - var warnings: [String] = [] - do { - metadata = try await client.listApplications() - } catch { - try Task.checkCancellation() - warnings.append("Application presentation metadata was unavailable; activity and visibility are unknown.") - } - let grouped = Dictionary(grouping: metadata, by: \.processIdentifier) - var matched: [Int32: ServiceApplicationInfo] = [:] - for app in inventory.items { - guard let generation = app.processStartIdentity, generation > 0 else { continue } - let candidates = (grouped[app.processIdentifier] ?? []).filter { $0.processStartIdentity == generation } - if candidates.count == 1 { matched[app.processIdentifier] = candidates[0] } - } - let active = metadata.filter(\.isActive) - let activityKnown = active.count == 1 - && active[0].processStartIdentity != nil - && matched[active[0].processIdentifier]?.processStartIdentity == active[0].processStartIdentity - if !activityKnown { - warnings.append("No unique active application with matching process-generation identity was observed; activity is unknown.") - } - if matched.count < inventory.items.count { - warnings.append("Some applications lacked presentation metadata matching their process generation; their activity and visibility are unknown.") - } - return try encode(Apps( - apps: inventory.items.map { App($0, metadata: matched[$0.processIdentifier], activityKnown: activityKnown) }, - inventory_completeness: inventory.completeness.rawValue, - inventory_warnings: inventory.warnings, - metadata_warnings: warnings - )) - case "windows": - let pid = try processID(args[2]) - let inventory = try await client.listWindowMutationInventory(target: .application("PID:\(pid)")) - return try encode(Windows( - pid: pid, - windows: inventory.items.map { Window($0, pid: pid) }, - inventory_completeness: inventory.completeness.rawValue, - inventory_warnings: inventory.warnings - )) - default: - let pid = try processID(args[2]) - guard let window = UInt32(args[3]), window > 0 else { throw ClientError.usage } - guard args[4].hasPrefix("/") else { throw ClientError.usage } - guard let mode = InspectionMode(rawValue: args.count == 6 ? args[5] : "accessibility") - else { throw ClientError.usage } - return try await inspect(client, pid: pid, window: window, path: args[4], mode: mode) - } - } - - private static func inspect( - _ client: PeekabooBridgeClient, pid: Int32, window: UInt32, path: String, mode: InspectionMode - ) async throws -> Data { - guard mode == .pixels else { return try await observe(client, pid: pid, window: window, path: path, pixels: nil) } - // Screenshot-only results have no detection ID, so the request pins its publication explicitly. - let snapshot = try await client.createSnapshot() - do { - return try await observe(client, pid: pid, window: window, path: path, pixels: snapshot) - } catch { - try? await client.cleanSnapshot(snapshotId: snapshot) - throw error - } - } - - private static func observe( - _ client: PeekabooBridgeClient, pid: Int32, window: UInt32, path: String, pixels: String? - ) async throws -> Data { - let accessibility = pixels == nil - let observation = try await client.desktopObservationWithOutcome(.init( - target: .pid(pid, window: .id(window)), - capture: .init(scale: .logical1x, focus: .background), - detection: .init( - mode: accessibility ? .accessibility : .none, - traversalBudget: .init(maxDepth: 15, maxElementCount: 200, maxChildrenPerNode: 100), - requiresFreshAccessibilityTree: accessibility - ), - output: .init(path: path, saveSnapshot: true, snapshotID: pixels, includeImageData: true), - timeout: .init(overall: 20, detection: 15) - )) - guard let target = observation.targetIdentity, - target.processIdentity.processIdentifier == pid, - let exact = target.exactWindow, exact.identity.windowID == Int(window) - else { throw ClientError.target } - let result = observation.payload - if let pixels { - guard result.files.publishedSnapshotID == pixels, result.elements == nil else { throw ClientError.pixels } - // Expose the ID only after the stored canonical projection grants pointer authority for this exact capture. - let detection = try await client.getDetectionResult(snapshotId: pixels) - let (context, identity, bounds) = try snapshotWindow(detection) - let authority = try coordinateAuthority( - pixels, detection, window: .init(identity: identity, bounds: bounds) - ) - guard detection.snapshotId == pixels, detection.elements.all.isEmpty, context.focusedElement == nil, - identity.hasSameStableReceipt(as: exact.identity), bounds == exact.bounds, - authority.context == CaptureCoordinateContext(metadata: result.capture.metadata, referenceID: pixels) - else { throw ClientError.pixels } - } - let image = try result.verifiedCaptureImageData(requirement: .requireDigest) - guard !image.isEmpty, image.count <= 32 * 1024 * 1024 else { throw ClientError.image } - // Reusable snapshots keep Bridge-owned artifact copies; the host removes this caller-visible file after resizing. - try image.write(to: URL(fileURLWithPath: path), options: [.atomic]) - let data = Inspection( - inspection_mode: accessibility ? InspectionMode.accessibility.rawValue : InspectionMode.pixels.rawValue, - note: accessibility ? nil : "Pixel snapshot: no Accessibility elements or focused control. Its snapshot_id authorizes only screenshot-point clicks, point scrolls, and drags in this exact window; element, text, key, and insertion actions are refused.", - application_name: result.target.app?.name, - window_title: result.target.window?.title, - snapshot_id: result.files.publishedSnapshotID, - element_count: result.elements?.metadata.elementCount ?? 0, - ui_elements: result.elements?.elements.all.map(Element.init) ?? [], - coordinate_context: result.elements?.metadata.captureCoordinateContext - ?? CaptureCoordinateContext(metadata: result.capture.metadata), - capture_warning: result.capture.warning, - detection_metadata: result.elements?.metadata, - diagnostics: result.diagnostics, - timings: result.timings - ) - return try encode(data, receipt: .init( - pid: pid, - window_id: Int(window), - process_start_identity_decimal: String(target.processIdentity.processStartIdentity) - )) - } - - private static func processID(_ value: String) throws -> Int32 { - guard let pid = Int32(value), pid > 0 else { throw ClientError.usage } - return pid - } - - private static func encode(_ data: T, receipt: Receipt? = nil) throws -> Data { - try JSONEncoder().encode(Success(data: data, target_receipt: receipt)) - } - - private static func write(_ data: Data) { - FileHandle.standardOutput.write(data) - FileHandle.standardOutput.write(Data([10])) - } -} - -private struct Success: Encodable { - let success = true - let data: T - let target_receipt: Receipt? -} - -private struct Failure: Encodable { - let success = false - let error: T -} - -private struct Message: Encodable { - let code: String - let message: String -} - -struct Receipt: Encodable { - let pid: Int32 - let window_id: Int? - let process_start_identity_decimal: String -} - -private struct Apps: Encodable { - let apps: [App] - let inventory_completeness: String - let inventory_warnings: [String] - let metadata_warnings: [String] -} - -private struct App: Encodable { - let name: String - let pid: Int32 - let bundle_id: String? - let is_active: Bool? - let is_active_known: Bool - let is_hidden: Bool? - let is_hidden_known: Bool - let process_start_identity_decimal: String? - let warnings: [String]? - let target: ManagementTarget? - - init(_ app: ServiceApplicationInfo, metadata: ServiceApplicationInfo?, activityKnown: Bool) { - name = app.name - pid = app.processIdentifier - bundle_id = app.bundleIdentifier - // Mutation inventory supplies identity, not presentation state. Missing reads must stay unknown. - is_active_known = metadata != nil && activityKnown - is_active = is_active_known ? metadata?.isActive : nil - is_hidden_known = metadata?.isHiddenKnown == true - is_hidden = is_hidden_known ? metadata?.isHidden : nil - process_start_identity_decimal = app.processStartIdentity.map(String.init) - let combined = (app.metadataWarnings ?? []) + (metadata?.metadataWarnings ?? []) - warnings = combined.isEmpty ? nil : Array(Set(combined)).sorted() - target = app.processIdentity.map(ManagementTarget.init) - } -} - -private struct Windows: Encodable { - let pid: Int32 - let windows: [Window] - let inventory_completeness: String - let inventory_warnings: [String] -} - -private struct Window: Encodable { - let window_id: Int - let window_title: String - let bounds: Bounds - let is_on_screen: Bool - let is_minimized: Bool - let is_key: Bool? - let observation_capability: String? - let observation_reason: String? - let process_start_identity_decimal: String? - let target: ManagementTarget? - - init(_ window: ServiceWindowInfo, pid: Int32) { - window_id = window.windowID - window_title = window.title - bounds = Bounds(window.bounds) - is_on_screen = window.isOnScreen - is_minimized = window.isMinimized - is_key = window.isKeyWindow - observation_capability = window.observationCapability?.mode.rawValue - observation_reason = window.observationCapability?.reason?.rawValue - process_start_identity_decimal = window.mutationIdentity.map { String($0.processIdentity.processStartIdentity) } - target = ManagementTarget(window: window, pid: pid) - } -} - -struct Bounds: Codable { - let x: Double - let y: Double - let width: Double - let height: Double - - init(_ rectangle: CGRect) { - x = rectangle.origin.x - y = rectangle.origin.y - width = rectangle.width - height = rectangle.height - } -} - -private struct Inspection: Encodable { - let inspection_mode: String - let note: String? - let application_name: String? - let window_title: String? - let snapshot_id: String? - let element_count: Int - let ui_elements: [Element] - let coordinate_context: CaptureCoordinateContext - let capture_warning: String? - let detection_metadata: DetectionMetadata? - let diagnostics: DesktopObservationDiagnostics - let timings: ObservationTimings -} - -private struct Element: Encodable { - let id: String - let role: String - let label: String? - let value: String? - let bounds: Bounds - let is_enabled: Bool - let is_selected: Bool? - let attributes: [String: String] - - init(_ element: DetectedElement) { - id = element.id - role = element.type.rawValue - label = element.label - value = element.value - bounds = Bounds(element.bounds) - is_enabled = element.isEnabled - is_selected = element.isSelected - attributes = element.attributes - } -} - -private enum InspectionMode: String { - case accessibility, pixels -} - -private enum ClientError: LocalizedError { - case usage, target, image, pixels - - var errorDescription: String? { - switch self { - case .usage: - "Usage: ace-desktop-client apps | windows | menus < JSON | menu < JSON | inspect [accessibility|pixels] | action < JSON" - case .target: - "The native observation did not confirm the requested process and window. Refresh the window list and try again." - case .image: - "The native observation returned an empty or oversized screenshot." - case .pixels: - "The native pixel observation did not publish a screenshot-only snapshot with exact-window coordinate authority." - } - } -} diff --git a/apps/desktop/native/sources/client/clipboard.swift b/apps/desktop/native/sources/client/clipboard.swift deleted file mode 100644 index f7c05e7..0000000 --- a/apps/desktop/native/sources/client/clipboard.swift +++ /dev/null @@ -1,133 +0,0 @@ -import Foundation -import PeekabooAutomationKit -import PeekabooBridge -import PeekabooFoundation - -private struct ClipboardRequest: Decodable { - enum Operation: String, Decodable { - case read = "clipboard-read" - case write = "clipboard-write" - } - enum Format: String, Decodable { - case text, image, files - } - let op: Operation - let format: Format? - let text: String? - let image: Data? - let paths: [String]? -} - -private struct ClipboardReply: Encodable { - let success = true - let data: T -} - -private struct ClipboardWriteResult: Encodable { - var outcome = "refused" - var native_outcome: DesktopActionOutcome? - var clipboard_changed: Bool? - var source: ClipboardImageContents.Source? - var file_count: Int? - var error: ClipboardMessage? -} - -private struct ClipboardMessage: Encodable { - let code: String - let message: String - let hint: String? -} - -func nativeClipboard(_ client: PeekabooBridgeClient) async throws -> Data { - var data = Data() - while let chunk = try FileHandle.standardInput.read(upToCount: 4096), !chunk.isEmpty { - data.append(chunk) - guard data.count <= 14 * 1024 * 1024 else { - throw DesktopActionFailure.preDispatchRefusal(reason: .invalidRequest, - message: "The encoded clipboard request exceeds 14 MiB.") - } - } - let request = try JSONDecoder().decode(ClipboardRequest.self, from: data) - if request.op == .read { - guard request.text == nil, request.image == nil, request.paths == nil, data.count <= 65_536 else { - throw DesktopActionFailure.preDispatchRefusal(reason: .invalidRequest, - message: "Clipboard reads do not accept content and must fit 64 KiB.") - } - if request.format == .files { - let result = try await client.clipboardFilesRead() - return try JSONEncoder().encode(ClipboardReply(data: result)) - } - if request.format == .image { - let result = try await client.clipboardImageRead() - return try JSONEncoder().encode(ClipboardReply(data: result)) - } - let result = try await client.clipboardTextRead() - return try JSONEncoder().encode(ClipboardReply(data: result)) - } - var result = ClipboardWriteResult() - var invoked = false - do { - if request.format == .files { - guard request.text == nil, request.image == nil, let paths = request.paths, - data.count <= 65_536 else { - throw DesktopActionFailure.preDispatchRefusal(reason: .invalidRequest, - message: "File writes require paths without other content and must fit 64 KiB.") - } - invoked = true - let written = try await client.clipboardFilesWrite(paths) - result.outcome = written.outcome - result.native_outcome = written.native_outcome - result.clipboard_changed = written.clipboard_changed - result.file_count = written.file_count - if let error = written.error { - result.error = .init(code: error.code, message: error.message, hint: error.hint) - } - return try JSONEncoder().encode(ClipboardReply(data: result)) - } - if request.format == .image { - guard request.text == nil, request.paths == nil, let image = request.image, - !image.isEmpty, image.count <= 10 * 1024 * 1024 else { - throw DesktopActionFailure.preDispatchRefusal(reason: .invalidRequest, - message: "Image writes require at most 10 MiB of image data and no text.") - } - invoked = true - let written = try await client.clipboardImageWrite(image) - result.outcome = written.outcome - result.native_outcome = written.native_outcome - result.clipboard_changed = written.clipboard_changed - result.source = written.source - if let error = written.error { - result.error = .init(code: error.code, message: error.message, hint: error.hint) - } - return try JSONEncoder().encode(ClipboardReply(data: result)) - } - guard request.format == nil, request.image == nil, request.paths == nil, data.count <= 65_536 else { - throw DesktopActionFailure.preDispatchRefusal(reason: .invalidRequest, - message: "Supply plain text, format image with image data, or format files with paths.") - } - guard let text = request.text, text.utf16.count <= 8192 else { - throw DesktopActionFailure.preDispatchRefusal(reason: .invalidRequest, - message: "Clipboard text must contain at most 8192 UTF-16 code units.") - } - invoked = true - let written = try await client.clipboardTextWrite(text) - result.outcome = written.outcome - result.native_outcome = written.native_outcome - result.clipboard_changed = written.clipboard_changed - if let error = written.error { - result.error = .init(code: error.code, message: error.message, hint: error.hint) - } - } catch let failure as DesktopActionFailure { - let changed = failure.outcome.dispatchState.mutationDispatched - result.outcome = changed ? "unknown" : "refused" - result.native_outcome = failure.outcome - result.clipboard_changed = changed - result.error = .init(code: failure.standardErrorCode?.rawValue ?? "CLIPBOARD_WRITE_FAILED", - message: failure.message, hint: failure.hint) - } catch { - result.outcome = invoked ? "unknown" : "refused" - result.error = .init(code: "CLIPBOARD_WRITE_FAILED", message: error.localizedDescription, - hint: invoked ? "Read the clipboard before deciding whether to write again; the write may already have happened." : nil) - } - return try JSONEncoder().encode(ClipboardReply(data: result)) -} diff --git a/apps/desktop/native/sources/client/launch.swift b/apps/desktop/native/sources/client/launch.swift deleted file mode 100644 index 237ff58..0000000 --- a/apps/desktop/native/sources/client/launch.swift +++ /dev/null @@ -1,163 +0,0 @@ -import CoreGraphics -import Foundation -import PeekabooAutomationKit -import PeekabooBridge -import PeekabooFoundation - -private struct LaunchError: LocalizedError { - let message: String - init(_ message: String) { self.message = message } - var errorDescription: String? { message } -} - -private struct LaunchedApplication: Encodable { - let name: String - let bundle_id: String? - let path: String? - let target: ManagementTarget -} - -private struct LaunchMessage: Encodable { - let code: String - let message: String - let hint: String? -} - -private struct LaunchResult: Encodable { - let action: String - var outcome = "refused" - var native_outcome: DesktopActionOutcome? - var application: LaunchedApplication? - var requires_fresh_observation = false - var message: String? - var error: LaunchMessage? -} - -private struct LaunchReply: Encodable { - let success = true - let data: LaunchResult - let target_receipt: Receipt? -} - -func nativeLaunch(_ client: PeekabooBridgeClient, handshake: PeekabooBridgeHandshakeResponse, opensItem: Bool = false) async throws -> Data { - var result = LaunchResult(action: opensItem ? "open" : "launch") - var receipt: Receipt? - var invoked = false - let uncertain = opensItem - ? "The item may still open later. Observe desktop_apps before any further action; do not blindly repeat the open." - : "The app may still open later. Observe desktop_apps before any further action; do not blindly repeat the launch." - do { - let request = try readLaunch(opensItem: opensItem) - guard handshake.supportedOperations.contains(.launchApplicationWithOptions), - (handshake.enabledOperations ?? handshake.supportedOperations).contains(.launchApplicationWithOptions) - else { throw LaunchError("The desktop runtime does not support attested application launch. Update the runtime before acting.") } - let session = CGSessionCopyCurrentDictionary() as NSDictionary? - if session?["CGSSessionScreenIsLocked"] as? Bool == true { - throw DesktopActionFailure.preDispatchRefusal( - reason: .targetUnavailable, - message: "The macOS GUI session is locked. The opening operation was not dispatched.", - hint: "Unlock the active user session before choosing a new action." - ) - } - // The receiving process may not exist until LaunchServices returns it. - _ = try await client.listApplicationMutationInventory() - guard let preflight = await client.lastOperationReceipt(), preflight.payload.operation == PeekabooBridgeRequest.listApplicationMutationInventory.operation else { - throw LaunchError("The native runtime cannot attest application targets. Update the desktop runtime before acting.") - } - try Task.checkCancellation() - invoked = true - let action = try await client.launchApplicationResult(request: request) - result.native_outcome = action.outcome - guard let process = action.payload.processIdentity, - let signed = await client.lastOperationReceipt(), signed.payload.operation == .launchApplicationWithOptions, - let outcome = action.outcome, signed.payload.outcome?.outcome == outcome, - case let .process(identity) = signed.payload.target, identity == process - else { throw LaunchError("The launch returned without its matching signed process receipt and outcome.") } - // The bridge validates the requested selector against this signed application response. - receipt = Receipt(pid: process.processIdentifier, window_id: nil, process_start_identity_decimal: String(process.processStartIdentity)) - result.application = LaunchedApplication(name: action.payload.name, bundle_id: action.payload.bundleIdentifier, - path: action.payload.bundlePath, target: ManagementTarget(process)) - result.requires_fresh_observation = outcome.dispatchState.mutationDispatched - if opensItem { - guard outcome.state == .dispatchedUnverified, outcome.evidence == .deliveryAccepted else { - throw LaunchError("The native runtime did not distinguish accepted item delivery from its effect. Inspect the receiving app before acting again.") - } - result.outcome = "completed" - result.message = "macOS accepted the item opening request. Its effect is unverified; inspect the receiving app before any further action." - } else { - switch outcome.state { - case .confirmedChange, .confirmedNoChange: result.outcome = "completed" - case .refused: result.outcome = "refused" - default: result.outcome = "unknown" - } - } - } catch let failure as DesktopActionFailure { - let dispatched = failure.outcome.dispatchState.mutationDispatched - result.outcome = dispatched ? "unknown" : "refused" - result.native_outcome = failure.outcome - result.requires_fresh_observation = dispatched - result.error = LaunchMessage(code: failure.standardErrorCode?.rawValue ?? "DESKTOP_ACTION_FAILED", - message: failure.message, hint: dispatched ? uncertain : failure.hint) - } catch { - result.outcome = invoked ? "unknown" : "refused" - result.requires_fresh_observation = invoked - result.error = LaunchMessage(code: (error as? PeekabooBridgeErrorEnvelope)?.code.rawValue ?? "DESKTOP_ACTION_FAILED", - message: error.localizedDescription, hint: invoked ? uncertain : nil) - } - return try JSONEncoder().encode(LaunchReply(data: result, target_receipt: receipt)) -} - -private func readLaunch(opensItem: Bool) throws -> ApplicationLaunchRequest { - var data = Data() - while let chunk = try FileHandle.standardInput.read(upToCount: 4096), !chunk.isEmpty { - data.append(chunk) - guard data.count <= 16_384 else { throw LaunchError("The opening request exceeds 16 KiB.") } - } - let operation = opensItem ? "open" : "launch" - let keys: Set = opensItem ? ["op", "item", "application"] : ["op", "application"] - guard let request = try JSONSerialization.jsonObject(with: data) as? [String: Any], - Set(request.keys).isSubset(of: keys), request["op"] as? String == operation - else { throw LaunchError("Use one item and optional application for open, or one application for launch.") } - var path: String? - var bundleID: String? - if !opensItem || request["application"] != nil { - guard let application = request["application"] as? [String: Any], application.count == 1 else { - throw LaunchError("Use exactly one application path or bundle_id.") - } - path = application["path"] as? String - bundleID = application["bundle_id"] as? String - if let path { - var directory: ObjCBool = false - guard path.utf16.count <= 4096, path.hasPrefix("/"), path.lowercased().hasSuffix(".app"), !path.contains("\0"), - FileManager.default.fileExists(atPath: path, isDirectory: &directory), directory.boolValue, - Bundle(url: URL(fileURLWithPath: path))?.executableURL != nil - else { throw LaunchError("Use an existing absolute .app path for launch.") } - } else { - guard let bundleID, bundleID.utf16.count <= 256, - bundleID.range(of: "^[A-Za-z0-9.-]+$", options: .regularExpression) != nil - else { throw LaunchError("Use an exact application bundle ID for launch.") } - } - } - var urls: [URL] = [] - if opensItem { - guard let item = request["item"] as? [String: Any], item.count == 1 else { - throw LaunchError("Open accepts exactly one item path or url.") - } - if let path = item["path"] as? String { - guard path.utf16.count <= 4096, path.hasPrefix("/"), !path.contains("\0"), - FileManager.default.fileExists(atPath: path) - else { throw LaunchError("Use an existing absolute item path.") } - urls = [URL(fileURLWithPath: path)] - } else { - guard let raw = item["url"] as? String, raw.utf16.count <= 4096, - !raw.unicodeScalars.contains(where: { CharacterSet.controlCharacters.contains($0) }), - let url = URL(string: raw, encodingInvalidCharacters: false), let scheme = url.scheme, - scheme.range(of: "^[A-Za-z][A-Za-z0-9+.-]*$", options: .regularExpression) != nil, - url.absoluteString == raw - else { throw LaunchError("Use a correctly encoded absolute URL with an explicit scheme and no control characters.") } - urls = [url] - } - } - return ApplicationLaunchRequest(applicationIdentifier: path, applicationBundleIdentifier: bundleID, - openURLs: urls, activates: true, waitUntilReady: true, waitForWindow: false, createsNewInstance: false) -} diff --git a/apps/desktop/native/sources/client/management.swift b/apps/desktop/native/sources/client/management.swift deleted file mode 100644 index 7aa2c29..0000000 --- a/apps/desktop/native/sources/client/management.swift +++ /dev/null @@ -1,280 +0,0 @@ -import CoreGraphics -import Foundation -import PeekabooAutomationKit -import PeekabooBridge -import PeekabooFoundation - -struct ManagementTarget: Codable { - let pid: Int32 - let process_start_identity_decimal: String - let window_id: Int? - let bounds: Bounds? - let is_minimized: Bool? - - init(_ identity: ApplicationProcessIdentity) { - pid = identity.processIdentifier - process_start_identity_decimal = String(identity.processStartIdentity) - window_id = nil - bounds = nil - is_minimized = nil - } - - init?(window: ServiceWindowInfo, pid: Int32) { - guard let identity = window.mutationIdentity, - identity.ownerProcessIdentifier == pid, - identity.windowID == window.windowID, - identity.capturedBounds == window.bounds - else { return nil } - self.pid = pid - process_start_identity_decimal = String(identity.ownerProcessStartIdentity) - window_id = identity.windowID - bounds = Bounds(window.bounds) - is_minimized = window.isMinimized - } - - func processIdentity() throws -> ApplicationProcessIdentity { - guard pid > 0, let generation = UInt64(process_start_identity_decimal), generation > 0, - String(generation) == process_start_identity_decimal - else { throw ManagementError("Use the exact application target from desktop inventory.") } - return ApplicationProcessIdentity(processIdentifier: pid, processStartIdentity: generation) - } - - func windowIdentity() throws -> WindowMutationIdentity { - let process = try processIdentity() - guard let window_id, let id = UInt32(exactly: window_id), id > 0, - let bounds, let is_minimized, - [bounds.x, bounds.y, bounds.width, bounds.height].allSatisfy(\.isFinite), - bounds.width > 0, bounds.height > 0 - else { throw ManagementError("Use the exact window target and original bounds from desktop_windows.") } - return WindowMutationIdentity( - windowID: window_id, - ownerProcessIdentifier: process.processIdentifier, - ownerProcessStartIdentity: process.processStartIdentity, - capturedBounds: CGRect(x: bounds.x, y: bounds.y, width: bounds.width, height: bounds.height), - isMinimized: is_minimized - ) - } -} - -private struct ManagementRequest: Decodable { - enum Operation: String, Decodable { - case activate, quit, close, focus, minimize, restore, move, resize - } - struct Position: Decodable { - let x: Double - let y: Double - } - struct Size: Decodable { - let width: Double - let height: Double - } - let op: Operation - let target: ManagementTarget - let position: Position? - let size: Size? - - func validate() throws { - switch op { - case .move: - guard let position, size == nil, position.x.isFinite, position.y.isFinite else { - throw ManagementError("Window position must contain finite x and y in desktop logical points.") - } - case .resize: - guard let size, position == nil, size.width.isFinite, size.height.isFinite, - size.width > 0, size.height > 0 - else { throw ManagementError("Window size must contain positive finite width and height in desktop logical points.") } - default: - guard position == nil, size == nil else { - throw ManagementError("Only move or resize accepts requested window geometry.") - } - } - } -} - -private struct ManagementError: LocalizedError { - let message: String - init(_ message: String) { self.message = message } - var errorDescription: String? { message } -} - -private struct ManagementMessage: Encodable { - let code: String - let message: String - let hint: String? -} - -private struct ManagementResult: Encodable { - var outcome = "refused" - var action = "management" - var native_outcome: DesktopActionOutcome? - var terminated: Bool? - var message: String? - var requires_fresh_observation = false - var error: ManagementMessage? -} - -private struct ManagementReply: Encodable { - let success = true - let data: ManagementResult - let target_receipt: Receipt? -} - -func nativeManagement(_ client: PeekabooBridgeClient, handshake: PeekabooBridgeHandshakeResponse) async throws -> Data { - var result = ManagementResult() - var receipt: Receipt? - var invoked = false - var closing: WindowMutationIdentity? - do { - let request = try readManagement() - result.action = request.op.rawValue - try request.validate() - let process = try request.target.processIdentity() - let window: WindowMutationIdentity? - if request.op == .activate || request.op == .quit { - guard request.target.window_id == nil, request.target.bounds == nil, request.target.is_minimized == nil else { - throw ManagementError("Activation and quit take an application target from desktop_apps.") - } - window = nil - } else { - window = try request.target.windowIdentity() - } - if request.op == .quit { - guard handshake.negotiatedVersion >= PeekabooBridgeConstants.processGenerationPinnedApplicationQuitVersion, - handshake.supportedOperations.contains(.quitApplication), - (handshake.enabledOperations ?? handshake.supportedOperations).contains(.quitApplication) - else { throw ManagementError("The desktop runtime does not support process-generation-pinned quit. Update the runtime before acting.") } - } - // Match Peekaboo's capture preflight; absent session state does not establish a lock. - let session = CGSessionCopyCurrentDictionary() as NSDictionary? - if session?["CGSSessionScreenIsLocked"] as? Bool == true { - throw DesktopActionFailure.preDispatchRefusal( - reason: .targetUnavailable, - message: "The macOS GUI session is locked. Desktop management was not dispatched.", - hint: "Unlock the active user session, then refresh desktop_apps or desktop_windows before choosing a new action." - ) - } - // The inventory receipt proves signed transport; optional source-build metadata is not required. - let inventory = try await client.listApplicationMutationInventory() - guard let preflight = await client.lastOperationReceipt(), preflight.payload.operation == PeekabooBridgeRequest.listApplicationMutationInventory.operation else { - throw ManagementError("The native runtime cannot attest exact management targets. Update the desktop runtime before acting.") - } - // A failed read-only preflight cannot have dispatched this management action. Native revalidation still owns races. - guard let application = inventory.items.first(where: { $0.processIdentifier == process.processIdentifier }), - application.processIdentity == process - else { throw ManagementError("The observed application generation could not be verified. Refresh desktop_apps before acting.") } - try Task.checkCancellation() - let outcome: DesktopActionOutcome? - let operation: PeekabooBridgeOperation - var terminated: Bool? - invoked = true - switch request.op { - case .activate: - operation = .activateApplication - let action = try await client.activateApplicationTargetedResult(request: .init( - identifier: "PID:\(process.processIdentifier)", expectedIdentity: process - )) - result.native_outcome = action.outcome - guard action.targetIdentity?.processIdentity == process, action.targetIdentity?.exactWindow == nil else { - throw ManagementError("Activation returned no matching application-only receipt.") - } - outcome = action.outcome - case .quit: - operation = .quitApplication - let action = try await client.quitApplicationResult(request: .init( - identifier: "PID:\(process.processIdentifier)", force: false, expectedIdentity: process - ), supportsPinnedQuit: true) - terminated = action.payload - outcome = action.outcome - case .close: - operation = .backgroundCloseWindow - closing = window - outcome = try await client.closeWindowResult( - target: .windowId(window!.windowID), expectedIdentity: window!, allowForegroundFallback: false - ).outcome - case .focus: - operation = .focusWindow - let action = try await client.focusWindowResult(target: .windowId(window!.windowID), expectedIdentity: window!) - result.native_outcome = action.outcome - guard action.targetIdentity?.exactWindow?.identity.hasSameStableReceipt(as: window!) == true else { - throw ManagementError("Focus returned no matching exact-window receipt.") - } - outcome = action.outcome - case .minimize: - operation = .minimizeWindow - outcome = try await client.minimizeWindowResult(target: .windowId(window!.windowID), expectedIdentity: window!).outcome - case .restore: - operation = .restoreWindow - outcome = try await client.restoreWindowResult(target: .windowId(window!.windowID), expectedIdentity: window!).outcome - case .move: - operation = .moveWindow - let position = request.position! - outcome = try await client.moveWindowResult( - target: .windowId(window!.windowID), expectedIdentity: window!, - to: CGPoint(x: position.x, y: position.y) - ).outcome - case .resize: - operation = .resizeWindow - let size = request.size! - outcome = try await client.resizeWindowResult( - target: .windowId(window!.windowID), expectedIdentity: window!, - to: CGSize(width: size.width, height: size.height) - ).outcome - } - result.native_outcome = outcome - // State and geometry results omit targetIdentity; the client's accepted signed receipt retains it. - guard let signed = await client.lastOperationReceipt(), signed.payload.operation == operation, - let outcome, signed.payload.outcome?.outcome == outcome - else { throw ManagementError("The management action returned without its matching verified operation receipt and outcome.") } - switch signed.payload.target { - case let .process(identity) where window == nil && identity == process: - receipt = Receipt(pid: identity.processIdentifier, window_id: nil, process_start_identity_decimal: String(identity.processStartIdentity)) - case let .window(identity) where window.map({ identity.hasSameStableReceipt(as: $0) }) == true: - receipt = Receipt(pid: identity.ownerProcessIdentifier, window_id: identity.windowID, process_start_identity_decimal: String(identity.ownerProcessStartIdentity)) - default: - throw ManagementError("The management action's signed target did not match the original inventory target.") - } - result.requires_fresh_observation = outcome.dispatchState.mutationDispatched - result.terminated = terminated - if terminated == false { - result.message = "The normal quit request was accepted, but termination was not confirmed. Inspect remaining windows for unsaved work or other dialogs; do not blindly retry or force quit." - } - if request.op == .close, !outcome.isConfirmed { - result.message = "The close request was not confirmed complete. Inspect remaining windows for unsaved work or other dialogs; do not blindly retry close." - } - switch outcome.state { - case .refused: - result.outcome = "refused" - case .indeterminate, .partial: - result.outcome = "unknown" - default: - result.outcome = outcome.evidence == .operationStillRunning || terminated == false || (request.op == .close && !outcome.isConfirmed) ? "unknown" : "completed" - } - } catch let failure as DesktopActionFailure { - result.outcome = failure.outcome.dispatchState.mutationDispatched ? "unknown" : "refused" - result.native_outcome = failure.outcome - // The bridge attributes failures only after verifying the signed request-bound target. - if let closing, failure.targetReceipt == closing.actionTargetReceipt { - receipt = Receipt(pid: closing.ownerProcessIdentifier, window_id: closing.windowID, process_start_identity_decimal: String(closing.ownerProcessStartIdentity)) - } - result.requires_fresh_observation = failure.outcome.dispatchState.mutationDispatched || failure.outcome.escalation == .refreshTarget - result.error = ManagementMessage(code: failure.standardErrorCode?.rawValue ?? "DESKTOP_ACTION_FAILED", message: failure.message, hint: failure.hint) - } catch { - result.outcome = invoked ? "unknown" : "refused" - result.requires_fresh_observation = invoked - result.error = ManagementMessage( - code: (error as? PeekabooBridgeErrorEnvelope)?.code.rawValue ?? "DESKTOP_ACTION_FAILED", - message: error.localizedDescription, - hint: invoked ? "Refresh the target before any retry; the action may already have happened." : nil - ) - } - return try JSONEncoder().encode(ManagementReply(data: result, target_receipt: receipt)) -} - -private func readManagement() throws -> ManagementRequest { - var data = Data() - while let chunk = try FileHandle.standardInput.read(upToCount: 4096), !chunk.isEmpty { - data.append(chunk) - guard data.count <= 16_384 else { throw ManagementError("The desktop management request exceeds 16 KiB.") } - } - return try JSONDecoder().decode(ManagementRequest.self, from: data) -} diff --git a/apps/desktop/native/sources/client/menu.swift b/apps/desktop/native/sources/client/menu.swift deleted file mode 100644 index dc8071b..0000000 --- a/apps/desktop/native/sources/client/menu.swift +++ /dev/null @@ -1,108 +0,0 @@ -import CoreGraphics -import Foundation -import PeekabooAutomationKit -import PeekabooBridge -import PeekabooFoundation - -private struct MenuCommandInput: Decodable { - let target: ManagementTarget - let path: [String] -} - -private struct MenuCommandError: LocalizedError { - let message: String - init(_ message: String) { self.message = message } - var errorDescription: String? { message } -} - -private struct MenuCommandMessage: Encodable { - let code: String - let message: String - let hint: String? -} - -private struct MenuCommandResult: Encodable { - var outcome = "refused" - let action = "menu" - var native_outcome: DesktopActionOutcome? - var requires_fresh_observation = false - var error: MenuCommandMessage? -} - -private struct MenuCommandReply: Encodable { - let success = true - let data: MenuCommandResult - let target_receipt: Receipt? -} - -func nativeMenuCommand(_ client: PeekabooBridgeClient, handshake: PeekabooBridgeHandshakeResponse) async throws -> Data { - var result = MenuCommandResult() - var receipt: Receipt? - var process: ApplicationProcessIdentity? - var invoked = false - do { - var data = Data() - while let chunk = try FileHandle.standardInput.read(upToCount: 4096), !chunk.isEmpty { - guard data.count + chunk.count <= 4096 else { throw MenuCommandError("Menu request exceeds the 4096-byte limit.") } - data.append(chunk) - } - let input = try JSONDecoder().decode(MenuCommandInput.self, from: data) - guard input.target.window_id == nil, input.target.bounds == nil, input.target.is_minimized == nil else { - throw MenuCommandError("Menu commands require only the application target from desktop_apps.") - } - let identity = try input.target.processIdentity() - process = identity - let request = try MenuCommandRequest(expectedIdentity: identity, path: input.path) - guard handshake.supportedOperations.contains(.menuCommand), - (handshake.enabledOperations ?? handshake.supportedOperations).contains(.menuCommand) - else { throw MenuCommandError("The native runtime does not support exact literal menu commands. Update the desktop runtime before acting.") } - let session = CGSessionCopyCurrentDictionary() as NSDictionary? - if session?["CGSSessionScreenIsLocked"] as? Bool == true { - throw DesktopActionFailure.preDispatchRefusal( - reason: .targetUnavailable, message: "The macOS GUI session is locked. No menu command was dispatched.", - hint: "Unlock the active user session and refresh the application and its menus." - ) - } - let inventory = try await client.listApplicationMutationInventory() - guard let preflight = await client.lastOperationReceipt(), - preflight.payload.operation == PeekabooBridgeRequest.listApplicationMutationInventory.operation, - inventory.items.contains(where: { $0.processIdentity == identity }) - else { throw MenuCommandError("The observed application generation could not be attested before the menu command.") } - try Task.checkCancellation() - invoked = true - let action = try await client.menuCommand(request) - result.native_outcome = action.outcome - guard let signed = await client.lastOperationReceipt(), signed.payload.operation == .menuCommand, - let outcome = action.outcome, signed.payload.outcome?.outcome == outcome, - action.targetIdentity?.processIdentity == identity, action.targetIdentity?.exactWindow == nil, - case let .process(verified) = signed.payload.target, verified == identity - else { throw MenuCommandError("The menu command returned without its matching signed application receipt and outcome.") } - receipt = Receipt(pid: identity.processIdentifier, window_id: nil, process_start_identity_decimal: String(identity.processStartIdentity)) - result.requires_fresh_observation = outcome.dispatchState.mutationDispatched - switch outcome.state { - case .refused: - result.outcome = "refused" - case .dispatchedUnverified where outcome.evidence == .deliveryAccepted: - result.outcome = "completed" - default: - result.outcome = "unknown" - } - } catch let failure as DesktopActionFailure { - result.native_outcome = failure.outcome - result.outcome = failure.outcome.dispatchState.mutationDispatched ? "unknown" : "refused" - if let process, failure.targetReceipt == process.actionTargetReceipt { - receipt = Receipt(pid: process.processIdentifier, window_id: nil, process_start_identity_decimal: String(process.processStartIdentity)) - } - result.requires_fresh_observation = failure.outcome.dispatchState.mutationDispatched || failure.outcome.escalation == .refreshTarget - result.error = MenuCommandMessage(code: failure.standardErrorCode?.rawValue ?? "DESKTOP_ACTION_FAILED", message: failure.message, hint: failure.hint) - } catch { - result.outcome = invoked ? "unknown" : "refused" - result.requires_fresh_observation = invoked - result.error = MenuCommandMessage( - code: (error as? PeekabooBridgeErrorEnvelope)?.code.rawValue ?? "DESKTOP_ACTION_FAILED", - message: error.localizedDescription, - hint: invoked ? "Observe the application; the menu command may already have happened. Do not blindly repeat it." : nil - ) - } - return try JSONEncoder().encode(MenuCommandReply(data: result, target_receipt: receipt)) -} diff --git a/apps/desktop/native/sources/client/menus.swift b/apps/desktop/native/sources/client/menus.swift deleted file mode 100644 index c7018e3..0000000 --- a/apps/desktop/native/sources/client/menus.swift +++ /dev/null @@ -1,119 +0,0 @@ -import Foundation -import PeekabooAutomationKit -import PeekabooBridge -import PeekabooFoundation - -private struct MenuRequest: Decodable { - let target: ManagementTarget - let path: [String]? -} - -private struct MenuRow: Encodable { - let path: [String] - let kind: String - let shortcut: String? -} - -private struct MenuFilter: Encodable { - let path: [String] - let scope = "returned_native_inventory" - let total: Int - let matched: Int -} - -private struct MenuInventory: Encodable { - let target: ManagementTarget - let application_name: String - let bundle_id: String? - let menus: [MenuRow] - let native_row_count: Int - let filter: MenuFilter? - let native_completeness = "unknown" - let cache_may_be_used = true - let cache_ttl_ms = 2000 - let warnings = [ - "Native menu reads may reuse a cached structure; its observation time is unavailable. The cache expires 2 seconds after the original traversal finishes.", - "Native completeness is unknown: lazy submenus, unavailable AX attributes and traversal limits may omit items. An empty subtree does not establish that no commands exist.", - "Paths are literal title arrays for discovery, not input authority. Enabled and checked states are omitted because unavailable native attributes receive default values." - ] -} - -private struct MenuReply: Encodable { - let success = true - let data: MenuInventory -} - -private struct MenuError: LocalizedError { - let message: String - init(_ message: String) { self.message = message } - var errorDescription: String? { message } -} - -func nativeMenus(_ client: PeekabooBridgeClient) async throws -> Data { - var input = Data() - while let chunk = try FileHandle.standardInput.read(upToCount: 4096), !chunk.isEmpty { - guard input.count + chunk.count <= 4096 else { throw MenuError("Menu inventory requires one exact application target.") } - input.append(chunk) - } - let request = try JSONDecoder().decode(MenuRequest.self, from: input) - let target = request.target - if let path = request.path { - guard !path.isEmpty, path.count <= 8, - path.allSatisfy({ !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && $0.utf16.count <= 512 }) - else { throw MenuError("Menu path requires 1 to 8 nonblank literal titles of at most 512 UTF-16 code units each.") } - } - guard target.window_id == nil, target.bounds == nil, target.is_minimized == nil else { - throw MenuError("Menu inventory takes an application target from desktop_apps.") - } - let process = try target.processIdentity() - try await verifyMenuApplication(client, process: process) - let structure = try await client.listMenus(appIdentifier: "PID:\(process.processIdentifier)") - // Save the menu's attestation before the following inventory replaces lastOperationReceipt. - let receipt = await client.lastOperationReceipt() - guard receipt?.payload.operation == .listMenus, structure.application.processIdentity == process else { - throw MenuError("The native menu response did not attest the requested application generation. Refresh desktop_apps.") - } - try await verifyMenuApplication(client, process: process) - var rows: [MenuRow] = [] - func append(_ items: [MenuItem], path: [String]) { - for item in items { - let path = path + [item.title] - rows.append(MenuRow(path: path, kind: item.isSeparator ? "separator" : "item", shortcut: item.keyboardShortcut?.displayString)) - append(item.submenu, path: path) - } - } - for menu in structure.menus { - rows.append(MenuRow(path: [menu.title], kind: "menu", shortcut: nil)) - append(menu.items, path: [menu.title]) - } - let total = rows.count - var filter: MenuFilter? - if let path = request.path { - for count in 1 ... path.count { - let prefix = Array(path.prefix(count)) - let matches = rows.filter { $0.path == prefix } - guard matches.count == 1 else { - throw MenuError(matches.isEmpty - ? "The requested menu path was not found in the returned native inventory; lazy menus or native limits may omit it." - : "The requested menu path is ambiguous in the returned native inventory.") - } - } - rows = rows.filter { $0.path.starts(with: path) } - filter = MenuFilter(path: path, total: total, matched: rows.count) - } - try Task.checkCancellation() - return try JSONEncoder().encode(MenuReply(data: MenuInventory( - target: ManagementTarget(process), application_name: structure.application.name, - bundle_id: structure.application.bundleIdentifier, menus: rows, native_row_count: total, filter: filter - ))) -} - -private func verifyMenuApplication(_ client: PeekabooBridgeClient, process: ApplicationProcessIdentity) async throws { - try Task.checkCancellation() - let inventory = try await client.listApplicationMutationInventory() - let receipt = await client.lastOperationReceipt() - guard receipt?.payload.operation == PeekabooBridgeRequest.listApplicationMutationInventory.operation, - inventory.items.contains(where: { $0.processIdentity == process }) - else { throw MenuError("The observed application generation could not be verified. Refresh desktop_apps before reading its menus.") } - try Task.checkCancellation() -} diff --git a/apps/desktop/native/sources/desktop/desktop.swift b/apps/desktop/native/sources/desktop/desktop.swift deleted file mode 100644 index c8ff296..0000000 --- a/apps/desktop/native/sources/desktop/desktop.swift +++ /dev/null @@ -1,174 +0,0 @@ -import AceSigning -import AppKit -import ApplicationServices -import CoreGraphics -import Darwin -import Foundation -import PeekabooAutomationKit -import PeekabooBridge - -private struct Status: Encodable { - var state = "stopped" - var error: String? - var accessibility = false - var screenRecording = false - var eventSynthesizing = false - var clipboardRead = ClipboardPolicy() -} - -private final class State: @unchecked Sendable { - private let lock = NSLock() - private var value = Status() - - func set(_ phase: String, error: String? = nil) { - lock.lock() - defer { lock.unlock() } - value.state = phase - value.error = error - } - - func read() -> Status { - lock.lock() - var result = value - lock.unlock() - result.accessibility = AXIsProcessTrusted() - result.screenRecording = CGPreflightScreenCaptureAccess() - result.eventSynthesizing = result.eventSynthesizing || CGPreflightPostEventAccess() - result.clipboardRead = ClipboardPolicy.current() - return result - } - - func recordEventSynthesizing(_ granted: Bool) { - lock.lock() - value.eventSynthesizing = value.eventSynthesizing || granted - lock.unlock() - } -} - -private let state = State() - -@MainActor -private final class Desktop { - static let shared = Desktop() - private var runtime: PeekabooEmbeddedBridgeRuntime? - private var tail: Task? - - func start(socket: String, client: String) { - let previous = tail - tail = Task { - await previous?.value - if let runtime { await runtime.stopChecked() } - runtime = nil - state.set("starting") - do { - let identity = try SigningIdentity.current() - let next = PeekabooEmbeddedBridgeRuntime.make(configuration: .init( - socketPath: socket, - allowlistedTeams: [identity.team], - allowlistedBundles: [client], - allowedOperations: [ - .listApplications, .listWindows, .listMenus, .menuCommand, .desktopObservation, - .createSnapshot, .cleanSnapshot, .ownsSnapshot, .getDetectionResult, .beginSnapshotMutation, - .finishSnapshotMutation, - .targetedClick, .exactWindowTargetedClick, .setValue, .exactWindowTargetedHotkey, - .selectText, .literalInsert, .clipboardTextRead, .clipboardImageRead, .clipboardFilesRead, .clipboardTextWrite, .clipboardImageWrite, .clipboardFilesWrite, .targetedScroll, .exactWindowDrag, - .launchApplicationWithOptions, .activateApplication, .quitApplication, .backgroundCloseWindow, - .focusWindow, .minimizeWindow, .restoreWindow, - .moveWindow, .resizeWindow, - ], - hostKind: .gui, - requestTimeoutSeconds: 25 - )) - runtime = next - try await next.startChecked() - state.set("ready") - } catch { - if let runtime { await runtime.stopChecked() } - runtime = nil - state.set("error", error: error.localizedDescription) - } - } - } - - func stop() { - let previous = tail - tail = Task { - await previous?.value - state.set("stopping") - if let runtime { await runtime.stopChecked() } - runtime = nil - state.set("stopped") - } - } -} - -@_cdecl("ace_desktop_start") -public func start(_ socket: UnsafePointer, _ client: UnsafePointer) { - let socket = String(cString: socket) - let client = String(cString: client) - openProjectPicker() - state.set("starting") - // Bun's thread must not wait for work that needs the application's main run loop. - DispatchQueue.main.async { - Desktop.shared.start(socket: socket, client: client) - } -} - -@_cdecl("ace_desktop_status") -public func status() -> UnsafeMutablePointer? { - let data = try! JSONEncoder().encode(state.read()) - return strdup(String(decoding: data, as: UTF8.self)) -} - -@_cdecl("ace_desktop_stop") -public func stop() { - closeProjectPicker() - state.set("stopping") - DispatchQueue.main.async { Desktop.shared.stop() } -} - -@_cdecl("ace_desktop_free") -public func release(_ value: UnsafeMutableRawPointer?) { - free(value) -} - -@_cdecl("ace_desktop_permission") -public func permission(_ kind: UnsafePointer) { - let kind = String(cString: kind) - DispatchQueue.main.async { - switch kind { - case "accessibility": - // The C SDK exposes this immutable option key as an unisolated mutable global. - let options = ["AXTrustedCheckOptionPrompt": true] - _ = AXIsProcessTrustedWithOptions(options as CFDictionary) - case "screenRecording": - _ = CGRequestScreenCaptureAccess() - case "eventSynthesizing": - // macOS caches preflight results; retain an interactive grant in Peekaboo's shared permission state too. - state.recordEventSynthesizing(PermissionsService().requestPostEventPermission()) - default: - break - } - } -} - -private struct ClipboardPolicy: Encodable { - var policy = "unknown" - var readAdmitted = false - var policyAvailable = false - - static func current() -> ClipboardPolicy { - guard #available(macOS 15.4, *) else { - return .init(policy: "unavailable_on_this_os", readAdmitted: true, policyAvailable: false) - } - let policy: String - switch NSPasteboard.general.accessBehavior { - case .default: policy = "default" - case .ask: policy = "ask" - case .alwaysAllow: policy = "always_allow" - case .alwaysDeny: policy = "always_deny" - @unknown default: policy = "unknown" - } - return .init(policy: policy, readAdmitted: policy == "always_allow", policyAvailable: true) - } -} diff --git a/apps/desktop/native/sources/signing/signing.swift b/apps/desktop/native/sources/signing/signing.swift deleted file mode 100644 index 660103c..0000000 --- a/apps/desktop/native/sources/signing/signing.swift +++ /dev/null @@ -1,49 +0,0 @@ -import Foundation -import Security - -public struct SigningIdentity: Sendable { - public let identifier: String - public let team: String - - public static func current() throws -> Self { - var code: SecCode? - guard SecCodeCopySelf(SecCSFlags(), &code) == errSecSuccess, let code else { - throw SigningError() - } - var staticCode: SecStaticCode? - guard SecCodeCopyStaticCode(code, SecCSFlags(), &staticCode) == errSecSuccess, - let staticCode - else { throw SigningError() } - var information: CFDictionary? - guard SecCodeCopySigningInformation( - staticCode, - SecCSFlags(rawValue: UInt32(kSecCSSigningInformation)), - &information - ) == errSecSuccess, - let values = information as? [String: Any], - let identifier = values[kSecCodeInfoIdentifier as String] as? String, - let team = values[kSecCodeInfoTeamIdentifier as String] as? String, - !identifier.isEmpty, !team.isEmpty - else { throw SigningError() } - - let text = "anchor apple generic and identifier \(quoted(identifier)) " - + "and certificate leaf[subject.OU] = \(quoted(team))" - var requirement: SecRequirement? - guard SecRequirementCreateWithString(text as CFString, SecCSFlags(), &requirement) == errSecSuccess, - let requirement, - SecCodeCheckValidity(code, SecCSFlags(), requirement) == errSecSuccess - else { throw SigningError() } - return Self(identifier: identifier, team: team) - } -} - -private func quoted(_ value: String) -> String { - "\"" + value.replacingOccurrences(of: "\\", with: "\\\\") - .replacingOccurrences(of: "\"", with: "\\\"") + "\"" -} - -private struct SigningError: LocalizedError { - var errorDescription: String? { - "Native desktop tools require Ace and its bundled client to be signed with an Apple Development or Developer ID identity." - } -} diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 73c9f9a..78e6f7c 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@ace/host": "workspace:*", + "@githubnext/desktop-tools": "catalog:desktop", "electrobun": "catalog:desktop" }, "devDependencies": { diff --git a/apps/desktop/scripts/helper.ts b/apps/desktop/scripts/helper.ts index 4ae2241..29f31b9 100644 --- a/apps/desktop/scripts/helper.ts +++ b/apps/desktop/scripts/helper.ts @@ -1,14 +1,5 @@ import { createHash } from "node:crypto"; -import { - copyFileSync, - mkdirSync, - mkdtempSync, - readdirSync, - readFileSync, - rmSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; +import { copyFileSync, cpSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -120,105 +111,42 @@ writeFileSync( ); const frameworks = join(contents, "Frameworks"); mkdirSync(frameworks, { recursive: true }); +// The standalone package owns the pinned, patched native build; Ace signs and stages its outputs. +const tools = join(native, ".build", "desktop-tools"); +const pkg = fileURLToPath(import.meta.resolve("@githubnext/desktop-tools/package.json")); run([ - "xcrun", - "swift", - "package", - "--package-path", - native, - "--force-resolved-versions", - "resolve", + compiler, + join(dirname(pkg), "dist", "build.js"), + "--out", + join(tools, "out"), + "--scratch", + join(tools, "scratch"), ]); -const resolved = JSON.parse(readFileSync(join(native, "Package.resolved"), "utf8")) as { - pins: { identity: string; state: { version: string; revision: string } }[]; -}; -const checkouts = join(native, ".build", "checkouts"); -const dependencies = new Map(readdirSync(checkouts).map((name) => [name.toLowerCase(), name])); -const peekaboo = resolved.pins.find(({ identity }) => identity === "peekaboo"); -const revision = "4d43dc9d80cd2aa3787a27f54b76d692db1dcf8f"; -if (peekaboo?.state.version !== "4.8.0" || peekaboo.state.revision !== revision) { - throw new Error("The native patches require Peekaboo 4.8.0 at its pinned revision"); -} -const checkout = dependencies.get("peekaboo"); -if (!checkout) throw new Error("The resolved Peekaboo checkout is missing"); -const git = ["git", "-C", join(checkouts, checkout)]; -const head = Bun.spawnSync([...git, "rev-parse", "HEAD"]); -if (!head.success || head.stdout.toString().trim() !== revision) { - throw new Error( - `The Peekaboo checkout must be at ${revision} before applying the native patches`, - ); -} -// Build the expected patch stack in a private index: later patches may change earlier patch contexts. -const patches = [ - "peekaboo-click.patch", - "peekaboo-insert.patch", - "peekaboo-pointer-window.patch", - "peekaboo-quit.patch", - "peekaboo-clipboard-text.patch", - "peekaboo-close.patch", - "peekaboo-clipboard-image.patch", - "peekaboo-launch.patch", - "peekaboo-clipboard-files.patch", - "peekaboo-point-focus.patch", - "peekaboo-clipboard-image-write.patch", - "peekaboo-open.patch", - "peekaboo-menu.patch", - "peekaboo-clipboard-files-write.patch", - "peekaboo-stale-click.patch", -] - .map((name) => join(native, "patches", name)); -const temporary = mkdtempSync(join(tmpdir(), "ace-native-patches-")); -try { - const env = { ...process.env, GIT_INDEX_FILE: join(temporary, "index") }; - const initial = Bun.spawnSync([...git, "read-tree", "HEAD"], { env }); - if (!initial.success) throw new Error("Cannot read the pinned native tree: " + initial.stderr); - const isExpected = () => { - const extra = Bun.spawnSync([...git, "ls-files", "--others", "--exclude-standard"], { env }); - const diff = Bun.spawnSync([...git, "diff", "--quiet", "--"], { env }); - if (!extra.success || (diff.exitCode !== 0 && diff.exitCode !== 1)) { - throw new Error("Cannot verify the native checkout: " + extra.stderr + "\n" + diff.stderr); - } - return !extra.stdout.toString().trim() && diff.success; - }; - let applied = -1; - for (let index = 0; index <= patches.length; index++) { - if (isExpected()) applied = index; - if (index === patches.length) break; - const expected = Bun.spawnSync([...git, "apply", "--cached", patches[index]!], { env }); - if (!expected.success) { - throw new Error("Cannot assemble the pinned native patch stack: " + expected.stderr); - } - } - if (applied < 0) { - throw new Error( - "The native checkout differs from every pinned patch prefix. Resolve source drift before building.", - ); - } - for (const patch of patches.slice(applied)) run([...git, "apply", patch]); - if (!isExpected()) throw new Error("The native checkout differs from the pinned patch stack."); -} finally { - rmSync(temporary, { recursive: true, force: true }); -} -const swiftBuild = [ +copyFileSync(join(tools, "out", "libDesktopTools.dylib"), join(bin, "libDesktopTools.dylib")); +copyFileSync(join(tools, "out", "desktop-tools-client"), join(bin, "ace-desktop-client")); +cpSync(join(tools, "out", "Licenses"), join(contents, "Resources", "Licenses"), { + recursive: true, +}); +run([ "xcrun", - "swift", - "build", - "--package-path", - native, - "--configuration", - "release", - "--force-resolved-versions", + "swiftc", + "-emit-library", + "-O", + "-swift-version", + "6", + "-module-name", + "AceProject", + "-target", + "arm64-apple-macos15.0", "-Xlinker", "-rpath", "-Xlinker", "@loader_path/../Frameworks", -]; -run(swiftBuild); -const location = Bun.spawnSync([...swiftBuild, "--show-bin-path"]); -if (!location.success) throw new Error("Cannot locate the native desktop build products"); -const products = location.stdout.toString().trim(); -const binaries = ["libAceDesktop.dylib", "ace-desktop-client"]; -for (const name of binaries) copyFileSync(join(products, name), join(bin, name)); + join(native, "project.swift"), + "-o", + join(bin, "libAceProject.dylib"), +]); +const binaries = ["libDesktopTools.dylib", "ace-desktop-client", "libAceProject.dylib"]; // Discover the runtimes from Mach-O dependencies instead of assuming a Swift library list. run([ "xcrun", @@ -230,18 +158,6 @@ run([ "--destination", frameworks, ]); -for (const { identity } of resolved.pins) { - const checkout = dependencies.get(identity); - if (!checkout) throw new Error(`The resolved native dependency ${identity} is missing`); - const source = join(checkouts, checkout); - const licenses = readdirSync(source).filter((name) => - /^(LICENSE|LICENCE|NOTICE)([.-].*)?$/i.test(name) - ); - if (!licenses.length) throw new Error(`The native dependency ${identity} has no license file`); - const destination = join(contents, "Resources", "Licenses", identity); - mkdirSync(destination, { recursive: true }); - for (const name of licenses) copyFileSync(join(source, name), join(destination, name)); -} run(["/usr/bin/ditto", join(sdk, "Sparkle.framework"), join(frameworks, "Sparkle.framework")]); run([ "xcrun", diff --git a/apps/desktop/src/native.ts b/apps/desktop/src/native.ts index 9c72ad0..a08d5e2 100644 --- a/apps/desktop/src/native.ts +++ b/apps/desktop/src/native.ts @@ -7,35 +7,42 @@ import { config } from "@ace/host/config"; import type { NativeAction, NativeState } from "./protocol"; export function native(identifier: string) { - const { symbols } = dlopen(join(dirname(process.execPath), "libAceDesktop.dylib"), { - ace_desktop_start: { args: ["cstring", "cstring"], returns: "void" }, - ace_desktop_status: { args: [], returns: "ptr" }, - ace_desktop_stop: { args: [], returns: "void" }, - ace_desktop_free: { args: ["ptr"], returns: "void" }, - ace_desktop_permission: { args: ["cstring"], returns: "void" }, - ace_desktop_project_start: { args: ["cstring"], returns: "void" }, - ace_desktop_project_status: { args: [], returns: "ptr" }, + const bin = dirname(process.execPath); + const { symbols } = dlopen(join(bin, "libDesktopTools.dylib"), { + desktop_tools_start: { args: ["cstring", "cstring"], returns: "void" }, + desktop_tools_status: { args: [], returns: "ptr" }, + desktop_tools_stop: { args: [], returns: "void" }, + desktop_tools_free: { args: ["ptr"], returns: "void" }, + desktop_tools_permission: { args: ["cstring"], returns: "void" }, }); - let picker: Promise | undefined; + const { symbols: picker } = dlopen(join(bin, "libAceProject.dylib"), { + ace_project_open: { args: [], returns: "void" }, + ace_project_close: { args: [], returns: "void" }, + ace_project_start: { args: ["cstring"], returns: "void" }, + ace_project_status: { args: [], returns: "ptr" }, + ace_project_free: { args: ["ptr"], returns: "void" }, + }); + let pending: Promise | undefined; - function read(pointer: Pointer | null): T { + function read(pointer: Pointer | null, free: (pointer: Pointer) => void): T { if (!pointer) throw new Error("Could not read native desktop response"); try { return JSON.parse(new CString(pointer).toString()) as T; } finally { - symbols.ace_desktop_free(pointer); + free(pointer); } } function status(): NativeState { - return read(symbols.ace_desktop_status()); + return read(symbols.desktop_tools_status(), symbols.desktop_tools_free); } async function project(path: string): Promise { - symbols.ace_desktop_project_start(Buffer.from(`${path}\0`)); + picker.ace_project_start(Buffer.from(`${path}\0`)); while (true) { const result = read<{ pending: boolean; path: string | null; error?: string }>( - symbols.ace_desktop_project_status(), + picker.ace_project_status(), + picker.ace_project_free, ); if (result.error) throw new Error(result.error); if (!result.pending) return result.path; @@ -46,12 +53,13 @@ export function native(identifier: string) { return { status, project(path: string): Promise { - if (picker) return picker; - picker = project(path).finally(() => picker = undefined); - return picker; + if (pending) return pending; + pending = project(path).finally(() => pending = undefined); + return pending; }, start() { - symbols.ace_desktop_start( + picker.ace_project_open(); + symbols.desktop_tools_start( Buffer.from(`${join(config.home, "desktop.sock")}\0`), Buffer.from(`${identifier}.desktop-client\0`), ); @@ -63,13 +71,14 @@ export function native(identifier: string) { ) { throw new Error("Unknown native desktop permission"); } - symbols.ace_desktop_permission(Buffer.from(`${action.permission}\0`)); + symbols.desktop_tools_permission(Buffer.from(`${action.permission}\0`)); } return status(); }, async stop(): Promise { if (status().state === "stopped") return; - symbols.ace_desktop_stop(); + picker.ace_project_close(); + symbols.desktop_tools_stop(); // Keep the library loaded while its bridge drains outstanding native operations. const deadline = Date.now() + 30_000; while (status().state !== "stopped") { diff --git a/apps/host/package.json b/apps/host/package.json index 818091a..f9842df 100644 --- a/apps/host/package.json +++ b/apps/host/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@ace/channel": "workspace:*", + "@githubnext/desktop-tools": "catalog:desktop", "@earendil-works/chord": "catalog:pi", "@earendil-works/pi-ai": "catalog:pi", "@earendil-works/pi-durable": "catalog:pi" diff --git a/apps/host/src/desktop.ts b/apps/host/src/desktop.ts index 9c6f244..d55bc44 100644 --- a/apps/host/src/desktop.ts +++ b/apps/host/src/desktop.ts @@ -1,1063 +1,24 @@ -import { execFile } from "node:child_process"; -import { constants, existsSync } from "node:fs"; -import { lstat, mkdtemp, open, readFile, rm, stat } from "node:fs/promises"; -import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; -import { promisify } from "node:util"; -import { - type Desktop, - DESKTOP_BUTTONS, - DESKTOP_CLICKS, - DESKTOP_DIRECTIONS, - DESKTOP_KEYS, - DESKTOP_MODIFIERS, - DESKTOP_SELECTIONS, - type DesktopAction, - type DesktopApplication, - type DesktopAppTarget, - type DesktopLaunch, - type DesktopManagement, - type DesktopOpen, - type DesktopOutcome, - type DesktopRequest, - type DesktopResult, - isDesktopAction, - isDesktopManagement, -} from "@ace/channel/desktop"; +import type { Desktop } from "@ace/channel/desktop"; +import { createDesktop, type DesktopClient } from "@githubnext/desktop-tools"; import { config } from "./config"; -const exec = promisify(execFile); -const MAX_TEXT = 32_000; -const MAX_IMAGE = 900_000; -const MAX_OUTPUT = 2_000_000; -const MAX_CLIPBOARD_IMAGE = 10 * 1024 * 1024; +let client: DesktopClient | undefined; -type Reply = { - success: boolean; - data: Record | null; - error?: { code: string; message: string; details?: string }; - target_receipt?: { pid: number; window_id?: number; process_start_identity_decimal?: string }; -}; - -class NativeError extends Error { - constructor(readonly code: string, message: string) { - super(message); - } -} - -async function run( - path: string, - args: string[], - signal: AbortSignal, - errorJson = false, - input?: string, -): Promise { - signal.throwIfAborted(); - try { - const pending = exec(path, args, { - encoding: "utf8", - signal, - killSignal: "SIGKILL", - maxBuffer: MAX_OUTPUT, - }); - let inputError: Error | undefined; - if (input !== undefined) { - // A signing or setup refusal can exit before reading stdin; its JSON still owns the outcome. - pending.child.stdin!.on("error", (error) => inputError = error); - pending.child.stdin!.end(input); - } - const { stdout } = await pending; - if (inputError && !stdout.trim()) throw inputError; - return stdout; - } catch (error) { - signal.throwIfAborted(); - const failed = error as Error & { code?: number | string; stdout?: string; stderr?: string }; - // The native client reports expected refusals as JSON on unsuccessful exits too. - if (errorJson && typeof failed.code === "number" && failed.stdout?.trim()) return failed.stdout; - throw new Error((failed.stderr?.trim() || failed.message).slice(0, 2000), { cause: error }); - } -} - -async function native( - args: string[], - signal: AbortSignal, - options: { - target?: Extract; - input?: string; - onDispatch?(): void; - } = {}, -): Promise> { - const path = process.env.ACE_DESKTOP_CLIENT - || join(dirname(process.execPath), "ace-desktop-client"); - if (!existsSync(path)) { - throw new Error( - "Native desktop tools require the Ace desktop app on this host. Source hosts can set ACE_DESKTOP_CLIENT to its bundled ace-desktop-client.", - ); - } - const socket = join(config.home, "desktop.sock"); - if (!existsSync(socket)) { - throw new Error("Open Ace on this host to enable native desktop tools."); - } - signal.throwIfAborted(); - options.onDispatch?.(); - const output = await run(path, [socket, ...args], signal, true, options.input); - let value: Reply; - try { - value = JSON.parse(output) as Reply; - } catch { - throw new Error("The Ace native client returned invalid JSON."); - } - if (!value || typeof value.success !== "boolean") { - throw new Error("The Ace native client returned an unsupported response."); - } - if (!value.success) { - const error = value.error; - const reason = error ? `${error.code}: ${error.message}` : "Native desktop operation failed"; - const permission = error?.code.toLowerCase().includes("permission") - ? args[0] === "clipboard" - ? " Check Ace's clipboard read status in This Mac and its clipboard access in macOS Settings." - : " Check Ace's Accessibility and Screen Recording access in Ace Settings." - : ""; - throw new NativeError(error?.code || "DESKTOP_ERROR", `${reason}.${permission}`); - } - if (!value.data || typeof value.data !== "object") { - throw new Error("The Ace native client returned no desktop data."); - } - const { target } = options; - if (target) { - const receipt = value.target_receipt; - if (!receipt) { - throw new Error("Native inspection returned no exact-window receipt."); - } - if (receipt.pid !== target.pid || receipt.window_id !== target.window) { - throw new Error( - "Native inspection returned a different application or window. Refresh the desktop inventory and select the target again.", - ); - } - } +export const desktop: Desktop = async (request, context) => { + client ??= createDesktop({ + client: process.env.ACE_DESKTOP_CLIENT || join(dirname(process.execPath), "ace-desktop-client"), + socket: join(config.home, "desktop.sock"), + name: "Ace", + }); + // Hosted links and pi rows carry only the bounded text and image, not the parsed copy. + const { text, image, outcome, isError } = await client(request, context.abortSignal); return { - ...value.data, - ...(value.target_receipt ? { target_receipt: value.target_receipt } : {}), + text, + ...(image ? { image } : {}), + ...(outcome ? { outcome } : {}), + ...(isError === undefined ? {} : { isError }), }; -} - -function bounded(data: Record, field: string): string { - const source = data[field]; - if (!Array.isArray(source)) throw new Error(`Native inspection returned no ${field} array.`); - const values = [...source]; - const value = { ...data, [field]: values }; - let text = JSON.stringify(value); - while (Buffer.byteLength(text) > MAX_TEXT && values.length) { - values.pop(); - text = JSON.stringify({ - ...value, - ace_truncated: true, - ace_omitted: source.length - values.length, - }); - } - if (Buffer.byteLength(text) > MAX_TEXT) { - throw new Error("Native inspection metadata exceeds the result limit."); - } - return text; -} - -function positive(value: number): string { - if (!Number.isSafeInteger(value) || value < 1) { - throw new Error("Choose a positive application PID and window ID from the desktop inventory."); - } - return String(value); -} - -async function screenshot(input: string, output: string, signal: AbortSignal) { - const file = await stat(input); - if (!file.isFile() || file.size > 32_000_000) { - throw new Error("The native screenshot is missing or too large."); - } - // A single result must fit the hosted channel's 2 MB SQLite row, including its base64 image. - for (const size of [1600, 1200, 800]) { - await run("/usr/bin/sips", [ - "-s", - "format", - "jpeg", - "-s", - "formatOptions", - "70", - "--resampleHeightWidthMax", - String(size), - input, - "--out", - output, - ], signal); - if ((await stat(output)).size > MAX_IMAGE) continue; - const info = await run( - "/usr/bin/sips", - ["-g", "pixelWidth", "-g", "pixelHeight", output], - signal, - ); - const width = Number(/pixelWidth:\s*(\d+)/.exec(info)?.[1]); - const height = Number(/pixelHeight:\s*(\d+)/.exec(info)?.[1]); - if (!width || !height) throw new Error("Could not read the screenshot dimensions."); - const bytes = await readFile(output); - if (bytes.length > MAX_IMAGE) { - throw new Error("The resized screenshot exceeds the result limit."); - } - return { image: { mimeType: "image/jpeg", data: bytes.toString("base64") }, width, height }; - } - throw new Error("The screenshot could not be resized within the result limit."); -} - -function clipboardFiles(data: Record): DesktopResult { - const files = data.files; - if ( - typeof data.present !== "boolean" || !Number.isSafeInteger(data.change_count) - || (data.present - ? !Array.isArray(files) || !files.length || files.length > 32 - || files.some((file) => - !file || typeof file.url !== "string" || typeof file.path !== "string" - || !file.path.startsWith("/") - ) - : files !== undefined) - ) throw new Error("The native clipboard file read returned an unsupported response."); - const text = JSON.stringify(data); - if (Buffer.byteLength(text) > 24_000) { - throw new Error("Clipboard file references exceed the complete 24 KB result limit."); - } - return { text }; -} - -function clipboardImage(data: Record): DesktopResult { - if (typeof data.present !== "boolean" || !Number.isSafeInteger(data.change_count)) { - throw new Error("The native clipboard image read returned an unsupported response."); - } - if (!data.present) { - if (data.image !== undefined || data.source !== undefined) { - throw new Error("An absent clipboard image returned unexpected image data."); - } - return { text: JSON.stringify(data) }; - } - const preview = data.image as Record | undefined; - const source = data.source as Record | undefined; - if ( - !preview || !source || typeof preview.data !== "string" - || !["image/png", "image/jpeg"].includes(String(preview.mimeType)) - || ![preview.width, preview.height].every((value) => - typeof value === "number" && Number.isSafeInteger(value) && value > 0 && value <= 1600 - ) - || preview.data.length > Math.ceil(MAX_IMAGE / 3) * 4 - ) throw new Error("The native clipboard image preview is missing or exceeds its limits."); - const bytes = Buffer.from(preview.data, "base64"); - if ( - !bytes.length || bytes.length > MAX_IMAGE || bytes.length !== preview.bytes - || bytes.toString("base64") !== preview.data - ) throw new Error("The native clipboard image preview is not a complete bounded image."); - const { data: encoded, ...metadata } = preview; - const text = JSON.stringify({ - present: true, - change_count: data.change_count, - source, - preview: metadata, - }); - if (Buffer.byteLength(text) > MAX_TEXT) { - throw new Error("The clipboard image metadata exceeds the result limit."); - } - return { text, image: { mimeType: preview.mimeType as string, data: encoded as string } }; -} - -async function inspect( - request: Extract, - signal: AbortSignal, -): Promise { - const pid = positive(request.pid); - const window = positive(request.window); - const mode = request.mode ?? "accessibility"; - if (mode !== "accessibility" && mode !== "pixels") { - throw new Error("Choose accessibility or pixels inspection mode."); - } - const directory = await mkdtemp(join(tmpdir(), "ace-desktop-")); - try { - const path = join(directory, "capture.png"); - const args = ["inspect", pid, window, path]; - if (mode === "pixels") args.push(mode); - const data = await native(args, signal, { target: request }); - const { image, width, height } = await screenshot(path, join(directory, "image.jpg"), signal); - const { screenshot_raw: _raw, screenshot_annotated: _annotated, ...observation } = data; - const text = bounded({ - ...observation, - ace_image: { - width, - height, - note: - "Screenshot resized; Accessibility bounds remain in their original coordinate system. Pointer points use fractions of this image: x from the left edge and y from the top, each >= 0 and < 1.", - }, - }, "ui_elements"); - return { text, image }; - } catch (error) { - if (!(error instanceof NativeError)) throw error; - return { - isError: true, - text: JSON.stringify({ - inspection_error: { code: error.code, message: error.message.slice(0, 4000) }, - requested_target: { pid: request.pid, window_id: request.window, mode }, - target_availability: await availability(request, signal), - guidance: - "This inspection dispatched no input and returned no observation snapshot. Availability was read after the failure and does not establish its cause. Refresh desktop_apps and desktop_windows if the target changed. Retry an incomplete Accessibility read once; pixels mode can inspect the same exact window without Accessibility, with point-only action authority. Native capture already retries a changed capture receipt once. Neither mode activates a window. Do not loop on an unavailable target or repeat an earlier action to recover an observation.", - }), - }; - } finally { - await rm(directory, { recursive: true, force: true }); - } -} - -function validatePoint(point: unknown) { - if (!point || typeof point !== "object" || !("x" in point) || !("y" in point)) { - throw new Error("Choose a normalized screenshot point with x and y coordinates."); - } - for (const value of [point.x, point.y]) { - if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value >= 1) { - throw new Error("Screenshot point coordinates must be at least 0 and less than 1."); - } - } -} - -async function availability( - request: Extract, - signal: AbortSignal, -): Promise> { - if (signal.aborted) return { error: "The desktop call ended before availability could be read." }; - const deadline = new AbortController(); - const timer = setTimeout(() => deadline.abort(), 2500); - try { - const context = AbortSignal.any([signal, deadline.signal]); - const results = await Promise.allSettled([ - native(["apps"], context), - native(["windows", String(request.pid)], context), - ]); - const result: Record = { observed_at: new Date().toISOString() }; - for ( - const [index, field, key, id, fields] of [ - [0, "apps", "pid", request.pid, [ - "pid", - "is_active", - "is_active_known", - "is_hidden", - "is_hidden_known", - "process_start_identity_decimal", - ]], - [1, "windows", "window_id", request.window, [ - "window_id", - "bounds", - "is_on_screen", - "is_minimized", - "is_key", - "observation_capability", - "observation_reason", - "process_start_identity_decimal", - ]], - ] as const - ) { - const value = results[index]; - if (value.status === "rejected") { - result[field] = { error: String(value.reason).slice(0, 500) }; - continue; - } - const items = value.value[field]; - if (!Array.isArray(items)) { - result[field] = { error: "Native inventory returned no items." }; - continue; - } - const item = items.find((item) => item[key] === id); - result[field] = { - inventory_completeness: value.value.inventory_completeness, - target: item ? Object.fromEntries(fields.map((field) => [field, item[field]])) : null, - }; - } - return result; - } finally { - clearTimeout(timer); - } -} - -function validateApplication(application: DesktopApplication) { - if (!application || typeof application !== "object" || Object.keys(application).length !== 1) { - throw new Error("Use exactly one application path or bundle_id."); - } - if ("path" in application) { - const path = application.path; - if ( - typeof path !== "string" || path.length > 4096 || !path.startsWith("/") - || !path.toLowerCase().endsWith(".app") || path.includes("\0") - ) throw new Error("Use an absolute .app path for launch."); - } else if ( - !("bundle_id" in application) || typeof application.bundle_id !== "string" - || application.bundle_id.length > 256 - || !/^[A-Za-z0-9.-]+$/.test(application.bundle_id) - ) throw new Error("Use an exact application bundle ID for launch."); -} - -function validateAction(request: DesktopAction) { - if (request.op === "launch") { - if (Object.keys(request).some((key) => !["op", "application"].includes(key))) { - throw new Error("Launch accepts exactly one application path or bundle_id."); - } - validateApplication(request.application); - return; - } - if (request.op === "open") { - const item = request.item; - if ( - Object.keys(request).some((key) => !["op", "item", "application"].includes(key)) - || !item || typeof item !== "object" || Object.keys(item).length !== 1 - ) throw new Error("Open accepts exactly one item path or url and an optional application."); - if ("path" in item) { - if ( - typeof item.path !== "string" || item.path.length > 4096 - || !item.path.startsWith("/") || item.path.includes("\0") - ) throw new Error("Use an existing absolute item path."); - } else if ( - !("url" in item) || typeof item.url !== "string" || item.url.length > 4096 - || !/^[A-Za-z][A-Za-z0-9+.-]*:/.test(item.url) - || /\p{Cc}/u.test(item.url) - ) { - throw new Error( - "Use a complete absolute URL with an explicit scheme and no control characters.", - ); - } - if (request.application !== undefined) validateApplication(request.application); - if (Buffer.byteLength(JSON.stringify(request)) > 16_384) { - throw new Error("The open request exceeds 16 KiB."); - } - return; - } - if (request.op === "menu") { - validateAppOnly(request.target); - validateMenuPath(request.path); - if (Buffer.byteLength(JSON.stringify(request)) > 4096) { - throw new Error("Menu request exceeds the 4096-byte limit."); - } - return; - } - if (isDesktopManagement(request)) return validateManagement(request); - if (request.op === "clipboard-write") { - if ("format" in request) { - if (request.format === "files") { - if ( - "text" in request || "path" in request || !Array.isArray(request.paths) - || !request.paths.length || request.paths.length > 32 - || request.paths.some((path) => - typeof path !== "string" || !path.startsWith("/") || path.length > 4096 - || path.includes("\0") - ) - ) throw new Error("Use format files with 1–32 absolute paths and no text or image path."); - return; - } - if ( - request.format !== "image" || "text" in request || "paths" in request - || typeof request.path !== "string" - || !request.path.startsWith("/") || request.path.length > 4096 - || request.path.includes("\0") - ) throw new Error("Use format image with one absolute image path and no text."); - return; - } - if ( - "path" in request || "paths" in request || typeof request.text !== "string" - || request.text.length > 8192 - ) { - throw new Error("Clipboard text must contain at most 8192 UTF-16 code units."); - } - return; - } - if (typeof request.snapshot !== "string" || !request.snapshot || request.snapshot.length > 256) { - throw new Error("Use the snapshot_id from a fresh desktop_inspect result."); - } - if (request.op === "drag") { - validatePoint(request.from); - validatePoint(request.to); - if (request.from.x === request.to.x && request.from.y === request.to.y) { - throw new Error("Choose distinct start and end points for a drag."); - } - if (request.button !== undefined && !DESKTOP_BUTTONS.includes(request.button)) { - throw new Error("Choose the left or right mouse button for a drag."); - } - if ( - request.duration_ms !== undefined - && (!Number.isInteger(request.duration_ms) || request.duration_ms < 1 - || request.duration_ms > 10000) - ) { - throw new Error("Drag duration must be 1 to 10000 milliseconds."); - } - return; - } - if (request.op === "click" || request.op === "scroll") { - if ((request.element === undefined) === (request.point === undefined)) { - throw new Error("Choose exactly one observed element ID or normalized screenshot point."); - } - if (request.point !== undefined) validatePoint(request.point); - else if ( - typeof request.element !== "string" || !request.element || request.element.length > 256 - ) { - throw new Error("Choose an element ID from the inspected snapshot."); - } - if (request.op === "click") { - if (request.kind !== undefined && !DESKTOP_CLICKS.includes(request.kind)) { - throw new Error("Choose single, double, right, middle, or triple click."); - } - } else if ( - !DESKTOP_DIRECTIONS.includes(request.direction) || !Number.isInteger(request.amount) - || request.amount < 1 || request.amount > 20 - ) { - throw new Error("Choose up, down, left, or right and 1 to 20 native scroll units."); - } - return; - } - if (request.op === "key") { - if (!DESKTOP_KEYS.includes(request.key)) throw new Error("Choose one supported key."); - const modifiers = request.modifiers; - if ( - modifiers !== undefined && ( - !Array.isArray(modifiers) || modifiers.length > 4 - || new Set(modifiers).size !== modifiers.length - || modifiers.some((modifier) => !DESKTOP_MODIFIERS.includes(modifier)) - ) - ) { - throw new Error("Use each of command, control, option, and shift at most once."); - } - return; - } - if (request.op === "insert") { - if (typeof request.text !== "string" || !request.text || request.text.length > 8192) { - throw new Error("Inserted text must contain 1 to 8192 UTF-16 code units."); - } - return; - } - if (typeof request.element !== "string" || !request.element || request.element.length > 256) { - throw new Error("Choose an element ID from the inspected snapshot."); - } - if (request.op === "type" && (typeof request.text !== "string" || request.text.length > 8192)) { - throw new Error("Replacement text must contain at most 8192 UTF-16 code units."); - } - if (request.op === "select") { - if (typeof request.text !== "string" || !request.text || request.text.length > 4096) { - throw new Error("Selection text must contain 1 to 4096 UTF-16 code units."); - } - for (const context of [request.prefix, request.suffix]) { - if (context !== undefined && (typeof context !== "string" || context.length > 2048)) { - throw new Error("Selection context must contain at most 2048 UTF-16 code units."); - } - } - if (request.selection !== undefined && !DESKTOP_SELECTIONS.includes(request.selection)) { - throw new Error("Choose text, cursor_before, or cursor_after for selection."); - } - } -} - -function validateMenuPath(path: unknown) { - if ( - !Array.isArray(path) || !path.length || path.length > 8 - || path.some((title) => typeof title !== "string" || !title.trim() || title.length > 512) - ) { - throw new Error( - "Menu path requires 1 to 8 nonblank literal titles of at most 512 UTF-16 code units each.", - ); - } -} - -function validateAppTarget(target: DesktopAppTarget) { - if ( - !target || typeof target !== "object" || !Number.isInteger(target.pid) - || target.pid < 1 || target.pid > 2_147_483_647 - || typeof target.process_start_identity_decimal !== "string" - || !/^[1-9][0-9]{0,19}$/.test(target.process_start_identity_decimal) - || BigInt(target.process_start_identity_decimal) > 18_446_744_073_709_551_615n - ) throw new Error("Pass the application's target object from fresh desktop inventory unchanged."); -} - -function validateAppOnly(target: DesktopAppTarget) { - validateAppTarget(target); - if (Object.keys(target).some((key) => !["pid", "process_start_identity_decimal"].includes(key))) { - throw new Error("Pass only the application's target object from desktop_apps."); - } -} - -function validateManagement(request: DesktopManagement) { - const target = request.target; - if (request.op === "activate" || request.op === "quit") { - validateAppOnly(target); - return; - } - validateAppTarget(target); - const window = request.target; - if ( - !Number.isInteger(window.window_id) || window.window_id < 1 || window.window_id > 4_294_967_295 - || typeof window.is_minimized !== "boolean" || !window.bounds - || ![window.bounds.x, window.bounds.y, window.bounds.width, window.bounds.height].every( - Number.isFinite, - ) - || window.bounds.width <= 0 || window.bounds.height <= 0 - ) { - throw new Error( - "Pass the window's target object, including its original bounds, from desktop_windows unchanged.", - ); - } - if (request.op === "move") { - const position = request.position; - if (!position || ![position.x, position.y].every(Number.isFinite)) { - throw new Error("Window position must contain finite x and y in desktop logical points."); - } - } - if (request.op === "resize") { - const size = request.size; - if ( - !size || ![size.width, size.height].every(Number.isFinite) || size.width <= 0 - || size.height <= 0 - ) { - throw new Error( - "Window size must contain positive finite width and height in desktop logical points.", - ); - } - } -} - -function actionResult(data: Record, outcome: DesktopOutcome): DesktopResult { - let text = JSON.stringify({ action: data }); - if (Buffer.byteLength(text) > MAX_TEXT) { - text = JSON.stringify({ - action: { - outcome, - target_receipt: data.target_receipt, - terminated: data.terminated, - clipboard_changed: data.clipboard_changed, - clipboard_cleanup: data.clipboard_cleanup, - clipboard_ownership: data.clipboard_ownership, - consumption: data.consumption, - }, - warning: "Native action metadata exceeded the result limit. Inspect the current state.", - }); - } - return { text, outcome, isError: outcome !== "completed" }; -} - -async function clipboardImageInput(path: string, signal: AbortSignal): Promise { - signal.throwIfAborted(); - // Nonblocking open prevents a named pipe from waiting before its regular-file check. - const file = await open(path, constants.O_RDONLY | constants.O_NONBLOCK); - try { - signal.throwIfAborted(); - const before = await file.stat({ bigint: true }); - if (!before.isFile() || before.size <= 0n || before.size > BigInt(MAX_CLIPBOARD_IMAGE)) { - throw new Error("Clipboard images require a nonempty regular file of at most 10 MiB."); - } - const bytes = Buffer.alloc(Number(before.size) + 1); - let size = 0; - while (size < bytes.length) { - signal.throwIfAborted(); - const read = await file.read(bytes, size, bytes.length - size, size); - if (!read.bytesRead) break; - size += read.bytesRead; - } - const after = await file.stat({ bigint: true }); - signal.throwIfAborted(); - if ( - size !== Number(before.size) || after.size !== before.size - || after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs - ) { - throw new Error( - "The image file changed while being read; choose a stable file before writing.", - ); - } - return bytes.subarray(0, size).toString("base64"); - } finally { - await file.close(); - } -} - -async function act(request: DesktopAction, signal: AbortSignal): Promise { - let dispatched = false; - let data: Record; - try { - validateAction(request); - if (request.op === "clipboard-write" && "format" in request && request.format === "files") { - for (const path of request.paths) { - signal.throwIfAborted(); - const entry = await lstat(path); - if (!entry.isFile() && !entry.isDirectory() && !entry.isSymbolicLink()) { - throw new Error( - "Clipboard file references require existing files, directories, or symbolic links.", - ); - } - } - signal.throwIfAborted(); - } - const input = - request.op === "clipboard-write" && "format" in request && request.format === "image" - ? { - op: request.op, - format: request.format, - image: await clipboardImageInput(request.path, signal), - } - : request; - const operation = request.op === "menu" - ? "menu" - : request.op === "clipboard-write" - ? "clipboard" - : request.op === "launch" - ? "launch" - : request.op === "open" - ? "open" - : isDesktopManagement(request) - ? "management" - : "action"; - data = await native([operation], signal, { - input: JSON.stringify(input), - onDispatch() { - dispatched = true; - }, - }); - if (!["completed", "refused", "unknown"].includes(String(data.outcome))) { - throw new Error("The Ace native client returned no action outcome."); - } - } catch (error) { - const outcome = dispatched ? "unknown" : "refused"; - return actionResult({ - outcome, - reason: (error instanceof Error ? error.message : String(error)).slice(0, 2000), - message: dispatched && request.op === "launch" - ? "The launch may still finish and open the app later. Observe desktop_apps before any further action; do not blindly repeat the launch." - : dispatched && request.op === "open" - ? "The item may still open later. Observe desktop_apps before any further action; do not blindly repeat the open." - : dispatched - ? "The desktop action may have partially run. Inspect the current state before retrying. Stopping does not undo input already delivered." - : "The desktop action was not sent to the native desktop.", - }, outcome); - } - const outcome = data.outcome as DesktopOutcome; - if (request.op === "menu") { - const receipt = data.target_receipt as Reply["target_receipt"]; - if ( - (outcome === "completed" || receipt) && (!receipt || receipt.window_id !== undefined - || receipt.pid !== request.target.pid - || receipt.process_start_identity_decimal !== request.target.process_start_identity_decimal) - ) { - return actionResult({ - ...data, - outcome: "unknown", - receipt_error: - "The menu command returned a different application receipt. Observe the intended app before any further action.", - }, "unknown"); - } - return actionResult(data, outcome); - } - - if ( - outcome === "unknown" - && ((request.op === "quit" && data.terminated === false) - || (request.op === "close" && data.target_receipt)) - ) { - return await observeManagement(request, data, signal); - } - if (outcome !== "completed" || request.op === "clipboard-write") { - return actionResult(data, outcome); - } - if (isDesktopManagement(request) || request.op === "launch" || request.op === "open") { - return await observeManagement(request, data, signal); - } - // Observation is separate from delivery: its failure must not turn completed input into a retry. - try { - const receipt = data.target_receipt as Reply["target_receipt"]; - if (!receipt?.process_start_identity_decimal || !receipt.window_id) { - throw new Error("No exact-window action receipt."); - } - const observation = await inspect({ - op: "inspect", - pid: receipt.pid, - window: receipt.window_id, - }, signal); - const fresh = JSON.parse(observation.text) as Record; - if (observation.isError) { - return actionResult({ - ...data, - observation_error: fresh, - message: - "The action completed, but a fresh observation was unavailable. Inspect again to verify the result; do not repeat the action blindly.", - }, outcome); - } - const current = fresh.target_receipt as Reply["target_receipt"]; - if (current?.process_start_identity_decimal !== receipt.process_start_identity_decimal) { - throw new Error("The target application changed after the action."); - } - return { - text: bounded({ ...fresh, action: data }, "ui_elements"), - image: observation.image, - outcome, - isError: false, - }; - } catch (error) { - return actionResult({ - ...data, - observation_error: (error instanceof Error ? error.message : String(error)).slice(0, 2000), - message: - "The action completed, but a fresh observation was unavailable. Inspect again to verify the result; do not repeat the action blindly.", - }, outcome); - } -} - -async function observeManagement( - request: DesktopManagement | DesktopLaunch | DesktopOpen, - action: Record, - signal: AbortSignal, -): Promise { - const data: Record = { action }; - const outcome = action.outcome === "unknown" ? "unknown" : "completed"; - const receipt = action.target_receipt as Reply["target_receipt"]; - const target = request.op === "launch" || request.op === "open" - ? (action.application as { target?: DesktopAppTarget } | undefined)?.target - : request.target; - if ( - !receipt || !target || receipt.pid !== target.pid - || receipt.process_start_identity_decimal !== target.process_start_identity_decimal - || (request.op === "activate" || request.op === "quit" || request.op === "launch" - || request.op === "open" - ? receipt.window_id !== undefined - : receipt.window_id !== request.target.window_id) - ) { - return actionResult({ - ...action, - outcome: "unknown", - reason: - "The native action returned a different target receipt. Refresh the target before any retry.", - }, "unknown"); - } - try { - const apps = await native(["apps"], signal); - if (!Array.isArray(apps.apps)) throw new Error("Native application inventory is unavailable."); - data.application_inventory_completeness = apps.inventory_completeness; - data.application_inventory_warnings = apps.inventory_warnings; - const app = apps.apps.find((app) => app.pid === receipt.pid); - data.application = app || null; - // The native receipt owns close/quit evidence; later inventory cannot undo or establish it. - if (!app && (request.op === "quit" || request.op === "close")) { - return managementResult(data, action, outcome); - } - if (!app) throw new Error("The target application was not returned by the later inventory."); - if (app.process_start_identity_decimal !== receipt.process_start_identity_decimal) { - throw new Error("The application changed process generation after the action."); - } - const windows = await native(["windows", String(receipt.pid)], signal); - if (!Array.isArray(windows.windows)) throw new Error("Native window inventory is unavailable."); - data.window_inventory_completeness = windows.inventory_completeness; - data.window_inventory_warnings = windows.inventory_warnings; - data.windows = windows.windows; - if ( - windows.windows.some((window) => - window.process_start_identity_decimal !== receipt.process_start_identity_decimal - ) - ) { - throw new Error( - "Later window inventory could not be bound to the original application generation.", - ); - } - if ( - request.op === "activate" || request.op === "quit" || request.op === "launch" - || request.op === "open" - || request.op === "close" - ) { - return managementResult(data, action, outcome); - } - if (!windows.windows.some((window) => window.window_id === receipt.window_id)) { - throw new Error("The exact window was not returned by the later inventory."); - } - // Minimization intentionally removes the visible capture target; refreshed inventory owns its state. - if (request.op === "minimize") return managementResult(data, action); - const observation = await inspect({ - op: "inspect", - pid: receipt.pid, - window: receipt.window_id!, - }, signal); - const fresh = JSON.parse(observation.text) as Record; - if (observation.isError) { - data.observation_error = fresh; - data.message = - "The native action completed. Later inventory or inspection was unavailable; refresh the target before any further action, without repeating the completed action blindly."; - return managementResult(data, action); - } - const current = fresh.target_receipt as Reply["target_receipt"]; - if (current?.process_start_identity_decimal !== receipt.process_start_identity_decimal) { - throw new Error("The application changed process generation before the later inspection."); - } - return { - text: bounded({ ...data, ...fresh }, "ui_elements"), - image: observation.image, - outcome: "completed", - isError: false, - }; - } catch (error) { - data.observation_error = (error instanceof Error ? error.message : String(error)).slice( - 0, - 2000, - ); - data.message = outcome === "completed" - ? "The native action completed. Later inventory or inspection was unavailable; refresh the target before any further action, without repeating the completed action blindly." - : "Native completion was not confirmed and later inventory was unavailable. Refresh the target before choosing any further action; do not blindly repeat the request."; - return managementResult(data, action, outcome); - } -} - -function managementResult( - data: Record, - action: Record, - outcome: "completed" | "unknown" = "completed", -): DesktopResult { - try { - const text = Array.isArray(data.windows) ? bounded(data, "windows") : JSON.stringify(data); - if (Buffer.byteLength(text) > MAX_TEXT) { - throw new Error("The later inventory exceeds the result limit."); - } - return { text, outcome, isError: outcome !== "completed" }; - } catch { - return actionResult({ - ...action, - observation_error: data.observation_error || "The later inventory exceeds the result limit.", - message: - "The native action outcome is preserved. Later inventory was omitted to fit the result limit; refresh the target without blindly repeating the action.", - }, outcome); - } -} - -export const desktop: Desktop = async (request, context) => { - if ( - !request - || ![ - "clipboard-read", - "clipboard-write", - "apps", - "windows", - "menus", - "menu", - "inspect", - "click", - "type", - "key", - "insert", - "select", - "scroll", - "drag", - "activate", - "launch", - "open", - "quit", - "close", - "focus", - "minimize", - "restore", - "move", - "resize", - ].includes( - request.op, - ) - ) { - throw new Error("Unknown native desktop request."); - } - if (process.platform !== "darwin") { - const reason = "Native desktop tools require macOS 15 or later."; - if (isDesktopAction(request)) return actionResult({ outcome: "refused", reason }, "refused"); - throw new Error(reason); - } - const deadline = new AbortController(); - const timer = setTimeout( - () => deadline.abort(new Error("Native desktop operation timed out after 30 seconds.")), - 30_000, - ); - const signal = context.abortSignal - ? AbortSignal.any([context.abortSignal, deadline.signal]) - : deadline.signal; - try { - if (isDesktopAction(request)) return await act(request, signal); - if (request.op === "inspect") return await inspect(request, signal); - if (request.op === "menus") { - validateAppOnly(request.target); - if (request.path !== undefined) validateMenuPath(request.path); - const input = JSON.stringify(request); - if (Buffer.byteLength(input) > 4096) { - throw new Error("Menu request exceeds the 4096-byte limit."); - } - const data = await native(["menus"], signal, { input }); - const target = data.target as DesktopAppTarget | undefined; - if ( - target?.pid !== request.target.pid - || target.process_start_identity_decimal !== request.target.process_start_identity_decimal - ) throw new Error("Native menu inventory returned a different application generation."); - if (request.path !== undefined) { - // An older client must not turn a scoped read into full menu disclosure. - const path = request.path, filter = data.filter as { path?: unknown } | undefined; - if ( - !Array.isArray(filter?.path) || filter.path.length !== path.length - || filter.path.some((title, index) => title !== path[index]) - || !Array.isArray(data.menus) || data.menus.some((row) => - !Array.isArray(row?.path) || path.some((title, index) => - row.path[index] !== title - ) - ) - ) throw new Error("Native menu inventory did not honor the requested literal path."); - } - return { text: bounded(data, "menus") }; - } - if (request.op === "clipboard-read") { - if ( - request.format !== undefined && request.format !== "text" && request.format !== "image" - && request.format !== "files" - ) { - throw new Error("Choose text, image, or files clipboard format."); - } - const data = await native(["clipboard"], signal, { input: JSON.stringify(request) }); - if (request.format === "image") return clipboardImage(data); - if (request.format === "files") return clipboardFiles(data); - if ( - typeof data.present !== "boolean" || !Number.isSafeInteger(data.change_count) - || (data.present ? typeof data.text !== "string" : data.text !== undefined) - ) throw new Error("The native clipboard read returned an unsupported response."); - const text = JSON.stringify(data); - if (Buffer.byteLength(text) > 24_000) { - throw new Error("Clipboard text exceeds the complete 24 KB result limit."); - } - return { text }; - } - let query: string | undefined; - if (request.op === "apps" && request.query !== undefined) { - if ( - typeof request.query !== "string" || request.query.length > 256 - || !request.query.trim() - ) { - throw new Error( - "Application query must contain non-whitespace text and at most 256 characters.", - ); - } - query = request.query.trim(); - } - const args = request.op === "apps" - ? ["apps"] - : ["windows", positive(request.pid)]; - const data = await native(args, signal); - if (query !== undefined) { - if (!Array.isArray(data.apps)) { - throw new Error("Native application inventory is unavailable."); - } - const apps = data.apps; - const search = query.toLowerCase(); - const matches = apps.filter((app) => - app.name.toLowerCase().includes(search) || app.bundle_id?.toLowerCase().includes(search) - ); - data.apps = matches; - data.filter = { - query, - fields: ["name", "bundle_id"], - scope: "returned_native_inventory", - total: apps.length, - matched: matches.length, - }; - } - return { text: bounded(data, request.op === "apps" ? "apps" : "windows") }; - } finally { - clearTimeout(timer); - } }; diff --git a/bun.lock b/bun.lock index 0aa187d..b230c94 100644 --- a/bun.lock +++ b/bun.lock @@ -38,9 +38,10 @@ }, "apps/desktop": { "name": "@ace/desktop", - "version": "0.0.3", + "version": "0.0.21", "dependencies": { "@ace/host": "workspace:*", + "@githubnext/desktop-tools": "catalog:desktop", "electrobun": "catalog:desktop", }, "devDependencies": { @@ -58,6 +59,7 @@ "@earendil-works/chord": "catalog:pi", "@earendil-works/pi-ai": "catalog:pi", "@earendil-works/pi-durable": "catalog:pi", + "@githubnext/desktop-tools": "catalog:desktop", }, "devDependencies": { "@types/bun": "catalog:bun", @@ -69,6 +71,7 @@ "@earendil-works/chord": "catalog:pi", "@earendil-works/pi-ai": "catalog:pi", "@earendil-works/pi-durable": "catalog:pi", + "@githubnext/desktop-tools": "catalog:desktop", }, }, "packages/split-tabs": { @@ -153,6 +156,7 @@ "wrangler": "4.140.0", }, "desktop": { + "@githubnext/desktop-tools": "github:githubnext/desktop-tools#021f2460508455a6569187692796e5986e2f29d4", "@types/three": "0.165.0", "electrobun": "1.18.1", }, @@ -449,6 +453,8 @@ "@fontsource-variable/inter": ["@fontsource-variable/inter@5.2.8", "", {}, "sha512-kOfP2D+ykbcX/P3IFnokOhVRNoTozo5/JxhAIVYLpea/UBmCQ/YWPBfWIDuBImXX/15KH+eKh4xpEUyS2sQQGQ=="], + "@githubnext/desktop-tools": ["@githubnext/desktop-tools@github:githubnext/desktop-tools#021f246", { "bin": { "desktop-tools-build": "dist/build.js" } }, "githubnext-desktop-tools-021f246", "sha512-y7janCtEljRG//Af59kT241aucev60ipZz/lk6M4XqnPP4oQXXrO/gNvnZgtVrCmlVEqFuWGAK/friljSzotqA=="], + "@google/genai": ["@google/genai@2.21.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-+PDtco2/Z0ONdzCGekCoCT+O1VJS9xJQNN4XzQpXG/t3El/SWWMkCWlFRO1KmivOHPa4Q0VjUYu1HBKCZ/v33Q=="], "@hono/node-server": ["@hono/node-server@2.1.3", "", { "peerDependencies": { "hono": "^4" } }, "sha512-TA//nWMqPhbfdfneACk6t5a9eqbS9lABEPyKn0/xZTah3H3U2XaVg85rJFl0/Fyit0I552YDHgXGVSf3GwqbUw=="], diff --git a/docs/architecture.md b/docs/architecture.md index 19815cf..532bd11 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -249,7 +249,9 @@ boundary as provider settings. The project picker selects a local Git checkout; errors remain in the form so the person can fix the path or provider setup. Agents inspect and operate native windows through an injected desktop capability. The desktop embeds -Peekaboo's native bridge inside Ace's UI process, which owns macOS Accessibility, Screen +[desktop-tools](https://github.com/githubnext/desktop-tools), a standalone package carrying a pinned, +patched Peekaboo bridge, its signed client, and the request/result contract, inside Ace's UI process, +which owns macOS Accessibility, Screen Recording, and Event Synthesizing permissions. The host invokes a bundled client over a local Unix socket. The bridge accepts only the client's exact identifier signed by Ace's team; the client verifies the host's signing team. Tools expose application and window inventories, observation, element or screenshot diff --git a/docs/desktop-tools.md b/docs/desktop-tools.md index 9d04515..223682d 100644 --- a/docs/desktop-tools.md +++ b/docs/desktop-tools.md @@ -18,8 +18,9 @@ CGEvent programs bypass the tools' snapshot and exact-target checks even without ## Setup -Native desktop tools require macOS 15 or later. Ace embeds Peekaboo's native library; there is no -separate Peekaboo installation or permission grant. In Ace Settings, open This Mac and enable +Native desktop tools require macOS 15 or later. Ace embeds the native library from +[desktop-tools](https://github.com/githubnext/desktop-tools), which carries Peekaboo's bridge and +Ace's patches; there is no separate Peekaboo installation or permission grant. In Ace Settings, open This Mac and enable Accessibility and Screen Recording. macOS grants those permissions to Ace on that machine. Keyboard and pointer event delivery also require Event Synthesizing, which Ace reports and requests separately. Clicking Accessibility controls, selecting text, and replacing field values use Accessibility permission. diff --git a/docs/updates.md b/docs/updates.md index 30da68c..ce9bf05 100644 --- a/docs/updates.md +++ b/docs/updates.md @@ -63,13 +63,16 @@ application from Electrobun's build archive; its self-extracting wrapper and `Up are unused. The Sparkle SDK and Electrobun CLI downloads are pinned by version and SHA-256. Native desktop -inspection builds the Swift package in `apps/desktop/native` with Swift 6.2 or newer; CI selects -Xcode 26.2. `Package.resolved` pins Peekaboo 4.8.0 and its dependencies, and ordinary builds require -those resolved versions. To intentionally update the lockfile, run -`xcrun swift package --package-path apps/desktop/native resolve` and commit the resulting file. - -The bundle contains `libAceDesktop.dylib` and `ace-desktop-client`, with needed Swift runtime -libraries discovered from their compiled dependencies. It does not ship Peekaboo's standalone app +inspection comes from [desktop-tools](https://github.com/githubnext/desktop-tools), pinned to one +commit in the root `desktop` catalog. Its build tool resolves Peekaboo 4.8.0 and its dependencies +from the package's `Package.resolved`, applies its pinned patch stack, and builds with Swift 6.2 or +newer in `apps/desktop/native/.build/desktop-tools`; CI selects Xcode 26.2. Ace builds only its +project picker, `libAceProject.dylib`, from `apps/desktop/native/project.swift`. Update the native +code in that repository, then move the catalog pin. + +The bundle contains `libDesktopTools.dylib`, `ace-desktop-client` (the package's +`desktop-tools-client`), and `libAceProject.dylib`, with needed Swift runtime libraries discovered +from their compiled dependencies. It does not ship Peekaboo's standalone app or CLI. Sparkle's license and the native dependencies' licenses and notices are included in Resources. Code is signed inside out, including the Swift runtimes, Ace Helper, and Sparkle's nested installers. The desktop client uses the app's actual bundle identifier followed by `.desktop-client`; diff --git a/package.json b/package.json index 4f83ded..b492d0c 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "@tailwindcss/vite": "4.3.0" }, "desktop": { + "@githubnext/desktop-tools": "github:githubnext/desktop-tools#021f2460508455a6569187692796e5986e2f29d4", "electrobun": "1.18.1", "@types/three": "0.165.0" } diff --git a/packages/channel/package.json b/packages/channel/package.json index 2617f4f..5c09671 100644 --- a/packages/channel/package.json +++ b/packages/channel/package.json @@ -16,6 +16,7 @@ "types": "tsgo --noEmit" }, "dependencies": { + "@githubnext/desktop-tools": "catalog:desktop", "@earendil-works/chord": "catalog:pi", "@earendil-works/pi-ai": "catalog:pi", "@earendil-works/pi-durable": "catalog:pi" diff --git a/packages/channel/src/desktop.ts b/packages/channel/src/desktop.ts index 8c71c59..808fb30 100644 --- a/packages/channel/src/desktop.ts +++ b/packages/channel/src/desktop.ts @@ -7,209 +7,22 @@ import { type ToolExecutionApi, type ToolExecutionResult, } from "@earendil-works/pi-durable"; +import { + DESKTOP_BUTTONS, + DESKTOP_CLICKS, + DESKTOP_DIRECTIONS, + DESKTOP_KEYS, + DESKTOP_MODIFIERS, + DESKTOP_SELECTIONS, + type DesktopAction, + type DesktopRequest, + type DesktopResult, +} from "@githubnext/desktop-tools/protocol"; -import type { Image } from "./protocol"; - -export type DesktopAppTarget = { - pid: number; - process_start_identity_decimal: string; -}; -export type DesktopWindowTarget = DesktopAppTarget & { - window_id: number; - bounds: { x: number; y: number; width: number; height: number }; - is_minimized: boolean; -}; -export type DesktopManagement = - | { op: "activate"; target: DesktopAppTarget } - | { op: "quit"; target: DesktopAppTarget } - | { op: "focus" | "minimize" | "restore" | "close"; target: DesktopWindowTarget } - | { op: "move"; target: DesktopWindowTarget; position: { x: number; y: number } } - | { op: "resize"; target: DesktopWindowTarget; size: { width: number; height: number } }; -export type DesktopApplication = { path: string } | { bundle_id: string }; -export type DesktopLaunch = { - op: "launch"; - application: DesktopApplication; -}; -export type DesktopOpen = { - op: "open"; - item: { path: string } | { url: string }; - application?: DesktopApplication; -}; - -export type DesktopRequest = - | DesktopManagement - | DesktopLaunch - | DesktopOpen - | { op: "clipboard-read"; format?: "text" | "image" | "files" } - | { op: "clipboard-write"; text: string } - | { op: "clipboard-write"; format: "image"; path: string } - | { op: "clipboard-write"; format: "files"; paths: string[] } - | { op: "apps"; query?: string } - | { op: "windows"; pid: number } - | { op: "menus"; target: DesktopAppTarget; path?: string[] } - | { op: "menu"; target: DesktopAppTarget; path: string[] } - | { op: "inspect"; pid: number; window: number; mode?: "accessibility" | "pixels" } - | { - op: "click"; - snapshot: string; - element?: string; - point?: DesktopPoint; - kind?: DesktopClick; - } - | { - op: "scroll"; - snapshot: string; - element?: string; - point?: DesktopPoint; - direction: DesktopDirection; - amount: number; - } - | { - op: "drag"; - snapshot: string; - from: DesktopPoint; - to: DesktopPoint; - button?: DesktopButton; - duration_ms?: number; - } - | { op: "type"; snapshot: string; element: string; text: string } - | { op: "insert"; snapshot: string; text: string } - | { - op: "select"; - snapshot: string; - element: string; - text: string; - prefix?: string; - suffix?: string; - selection?: DesktopSelection; - } - | { op: "key"; snapshot: string; key: DesktopKey; modifiers?: DesktopModifier[] }; - -export type DesktopPoint = { x: number; y: number }; -export const DESKTOP_CLICKS = ["single", "double", "right", "middle", "triple"] as const; -export const DESKTOP_DIRECTIONS = ["up", "down", "left", "right"] as const; -export const DESKTOP_BUTTONS = ["left", "right"] as const; -export type DesktopClick = (typeof DESKTOP_CLICKS)[number]; -export type DesktopDirection = (typeof DESKTOP_DIRECTIONS)[number]; -export type DesktopButton = (typeof DESKTOP_BUTTONS)[number]; - -export const DESKTOP_KEYS = [ - "enter", - "tab", - "escape", - "backspace", - "delete", - "up", - "down", - "left", - "right", - "space", - "home", - "end", - "pageup", - "pagedown", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "f1", - "f2", - "f3", - "f4", - "f5", - "f6", - "f7", - "f8", - "f9", - "f10", - "f11", - "f12", -] as const; -export const DESKTOP_MODIFIERS = ["command", "control", "option", "shift"] as const; -export const DESKTOP_SELECTIONS = ["text", "cursor_before", "cursor_after"] as const; -export type DesktopKey = (typeof DESKTOP_KEYS)[number]; -export type DesktopModifier = (typeof DESKTOP_MODIFIERS)[number]; -export type DesktopSelection = (typeof DESKTOP_SELECTIONS)[number]; -export type DesktopAction = Extract< - DesktopRequest, - { - op: - | "click" - | "type" - | "key" - | "insert" - | "select" - | "scroll" - | "drag" - | "activate" - | "launch" - | "open" - | "quit" - | "close" - | "focus" - | "minimize" - | "restore" - | "move" - | "resize" - | "clipboard-write" - | "menu"; - } ->; -export type DesktopOutcome = "completed" | "refused" | "unknown"; -export type DesktopResult = { - text: string; - image?: Image; - outcome?: DesktopOutcome; - isError?: boolean; -}; +// The request/result contract lives in the standalone package; pi tools and durability stay here. +export * from "@githubnext/desktop-tools/protocol"; export type Desktop = (request: DesktopRequest, context: Context) => Promise; -export function isDesktopAction(request: DesktopRequest): request is DesktopAction { - return request.op === "click" || request.op === "type" || request.op === "key" - || request.op === "insert" || request.op === "select" || request.op === "scroll" - || request.op === "drag" || request.op === "clipboard-write" || request.op === "launch" - || request.op === "open" || request.op === "menu" - || isDesktopManagement(request); -} - -export function isDesktopManagement(request: DesktopRequest): request is DesktopManagement { - return request.op === "activate" || request.op === "quit" || request.op === "focus" - || request.op === "minimize" || request.op === "restore" || request.op === "close" - || request.op === "move" || request.op === "resize"; -} - export function result(value: DesktopResult): ToolExecutionResult { return { content: [ From 7ca86608bf0121c0f446038b1005930873f6a4ba Mon Sep 17 00:00:00 2001 From: iamnbutler Date: Thu, 8 Oct 2026 08:33:56 -0400 Subject: [PATCH 2/2] fix(desktop): pin desktop-tools with idempotent notice copies --- bun.lock | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bun.lock b/bun.lock index b230c94..378b94e 100644 --- a/bun.lock +++ b/bun.lock @@ -156,7 +156,7 @@ "wrangler": "4.140.0", }, "desktop": { - "@githubnext/desktop-tools": "github:githubnext/desktop-tools#021f2460508455a6569187692796e5986e2f29d4", + "@githubnext/desktop-tools": "github:githubnext/desktop-tools#8efe67fbd38ed293c8e76778f01814b3bd67c5db", "@types/three": "0.165.0", "electrobun": "1.18.1", }, @@ -453,7 +453,7 @@ "@fontsource-variable/inter": ["@fontsource-variable/inter@5.2.8", "", {}, "sha512-kOfP2D+ykbcX/P3IFnokOhVRNoTozo5/JxhAIVYLpea/UBmCQ/YWPBfWIDuBImXX/15KH+eKh4xpEUyS2sQQGQ=="], - "@githubnext/desktop-tools": ["@githubnext/desktop-tools@github:githubnext/desktop-tools#021f246", { "bin": { "desktop-tools-build": "dist/build.js" } }, "githubnext-desktop-tools-021f246", "sha512-y7janCtEljRG//Af59kT241aucev60ipZz/lk6M4XqnPP4oQXXrO/gNvnZgtVrCmlVEqFuWGAK/friljSzotqA=="], + "@githubnext/desktop-tools": ["@githubnext/desktop-tools@github:githubnext/desktop-tools#8efe67f", { "bin": { "desktop-tools-build": "dist/build.js" } }, "githubnext-desktop-tools-8efe67f", "sha512-aR+M+EwhRHpmOzQQVsQWlkHcdJoKUi6YcRfCXSfWyM2pqBXTn9z4LpTjuzcsMR9wvNZ1P53r/wrIECCWsz/eZg=="], "@google/genai": ["@google/genai@2.21.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-+PDtco2/Z0ONdzCGekCoCT+O1VJS9xJQNN4XzQpXG/t3El/SWWMkCWlFRO1KmivOHPa4Q0VjUYu1HBKCZ/v33Q=="], diff --git a/package.json b/package.json index b492d0c..2f98ee5 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ "@tailwindcss/vite": "4.3.0" }, "desktop": { - "@githubnext/desktop-tools": "github:githubnext/desktop-tools#021f2460508455a6569187692796e5986e2f29d4", + "@githubnext/desktop-tools": "github:githubnext/desktop-tools#8efe67fbd38ed293c8e76778f01814b3bd67c5db", "electrobun": "1.18.1", "@types/three": "0.165.0" }