Open the Bot's screen only when there is something to watch, and offer an unconnected app in the conversation

CHANGELOG.md

@@ -32,6 +32,38 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
   `code_required` (below). A server and a computer must be upgraded together: an older computer has
   no `/sign-in/code`.
 
+### The Bot's screen opens only when there is something to watch
+
+Asking a Bot a question it answered in words could still open its computer beside the conversation,
+on a still screen, for an action that never happened.
+
+- **A call that never reaches the screen no longer opens it.** Reading, writing or listing a file,
+  and running a command, happen on the computer rather than on its display. Only work that happens
+  on the screen opens it.
+- **A refused action takes its screen back.** The screen is announced before the action so a page
+  can be watched while it loads; an action a policy, a capability switch or a network rule refuses
+  never touches the browser, so the screen it opened closes again. A refusal does not close a screen
+  some earlier action in the same run legitimately opened, and an action that was permitted and
+  failed keeps its screen, because the page may have moved.
+- **A Bot is no longer given computer tools this deployment would refuse.** Admin → Controls already
+  decided whether anybody may use a Bot's browser (`Cloud browser use`), its files and shell
+  (`Cloud computer use`) and the network (`Cloud network access`), and the boundary enforced all
+  three — but the Bot was still offered the tools, so it reached for one, was refused, and spent the
+  turn saying so. The tools a switch turns off are now withheld from the Bot in the first place.
+
+### An app you have not connected is offered where you are reading
+
+A Bot asked to use an app whose account the person had not connected answered with the name of a
+screen somewhere else: "Connect it in Settings and ask again." The conversation now carries the
+thing itself — a card naming the app, with a Connect button.
+
+- Pressing Connect opens the vendor's consent in a **new tab**, so the conversation is not replaced
+  by a page load and the question is still there afterwards. The card watches for the account to
+  appear and says so only once the server lists it, never merely because somebody came back.
+- It covers an account that was never connected, one disconnected while a call was in flight, and
+  access that was withdrawn. An app shared across the deployment still says to ask an administrator,
+  since the person reading cannot connect it.
+
 ### A reply's tool calls fold into one line
 
 A Bot that searched, read two pages and checked Drive before answering drew four lines above its

app/src/components/channels/chat-transcript.tsx

@@ -59,6 +59,7 @@ import {
   toolResultFailed,
 } from "@/lib/plugins/tool-result";
 import { readTeamBotConsent } from "@/lib/team-bots";
+import { readConnectPrompt } from "@openbot/shared/connect-prompt";
 import { cn } from "@/lib/utils";
 import { VOICE_CHAT_ACTIVITY, type VoiceChatEntry } from "@/lib/voice/archive";
 import {
@@ -72,6 +73,7 @@ import {
   toVisibleChatItems,
 } from "./chat-messages";
 import type { QueuedMessage } from "./composer";
+import { ConnectAppCard } from "./connect-app-card";
 import { TeamBotConsentCard } from "./team-bot-consent";
 import { ToolRenderBoundary } from "./tool-boundary";
 import { ToolLine } from "./tool-line";
@@ -1371,6 +1373,9 @@ function ServerToolLine({ name, result }: { name: string; result?: string }) {
   // A Team Bot asking to use this person's own account draws its consent card, not the sentence.
   const consent = refused ? readTeamBotConsent(body) : null;
   if (consent) return <TeamBotConsentCard {...consent} />;
+  // An app this person has not connected draws the connect button, not the name of a screen.
+  const connect = refused ? readConnectPrompt(body) : null;
+  if (connect) return <ConnectAppCard {...connect} />;
   return (
     <ToolLine
       {...(detail ? { detail } : {})}

app/src/components/channels/connect-app-card.tsx

@@ -0,0 +1,172 @@
+import { useMutation, useQuery } from "@tanstack/react-query";
+import { Link } from "@tanstack/react-router";
+import { useEffect, useRef, useState } from "react";
+import { Button } from "@/components/ui/button";
+import { connectAccountMutationOptions } from "@/lib/plugins/mutations";
+import {
+  connectionsQueryOptions,
+  personalConnections,
+} from "@/lib/plugins/queries";
+import { queryClient } from "@/query-client";
+
+/**
+ * The app a Bot needed, offered where the person is reading, instead of the name of a screen.
+ *
+ * A NEW TAB RATHER THAN THIS ONE, and that is the whole shape of this card. A consent screen is the
+ * vendor's own page and the trip to it is a full navigation, so sending this tab would take the
+ * person out of the conversation they are in the middle of and land them, several redirects later,
+ * on the connected-accounts page with their question somewhere behind them. The flow finishes over
+ * there; this tab stays where it was and watches for the connection to appear.
+ *
+ * WATCHING, NOT TRUSTING. The return trip proves nothing — it is an ordinary redirect in a tab this
+ * one cannot read — so nothing here treats "they came back" as "it worked". The card polls this
+ * person's own connections while it waits and only says connected when the server lists one.
+ */
+export function ConnectAppCard({
+  serverId,
+  title,
+  message,
+}: {
+  serverId: string;
+  title: string;
+  message: string;
+}) {
+  const [waiting, setWaiting] = useState(false);
+  /** This app takes a key somebody types, so there is no page to send them to. */
+  const [needsKey, setNeedsKey] = useState(false);
+  /** The authorization url, when the browser refused to open the tab holding it. */
+  const [blockedUrl, setBlockedUrl] = useState<string | null>(null);
+  const connect = useMutation(connectAccountMutationOptions());
+  const connections = useQuery({
+    ...connectionsQueryOptions(),
+    // Only while a consent is open in the other tab; otherwise this is an ordinary cached read.
+    refetchInterval: waiting ? 2_000 : false,
+  });
+  /*
+   * A ROW IS NOT ACCESS, which is the whole reason `revoked` is read here. Withdrawing somebody's
+   * access leaves their row in place pointing at a revoked credential, so a card drawn over the
+   * "your access was withdrawn" refusal would have said the app was connected and taken away the
+   * button that is the only way to get it back.
+   */
+  const connected = personalConnections(
+    connections.data?.connections ?? [],
+  ).some((row) => row.serverId === serverId && row.revoked !== true);
+
+  /*
+   * The grants a Bot holds are read per conversation, so the turn that follows has to be asked
+   * against a store that knows about the new account. Invalidated once, on the edge.
+   */
+  const was = useRef(connected);
+  useEffect(() => {
+    if (connected && !was.current) {
+      setWaiting(false);
+      void queryClient.invalidateQueries();
+    }
+    was.current = connected;
+  }, [connected]);
+
+  /**
+   * The tab, opened on the click itself rather than on the answer to it.
+   *
+   * A BROWSER ONLY LETS A PAGE OPEN A TAB WHILE A CLICK IS STILL RECENT. Chrome's transient
+   * activation lasts about five seconds, and the request that mints the authorization url has to
+   * finish first — so on a slow deployment `window.open` was called with the activation already
+   * expired, the tab was blocked, and the card cheerfully said to finish in a tab that did not
+   * exist. The blank tab is claimed first and navigated once the url arrives.
+   *
+   * `noopener` CANNOT BE PASSED HERE: it makes `window.open` answer null, and null is a handle
+   * nothing can navigate later. The opener is severed on the handle instead, which is the same
+   * guarantee by a different route.
+   */
+  const claimTab = (): Window | null => {
+    const tab = window.open("", "_blank");
+    if (tab) tab.opener = null;
+    return tab;
+  };
+
+  const open = () => {
+    const tab = claimTab();
+    connect.mutate(serverId, {
+      onSuccess: (authorizationUrl) => {
+        /*
+         * No url means this app is connected by typing a key rather than by visiting a page — the
+         * route answers `{ fields }` and the mutation unwraps nothing. Saying so, with the page
+         * that has the form, beats a button that silently does nothing.
+         */
+        if (!authorizationUrl) {
+          tab?.close();
+          setNeedsKey(true);
+          return;
+        }
+        setWaiting(true);
+        if (tab) tab.location.replace(authorizationUrl);
+        // The tab was blocked. The url is still good, so offer it as something to click.
+        else setBlockedUrl(authorizationUrl);
+      },
+      onError: () => tab?.close(),
+    });
+  };
+
+  return (
+    <div
+      className="my-2 flex w-full max-w-2xl flex-col gap-3 rounded-xl border border-border p-4"
+      data-connect-app={serverId}
+    >
+      <div className="flex flex-col gap-1">
+        <p className="font-medium text-sm">Connect {title}</p>
+        <p className="text-muted-foreground text-sm">{message}</p>
+      </div>
+      {connected ? (
+        <p className="text-sm" role="status">
+          {title} is connected. Ask again and the Bot can use it.
+        </p>
+      ) : needsKey ? (
+        <p className="text-sm" role="status">
+          {title} is connected with a key you hold rather than by visiting a
+          page.{" "}
+          <Link
+            className="underline underline-offset-2"
+            params={{ key: serverId }}
+            to="/settings/connected-accounts/$key"
+          >
+            Enter it on {title}'s page
+          </Link>
+          , then ask again.
+        </p>
+      ) : (
+        <div className="flex flex-wrap items-center gap-3">
+          {/* The heading above already names the app; the button says what pressing it does. */}
+          <Button disabled={connect.isPending} onClick={open} size="sm">
+            {waiting ? `Waiting for ${title}…` : "Connect"}
+          </Button>
+          {waiting ? (
+            <p className="text-muted-foreground text-sm" role="status">
+              Finish in the tab that opened, then come back here.
+            </p>
+          ) : null}
+          {blockedUrl ? (
+            <p className="text-sm" role="status">
+              This browser blocked the tab.{" "}
+              <a
+                className="underline underline-offset-2"
+                href={blockedUrl}
+                rel="noreferrer"
+                target="_blank"
+              >
+                Open {title}'s sign-in
+              </a>
+              .
+            </p>
+          ) : null}
+          {connect.isError ? (
+            <p className="text-destructive text-sm" role="status">
+              {connect.error instanceof Error
+                ? connect.error.message
+                : `${title} could not be connected.`}
+            </p>
+          ) : null}
+        </div>
+      )}
+    </div>
+  );
+}

app/src/components/channels/tool-activity.tsx

@@ -15,6 +15,7 @@ import {
   toolResultFailed,
 } from "@/lib/plugins/tool-result";
 import { readTeamBotConsent } from "@/lib/team-bots";
+import { readConnectPrompt } from "@openbot/shared/connect-prompt";
 import { researchRoleOfTool } from "@openbot/shared/research";
 import { browserStepDetails, isBrowserStep } from "./browser-activity";
 import type { VisibleChatItem } from "./chat-messages";
@@ -69,16 +70,22 @@ export function groupToolSteps(
 function foldable(item: ToolItem): boolean {
   if (isBrowserStep(item)) return true;
   if (!item.toolCall.function.name.startsWith("mcp__")) return false;
-  return !asksForConsent(item.result);
+  return !asksThePerson(item.result);
 }
 
-function asksForConsent(result: string | undefined): boolean {
+/**
+ * Whether this refusal is a question for the person rather than a line about what a Bot did.
+ *
+ * Both of these draw a card with something to press — a Team Bot asking to act as them, and an app
+ * they have not connected — and neither may sit behind a disclosure for the same reason: a question
+ * nobody can see is a conversation that has stopped without saying so.
+ */
+function asksThePerson(result: string | undefined): boolean {
   if (result === undefined) return false;
   const answer = asText(result);
   if (!answer.startsWith(REFUSAL_MARKER)) return false;
-  return (
-    readTeamBotConsent(answer.slice(REFUSAL_MARKER.length).trim()) !== null
-  );
+  const body = answer.slice(REFUSAL_MARKER.length).trim();
+  return readTeamBotConsent(body) !== null || readConnectPrompt(body) !== null;
 }
 
 type StepKind = "search" | "fetch" | "browser" | "connector";

app/src/components/computer/use-auto-watch-screen.ts

@@ -0,0 +1,77 @@
+import { useEffect, useRef } from "react";
+import {
+  onComputerActivity,
+  onComputerActivityWithdrawn,
+} from "@/lib/copilot/computer-activity";
+
+/**
+ * The live screen opening itself beside a conversation, once per run, and closing again when the
+ * call that opened it turns out to have been refused.
+ *
+ * A HOOK RATHER THAN AN EFFECT IN THE ROUTE, because what it decides is worth stating in tests
+ * without a router, a channel and a chat transcript around it. The route keeps what is genuinely
+ * its own: the pane is a URL flag it shares with the Bot's profile.
+ *
+ * The person's pane comes first. It does not open over something they opened (`busy`), it does not
+ * reopen what they are already watching, and a run they closed stays closed for the rest of it.
+ */
+export function useAutoWatchScreen({
+  botId,
+  watching,
+  busy,
+  open,
+  close,
+}: {
+  botId: string | undefined;
+  /** The pane is currently showing this Bot's screen. */
+  watching: boolean;
+  /** The pane is showing something else the person asked for, which auto-open must not take. */
+  busy: boolean;
+  open: () => void;
+  close: () => void;
+}): { dismiss: () => void } {
+  /** The run the Bot is in, so a dismissal can name the one it applies to. */
+  const run = useRef<number | null>(null);
+  const dismissed = useRef<number | null>(null);
+  /**
+   * The announcement this pane opened for, or null when it opened for nothing.
+   *
+   * Only this call may close it again. A pane the person opened never has one, and a pane showing
+   * a page some earlier call loaded holds that call's announcement rather than a later one's.
+   */
+  const openedFor = useRef<number | null>(null);
+
+  // Read inside subscriptions that outlive the render they were made in.
+  const latest = useRef({ watching, busy, open, close });
+  latest.current = { watching, busy, open, close };
+
+  useEffect(() => {
+    if (!botId) return;
+    const stopActivity = onComputerActivity((activity) => {
+      if (activity.botId !== botId) return;
+      run.current = activity.epoch;
+      if (dismissed.current === activity.epoch) return;
+      const { watching: isWatching, busy: isBusy, open: show } = latest.current;
+      if (isWatching || isBusy) return;
+      openedFor.current = activity.announcement;
+      show();
+    });
+    const stopWithdrawals = onComputerActivityWithdrawn((withdrawal) => {
+      if (withdrawal.botId !== botId) return;
+      if (openedFor.current !== withdrawal.announcement) return;
+      openedFor.current = null;
+      latest.current.close();
+    });
+    return () => {
+      stopActivity();
+      stopWithdrawals();
+    };
+  }, [botId]);
+
+  return {
+    dismiss: () => {
+      dismissed.current = run.current;
+      openedFor.current = null;
+    },
+  };
+}