feat: reliable browser challenge handoff and optional local Chrome

.env.example

@@ -304,11 +304,22 @@ COMPUTER_TOKEN=
 # docker-compose.yml. Pointing it somewhere unmounted disables login persistence.
 # PROFILES_DIR=/profiles
 
-# Browser process used by a Bot's computer. `headless` preserves the smaller default. `headed` runs
-# full Chromium on a private virtual display; the existing live screen and take-the-wheel controls
-# are still how a person sees and drives it.
+# Browser process used by a Bot's computer. Managed mode always uses full Chromium. `headless` is
+# the default; Linux `headed` uses a private virtual display with the same live-screen controls.
+# COMPUTER_BROWSER_BACKEND=managed
 # COMPUTER_BROWSER_MODE=headless
 
+# Opt-in installed Chrome on the same machine as a local API deployment:
+# bun scripts/start-local-chrome-computer.ts
+# The helper requires COMPUTER_TOKEN, binds 127.0.0.1:4101, and keeps dedicated per-Bot profiles.
+# Set AGENT_COMPUTER_URL=http://127.0.0.1:4101 on the API, clear COMPUTER_SUPERVISOR_URL and
+# COMPUTER_SANDBOX_NAMESPACE, then restart the API. See docs/configuration.md for full setup.
+# Local mode requires headed; do not leave an explicit headless override set in its environment.
+# COMPUTER_BROWSER_BACKEND=local-chrome
+# COMPUTER_BROWSER_MODE=headed
+# Optional absolute app-owned data root; defaults to the platform's OpenBot user-data directory.
+# OPENBOT_LOCAL_COMPUTER_DIR=
+
 # Which Bot this computer belongs to, and therefore which profile directory it uses.
 # COMPUTER_BOT_ID=shared
 

CHANGELOG.md

@@ -8,6 +8,14 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
 
 ## Unreleased
 
+### Browser challenges can be handed to a person without losing the Bot's page
+
+Bots pause for actionable browser challenges and resume from a fresh page snapshot after an explicit
+handback. Requests survive viewer reconnects and distinguish completion from cancellation, expiry,
+or an interrupted browser session. Managed browsing now uses full Chromium in headless or headed
+mode. Local API deployments can opt into installed Chrome with dedicated per-Bot profiles, the same
+in-app viewer, a loopback-only computer endpoint, and host shell execution disabled.
+
 ### A wiped or restarted shared computer no longer leaves refs pointing at the dead page
 
 Snapshots are ordered on the run of the browser that took them as well as the generation, so a

agent-computer/src/action-barrier.ts

@@ -0,0 +1,48 @@
+/** Synchronous admission, asynchronous draining. No work is queued or replayed. */
+export function createActionBarrier() {
+  let closed = false;
+  let active = 0;
+  const waiters = new Set<() => void>();
+  return {
+    enter(): () => void {
+      if (closed)
+        throw new Error("Action admission is closed for this computer.");
+      active += 1;
+      let released = false;
+      return () => {
+        if (released) return;
+        released = true;
+        active -= 1;
+        if (active === 0) for (const wake of [...waiters]) wake();
+      };
+    },
+    close() {
+      closed = true;
+    },
+    open() {
+      closed = false;
+    },
+    pending() {
+      return active;
+    },
+    drain(timeoutMs = 30_000): Promise<void> {
+      if (active === 0) return Promise.resolve();
+      return new Promise((resolve, reject) => {
+        const wake = () => {
+          clearTimeout(timer);
+          waiters.delete(wake);
+          resolve();
+        };
+        const timer = setTimeout(() => {
+          waiters.delete(wake);
+          reject(
+            new Error(
+              "An admitted action is still finishing; human control has not been granted. Try taking control again.",
+            ),
+          );
+        }, timeoutMs);
+        waiters.add(wake);
+      });
+    },
+  };
+}

agent-computer/src/authorisation.ts

@@ -78,3 +78,8 @@ const ACTING_PATHS = new Set([
 export function actsOnTheComputer(pathname: string): boolean {
   return ACTING_PATHS.has(pathname);
 }
+
+/** Only page mutations need a fresh browser snapshot after handback. */
+export function mutatesBrowser(pathname: string): boolean {
+  return ["/navigate", "/click", "/type", "/key", "/scroll"].includes(pathname);
+}

agent-computer/src/browser-runtime.ts

@@ -0,0 +1,40 @@
+import { browserModeFromEnv, type BrowserMode } from "./browser-mode";
+
+export type BrowserRuntime = {
+  backend: "managed" | "local-chrome";
+  channel: "chromium" | "chrome";
+  mode: BrowserMode;
+  useVirtualDisplay: boolean;
+  hostname?: "127.0.0.1";
+  allowExec: boolean;
+};
+
+/** The same launch decision is used by profiles and the computer HTTP process. */
+export function browserRuntimeFromEnv(
+  env: Record<string, string | undefined>,
+  platform: string = process.platform,
+): BrowserRuntime {
+  const backend = env.COMPUTER_BROWSER_BACKEND?.trim() || "managed";
+  if (backend !== "managed" && backend !== "local-chrome") {
+    throw new Error(
+      "COMPUTER_BROWSER_BACKEND must be managed or local-chrome.",
+    );
+  }
+  const local = backend === "local-chrome";
+  const mode = browserModeFromEnv(
+    env.COMPUTER_BROWSER_MODE?.trim() || (local ? "headed" : "headless"),
+  );
+  if (local && mode !== "headed") {
+    throw new Error(
+      "COMPUTER_BROWSER_BACKEND=local-chrome requires COMPUTER_BROWSER_MODE=headed.",
+    );
+  }
+  return {
+    backend,
+    channel: local ? "chrome" : "chromium",
+    mode,
+    useVirtualDisplay: platform === "linux" && mode === "headed",
+    ...(local ? { hostname: "127.0.0.1" as const } : {}),
+    allowExec: !local,
+  };
+}

agent-computer/src/challenge.ts

@@ -0,0 +1,102 @@
+import type { Page } from "playwright";
+import type { BrowserChallenge } from "../../shared/computer-control";
+import type { Control } from "./control";
+type Signal = {
+  cfMitigated?: string | null;
+  text: string;
+  visibleChallengeControl: boolean;
+};
+type Classification = Omit<BrowserChallenge, "requestId">;
+/** Generic access denial and score-only blocks offer no actionable human challenge. */
+export function classifyChallenge(signal: Signal): Classification | undefined {
+  if (signal.cfMitigated?.trim().toLowerCase() === "challenge")
+    return {
+      kind: "cloudflare",
+      reason:
+        "This site presented a Cloudflare security challenge. Please complete it in the existing browser, then hand control back.",
+    };
+  if (
+    signal.visibleChallengeControl &&
+    /verify (?:that )?you (?:are|'re) (?:a )?human|confirm (?:that )?you (?:are|'re) (?:a )?human|i(?:'|’)m not a robot|complete (?:the|this) (?:captcha|security check)|select all (?:images|squares)|prove (?:that )?you (?:are|'re) (?:a )?human/i.test(
+      signal.text,
+    )
+  ) {
+    return {
+      kind: "visible-challenge",
+      reason:
+        "This page has a visible human verification challenge. Please complete it in the existing browser, then hand control back.",
+    };
+  }
+}
+
+/** Called only on navigate/read/snapshot. Does not solve, click, or reload a challenge. */
+export async function detectChallenge(
+  page: Page,
+  control: Control,
+  options: { cfMitigated?: string | null; toolCallId?: string } = {},
+): Promise<BrowserChallenge | undefined> {
+  const signal =
+    options.cfMitigated?.trim().toLowerCase() === "challenge"
+      ? {
+          text: "",
+          visibleChallengeControl: false,
+          cfMitigated: options.cfMitigated,
+        }
+      : await page.evaluate(() => {
+          const visible = (element: Element) => {
+            const style = getComputedStyle(element);
+            const bounds = element.getBoundingClientRect();
+            return (
+              style.display !== "none" &&
+              style.visibility !== "hidden" &&
+              Number(style.opacity) !== 0 &&
+              bounds.width > 0 &&
+              bounds.height > 0 &&
+              bounds.bottom > 0 &&
+              bounds.right > 0 &&
+              bounds.top < innerHeight &&
+              bounds.left < innerWidth
+            );
+          };
+          const frames = Array.from(document.querySelectorAll("iframe[src]"));
+          const challengeFrames = frames.filter((frame) => {
+            const src = frame.getAttribute("src") ?? "";
+            return (
+              /(?:google\.com\/recaptcha|recaptcha\.net\/recaptcha|hcaptcha\.com\/|challenges\.cloudflare\.com\/)/i.test(
+                src,
+              ) && visible(frame)
+            );
+          });
+          const controls = Array.from(
+            document.querySelectorAll(
+              'input[type="checkbox"], [role="checkbox"], button',
+            ),
+          );
+          const visibleChallengeControl =
+            challengeFrames.length > 0 ||
+            controls.some(
+              (control) =>
+                visible(control) &&
+                /human|not a robot|verify|captcha/i.test(
+                  control.getAttribute("aria-label") ??
+                    control.closest("label")?.textContent ??
+                    control.textContent ??
+                    "",
+                ),
+            );
+          return {
+            text: (document.body?.innerText ?? "").slice(0, 20_000),
+            visibleChallengeControl,
+          };
+        });
+  const result = classifyChallenge(signal);
+  if (!result) return undefined;
+  const request = control.requestHelp(
+    result.reason,
+    options.toolCallId,
+    result.kind,
+  ).request;
+  if (!request)
+    throw new Error("The challenge handoff did not produce a request.");
+  return { ...result, requestId: request.id };
+}

agent-computer/src/control-store.ts

@@ -0,0 +1,144 @@
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+import type { HandoffRequest } from "../../shared/computer-control";
+import { isPlainBotId } from "./bot-id";
+
+export type StoredControl = {
+  version: 1;
+  holder: "bot" | "human";
+  since: string;
+  resumeSnapshotRequired: boolean;
+  recoveryRequired: boolean;
+  currentRequestId?: string;
+  requests: HandoffRequest[];
+  aliases: Record<string, string>;
+};
+export type ControlStore = {
+  load(): StoredControl | undefined;
+  save(state: StoredControl): void;
+};
+const statuses = new Set([
+  "waiting",
+  "taken",
+  "completed",
+  "cancelled",
+  "expired",
+  "interrupted",
+]);
+const sources = new Set(["model", "manual", "cloudflare", "visible-challenge"]);
+const record = (v: unknown): v is Record<string, unknown> =>
+  !!v && typeof v === "object" && !Array.isArray(v);
+const timestamp = (v: unknown): v is string =>
+  typeof v === "string" && Number.isFinite(Date.parse(v));
+
+function validRequest(v: unknown): v is HandoffRequest {
+  return (
+    record(v) &&
+    typeof v.id === "string" &&
+    v.id.length > 0 &&
+    v.id.length <= 200 &&
+    typeof v.reason === "string" &&
+    v.reason.length <= 500 &&
+    typeof v.source === "string" &&
+    sources.has(v.source) &&
+    typeof v.status === "string" &&
+    statuses.has(v.status) &&
+    timestamp(v.createdAt) &&
+    timestamp(v.updatedAt) &&
+    (v.toolCallId === undefined ||
+      (typeof v.toolCallId === "string" && v.toolCallId.length <= 200)) &&
+    (v.expiresAt === undefined || timestamp(v.expiresAt)) &&
+    (v.finishedAt === undefined || timestamp(v.finishedAt)) &&
+    (v.interruption === undefined || typeof v.interruption === "string") &&
+    (v.status !== "waiting" || timestamp(v.expiresAt)) &&
+    (v.status === "waiting" || v.expiresAt === undefined) &&
+    (v.status === "waiting" || v.status === "taken"
+      ? v.finishedAt === undefined
+      : timestamp(v.finishedAt))
+  );
+}
+
+function validate(value: unknown): StoredControl {
+  if (
+    !record(value) ||
+    value.version !== 1 ||
+    !["bot", "human"].includes(String(value.holder)) ||
+    !timestamp(value.since) ||
+    typeof value.resumeSnapshotRequired !== "boolean" ||
+    typeof value.recoveryRequired !== "boolean" ||
+    !Array.isArray(value.requests) ||
+    value.requests.length > 33 ||
+    !value.requests.every(validRequest) ||
+    !record(value.aliases) ||
+    Object.keys(value.aliases).length > 256 ||
+    (value.currentRequestId !== undefined &&
+      typeof value.currentRequestId !== "string")
+  )
+    throw new Error("Corrupt computer control state.");
+  const ids = new Set(value.requests.map((r) => r.id));
+  if (
+    ids.size !== value.requests.length ||
+    (value.currentRequestId !== undefined &&
+      !ids.has(value.currentRequestId)) ||
+    Object.entries(value.aliases).some(
+      ([key, id]) => key.length > 200 || typeof id !== "string" || !ids.has(id),
+    )
+  )
+    throw new Error("Corrupt computer control request history.");
+  const current = value.requests.find((r) => r.id === value.currentRequestId);
+  if (
+    (current?.status === "taken" && value.holder !== "human") ||
+    (value.requests.length > 0 && !current) ||
+    (value.holder === "human" &&
+      current?.status !== "taken" &&
+      current?.status !== "cancelled") ||
+    value.requests.some(
+      (r) =>
+        (r.status === "waiting" || r.status === "taken") &&
+        r.id !== value.currentRequestId,
+    )
+  )
+    throw new Error("Corrupt computer control ownership.");
+  return value as StoredControl;
+}
+
+/** Atomic replacement outside Chromium's profile. Never salvage corrupt data as success. */
+export function createControlStore(
+  profilesDirectory: string,
+  botId: string,
+): ControlStore {
+  if (!isPlainBotId(botId)) throw new Error("That is not a usable bot id.");
+  const directory = join(profilesDirectory, ".control");
+  const path = join(directory, `${botId}.json`);
+  return {
+    load() {
+      if (!existsSync(path)) return undefined;
+      const contents = readFileSync(path, "utf8");
+      if (contents.length > 256_000)
+        throw new Error("Computer control state exceeds its size limit.");
+      return validate(JSON.parse(contents));
+    },
+    save(state) {
+      validate(state);
+      mkdirSync(directory, { recursive: true, mode: 0o700 });
+      const temporary = `${path}.${crypto.randomUUID()}.tmp`;
+      const fd = openSync(temporary, "wx", 0o600);
+      try {
+        writeFileSync(fd, JSON.stringify(state));
+        fsyncSync(fd);
+      } finally {
+        closeSync(fd);
+      }
+      renameSync(temporary, path);
+    },
+  };
+}