diff --git a/.env.example b/.env.example index 790f599e7..8741a3d4e 100644 --- a/.env.example +++ b/.env.example @@ -304,11 +304,22 @@ COMPUTER_TOKEN= # docker-compose.yml. Pointing it somewhere unmounted disables login persistence. # PROFILES_DIR=/profiles -# Browser process used by a Bot's computer. `headless` preserves the smaller default. `headed` runs -# full Chromium on a private virtual display; the existing live screen and take-the-wheel controls -# are still how a person sees and drives it. +# Browser process used by a Bot's computer. Managed mode always uses full Chromium. `headless` is +# the default; Linux `headed` uses a private virtual display with the same live-screen controls. +# COMPUTER_BROWSER_BACKEND=managed # COMPUTER_BROWSER_MODE=headless +# Opt-in installed Chrome on the same machine as a local API deployment: +# bun scripts/start-local-chrome-computer.ts +# The helper requires COMPUTER_TOKEN, binds 127.0.0.1:4101, and keeps dedicated per-Bot profiles. +# Set AGENT_COMPUTER_URL=http://127.0.0.1:4101 on the API, clear COMPUTER_SUPERVISOR_URL and +# COMPUTER_SANDBOX_NAMESPACE, then restart the API. See docs/configuration.md for full setup. +# Local mode requires headed; do not leave an explicit headless override set in its environment. +# COMPUTER_BROWSER_BACKEND=local-chrome +# COMPUTER_BROWSER_MODE=headed +# Optional absolute app-owned data root; defaults to the platform's OpenBot user-data directory. +# OPENBOT_LOCAL_COMPUTER_DIR= + # Which Bot this computer belongs to, and therefore which profile directory it uses. # COMPUTER_BOT_ID=shared diff --git a/CHANGELOG.md b/CHANGELOG.md index 9480bc1ec..72d9f52d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### Browser challenges can be handed to a person without losing the Bot's page + +Bots pause for actionable browser challenges and resume from a fresh page snapshot after an explicit +handback. Requests survive viewer reconnects and distinguish completion from cancellation, expiry, +or an interrupted browser session. Managed browsing now uses full Chromium in headless or headed +mode. Local API deployments can opt into installed Chrome with dedicated per-Bot profiles, the same +in-app viewer, a loopback-only computer endpoint, and host shell execution disabled. + ### A wiped or restarted shared computer no longer leaves refs pointing at the dead page Snapshots are ordered on the run of the browser that took them as well as the generation, so a diff --git a/agent-computer/src/action-barrier.ts b/agent-computer/src/action-barrier.ts new file mode 100644 index 000000000..15ec64200 --- /dev/null +++ b/agent-computer/src/action-barrier.ts @@ -0,0 +1,48 @@ +/** Synchronous admission, asynchronous draining. No work is queued or replayed. */ +export function createActionBarrier() { + let closed = false; + let active = 0; + const waiters = new Set<() => void>(); + return { + enter(): () => void { + if (closed) + throw new Error("Action admission is closed for this computer."); + active += 1; + let released = false; + return () => { + if (released) return; + released = true; + active -= 1; + if (active === 0) for (const wake of [...waiters]) wake(); + }; + }, + close() { + closed = true; + }, + open() { + closed = false; + }, + pending() { + return active; + }, + drain(timeoutMs = 30_000): Promise { + if (active === 0) return Promise.resolve(); + return new Promise((resolve, reject) => { + const wake = () => { + clearTimeout(timer); + waiters.delete(wake); + resolve(); + }; + const timer = setTimeout(() => { + waiters.delete(wake); + reject( + new Error( + "An admitted action is still finishing; human control has not been granted. Try taking control again.", + ), + ); + }, timeoutMs); + waiters.add(wake); + }); + }, + }; +} diff --git a/agent-computer/src/authorisation.ts b/agent-computer/src/authorisation.ts index 57ffade8e..f0dfbbec1 100644 --- a/agent-computer/src/authorisation.ts +++ b/agent-computer/src/authorisation.ts @@ -78,3 +78,8 @@ const ACTING_PATHS = new Set([ export function actsOnTheComputer(pathname: string): boolean { return ACTING_PATHS.has(pathname); } + +/** Only page mutations need a fresh browser snapshot after handback. */ +export function mutatesBrowser(pathname: string): boolean { + return ["/navigate", "/click", "/type", "/key", "/scroll"].includes(pathname); +} diff --git a/agent-computer/src/browser-runtime.ts b/agent-computer/src/browser-runtime.ts new file mode 100644 index 000000000..b4123fc80 --- /dev/null +++ b/agent-computer/src/browser-runtime.ts @@ -0,0 +1,40 @@ +import { browserModeFromEnv, type BrowserMode } from "./browser-mode"; + +export type BrowserRuntime = { + backend: "managed" | "local-chrome"; + channel: "chromium" | "chrome"; + mode: BrowserMode; + useVirtualDisplay: boolean; + hostname?: "127.0.0.1"; + allowExec: boolean; +}; + +/** The same launch decision is used by profiles and the computer HTTP process. */ +export function browserRuntimeFromEnv( + env: Record, + platform: string = process.platform, +): BrowserRuntime { + const backend = env.COMPUTER_BROWSER_BACKEND?.trim() || "managed"; + if (backend !== "managed" && backend !== "local-chrome") { + throw new Error( + "COMPUTER_BROWSER_BACKEND must be managed or local-chrome.", + ); + } + const local = backend === "local-chrome"; + const mode = browserModeFromEnv( + env.COMPUTER_BROWSER_MODE?.trim() || (local ? "headed" : "headless"), + ); + if (local && mode !== "headed") { + throw new Error( + "COMPUTER_BROWSER_BACKEND=local-chrome requires COMPUTER_BROWSER_MODE=headed.", + ); + } + return { + backend, + channel: local ? "chrome" : "chromium", + mode, + useVirtualDisplay: platform === "linux" && mode === "headed", + ...(local ? { hostname: "127.0.0.1" as const } : {}), + allowExec: !local, + }; +} diff --git a/agent-computer/src/challenge.ts b/agent-computer/src/challenge.ts new file mode 100644 index 000000000..668753eb3 --- /dev/null +++ b/agent-computer/src/challenge.ts @@ -0,0 +1,102 @@ +import type { Page } from "playwright"; +import type { BrowserChallenge } from "../../shared/computer-control"; +import type { Control } from "./control"; +type Signal = { + cfMitigated?: string | null; + text: string; + visibleChallengeControl: boolean; +}; +type Classification = Omit; +/** Generic access denial and score-only blocks offer no actionable human challenge. */ +export function classifyChallenge(signal: Signal): Classification | undefined { + if (signal.cfMitigated?.trim().toLowerCase() === "challenge") + return { + kind: "cloudflare", + reason: + "This site presented a Cloudflare security challenge. Please complete it in the existing browser, then hand control back.", + }; + if ( + signal.visibleChallengeControl && + /verify (?:that )?you (?:are|'re) (?:a )?human|confirm (?:that )?you (?:are|'re) (?:a )?human|i(?:'|’)m not a robot|complete (?:the|this) (?:captcha|security check)|select all (?:images|squares)|prove (?:that )?you (?:are|'re) (?:a )?human/i.test( + signal.text, + ) + ) { + return { + kind: "visible-challenge", + reason: + "This page has a visible human verification challenge. Please complete it in the existing browser, then hand control back.", + }; + } +} + +/** Called only on navigate/read/snapshot. Does not solve, click, or reload a challenge. */ +export async function detectChallenge( + page: Page, + control: Control, + options: { cfMitigated?: string | null; toolCallId?: string } = {}, +): Promise { + const signal = + options.cfMitigated?.trim().toLowerCase() === "challenge" + ? { + text: "", + visibleChallengeControl: false, + cfMitigated: options.cfMitigated, + } + : await page.evaluate(() => { + const visible = (element: Element) => { + const style = getComputedStyle(element); + const bounds = element.getBoundingClientRect(); + return ( + style.display !== "none" && + style.visibility !== "hidden" && + Number(style.opacity) !== 0 && + bounds.width > 0 && + bounds.height > 0 && + bounds.bottom > 0 && + bounds.right > 0 && + bounds.top < innerHeight && + bounds.left < innerWidth + ); + }; + const frames = Array.from(document.querySelectorAll("iframe[src]")); + const challengeFrames = frames.filter((frame) => { + const src = frame.getAttribute("src") ?? ""; + return ( + /(?:google\.com\/recaptcha|recaptcha\.net\/recaptcha|hcaptcha\.com\/|challenges\.cloudflare\.com\/)/i.test( + src, + ) && visible(frame) + ); + }); + const controls = Array.from( + document.querySelectorAll( + 'input[type="checkbox"], [role="checkbox"], button', + ), + ); + const visibleChallengeControl = + challengeFrames.length > 0 || + controls.some( + (control) => + visible(control) && + /human|not a robot|verify|captcha/i.test( + control.getAttribute("aria-label") ?? + control.closest("label")?.textContent ?? + control.textContent ?? + "", + ), + ); + return { + text: (document.body?.innerText ?? "").slice(0, 20_000), + visibleChallengeControl, + }; + }); + const result = classifyChallenge(signal); + if (!result) return undefined; + const request = control.requestHelp( + result.reason, + options.toolCallId, + result.kind, + ).request; + if (!request) + throw new Error("The challenge handoff did not produce a request."); + return { ...result, requestId: request.id }; +} diff --git a/agent-computer/src/control-store.ts b/agent-computer/src/control-store.ts new file mode 100644 index 000000000..2daf95cad --- /dev/null +++ b/agent-computer/src/control-store.ts @@ -0,0 +1,144 @@ +import { + closeSync, + existsSync, + fsyncSync, + mkdirSync, + openSync, + readFileSync, + renameSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import type { HandoffRequest } from "../../shared/computer-control"; +import { isPlainBotId } from "./bot-id"; + +export type StoredControl = { + version: 1; + holder: "bot" | "human"; + since: string; + resumeSnapshotRequired: boolean; + recoveryRequired: boolean; + currentRequestId?: string; + requests: HandoffRequest[]; + aliases: Record; +}; +export type ControlStore = { + load(): StoredControl | undefined; + save(state: StoredControl): void; +}; +const statuses = new Set([ + "waiting", + "taken", + "completed", + "cancelled", + "expired", + "interrupted", +]); +const sources = new Set(["model", "manual", "cloudflare", "visible-challenge"]); +const record = (v: unknown): v is Record => + !!v && typeof v === "object" && !Array.isArray(v); +const timestamp = (v: unknown): v is string => + typeof v === "string" && Number.isFinite(Date.parse(v)); + +function validRequest(v: unknown): v is HandoffRequest { + return ( + record(v) && + typeof v.id === "string" && + v.id.length > 0 && + v.id.length <= 200 && + typeof v.reason === "string" && + v.reason.length <= 500 && + typeof v.source === "string" && + sources.has(v.source) && + typeof v.status === "string" && + statuses.has(v.status) && + timestamp(v.createdAt) && + timestamp(v.updatedAt) && + (v.toolCallId === undefined || + (typeof v.toolCallId === "string" && v.toolCallId.length <= 200)) && + (v.expiresAt === undefined || timestamp(v.expiresAt)) && + (v.finishedAt === undefined || timestamp(v.finishedAt)) && + (v.interruption === undefined || typeof v.interruption === "string") && + (v.status !== "waiting" || timestamp(v.expiresAt)) && + (v.status === "waiting" || v.expiresAt === undefined) && + (v.status === "waiting" || v.status === "taken" + ? v.finishedAt === undefined + : timestamp(v.finishedAt)) + ); +} + +function validate(value: unknown): StoredControl { + if ( + !record(value) || + value.version !== 1 || + !["bot", "human"].includes(String(value.holder)) || + !timestamp(value.since) || + typeof value.resumeSnapshotRequired !== "boolean" || + typeof value.recoveryRequired !== "boolean" || + !Array.isArray(value.requests) || + value.requests.length > 33 || + !value.requests.every(validRequest) || + !record(value.aliases) || + Object.keys(value.aliases).length > 256 || + (value.currentRequestId !== undefined && + typeof value.currentRequestId !== "string") + ) + throw new Error("Corrupt computer control state."); + const ids = new Set(value.requests.map((r) => r.id)); + if ( + ids.size !== value.requests.length || + (value.currentRequestId !== undefined && + !ids.has(value.currentRequestId)) || + Object.entries(value.aliases).some( + ([key, id]) => key.length > 200 || typeof id !== "string" || !ids.has(id), + ) + ) + throw new Error("Corrupt computer control request history."); + const current = value.requests.find((r) => r.id === value.currentRequestId); + if ( + (current?.status === "taken" && value.holder !== "human") || + (value.requests.length > 0 && !current) || + (value.holder === "human" && + current?.status !== "taken" && + current?.status !== "cancelled") || + value.requests.some( + (r) => + (r.status === "waiting" || r.status === "taken") && + r.id !== value.currentRequestId, + ) + ) + throw new Error("Corrupt computer control ownership."); + return value as StoredControl; +} + +/** Atomic replacement outside Chromium's profile. Never salvage corrupt data as success. */ +export function createControlStore( + profilesDirectory: string, + botId: string, +): ControlStore { + if (!isPlainBotId(botId)) throw new Error("That is not a usable bot id."); + const directory = join(profilesDirectory, ".control"); + const path = join(directory, `${botId}.json`); + return { + load() { + if (!existsSync(path)) return undefined; + const contents = readFileSync(path, "utf8"); + if (contents.length > 256_000) + throw new Error("Computer control state exceeds its size limit."); + return validate(JSON.parse(contents)); + }, + save(state) { + validate(state); + mkdirSync(directory, { recursive: true, mode: 0o700 }); + const temporary = `${path}.${crypto.randomUUID()}.tmp`; + const fd = openSync(temporary, "wx", 0o600); + try { + writeFileSync(fd, JSON.stringify(state)); + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(temporary, path); + }, + }; +} diff --git a/agent-computer/src/control.ts b/agent-computer/src/control.ts index 026f7afa4..d09a2599a 100644 --- a/agent-computer/src/control.ts +++ b/agent-computer/src/control.ts @@ -1,312 +1,368 @@ -/** - * Who has the wheel. - * - * One browser has at most one driver. When a Bot meets a login wall it can ask for - * help; a person takes control, does the part only they can do, and hands back. While a person holds - * control every acting call from the Bot is refused, because two drivers on one page is how a Bot - * clicks "Confirm" on a form a human was still filling in. - * - * State lives in this process rather than in the server because this process owns the browser, and a - * takeover that the browser does not know about is not a takeover. The server records it and decides - * who may ask for it; this decides whether the next action happens. - * - * This module has no Playwright import, so state-machine tests do not need a browser. Browser work - * stays in `index.ts`. - */ - -export type ControlState = { - holder: "bot" | "human"; - since: string; - /** Why the Bot asked, so the person knows what they are being handed. Set when the Bot requests. */ - reason?: string; - /** True once the Bot has asked for help and no person has taken the wheel yet. */ - requested: boolean; - /** - * When the Bot asked, so an unanswered request can stop being shown. - * - * A request nobody answers used to last forever. The run that made it had already ended, but the - * prompt stayed on the computer, and control belongs to the computer rather than to a conversation - * — so every later conversation with that Bot showed a live "Take control" for work it was not - * doing, with the reason the Bot gave, written for whoever asked and rendered to whoever looked. - */ - requestedAt?: string; - /** - * A secret the Bot is waiting for, described by its label only. - * - * Secret entry is scoped rather than a full takeover. The Bot names the field, says what it needs, - * and the person types into a masked box that goes straight to the page. - * - * The label is all that is ever stored. The value passes through one request and is not kept here, - * not returned, and not on any path the model reads. - */ - secretWanted?: string; - /** - * Which field the secret goes in, as a ref from the Bot's snapshot. - * - * Required so a secret cannot be sent to whichever field happens to have focus. - */ - secretRef?: string; - secretSnapshotId?: number; -}; - -/** Refusal because a person is driving. Distinct from a failure, so the Bot can be told to wait. */ +import type { + ComputerControlState, + HandoffRequest, + HandoffSource, +} from "../../shared/computer-control"; +import { createActionBarrier } from "./action-barrier"; +import type { ControlStore, StoredControl } from "./control-store"; +export type ControlState = ComputerControlState; export class ControlError extends Error { constructor(message: string) { super(message); this.name = "ControlError"; } } - -/** What a caller must say to ask for a secret. Rejected as a request error, not thrown. */ export class ControlRequestError extends Error { - constructor(message: string) { + constructor( + message: string, + readonly status: 400 | 404 | 409 = 400, + ) { super(message); this.name = "ControlRequestError"; } } - +export class SnapshotRequiredError extends Error { + constructor() { + super("Take a fresh browser snapshot after handback before acting."); + this.name = "SnapshotRequiredError"; + } +} export const NO_SECRET_PENDING = "Nothing is waiting for a secret."; -/** - * How long an unanswered request to take the wheel is shown for. - * - * Long enough that somebody who stepped away can still act on it, short enough that it does not - * follow the Bot into tomorrow's conversations. The run that made it is already over either way: - * nothing resumes when a person takes the wheel this late, so the value trades "still useful" against - * "still on screen" and nothing else. - */ export const HELP_REQUEST_TTL_MS = 10 * 60 * 1000; - -/** - * How long an unanswered request for a secret is shown for. - * - * The same window as the ask above, for the same reason, and named separately because they are two - * different prompts and shortening one should not silently shorten the other. A secret request is - * the narrower of the two — it names a field on a page — so nothing about it survives the run that - * made it any better than a request to take the wheel does. - */ export const SECRET_REQUEST_TTL_MS = HELP_REQUEST_TTL_MS; - export const HUMAN_HAS_CONTROL = - "A person has control of the computer right now. Wait for them to hand it back before acting."; + "A person has control or has been asked to take control. Wait for them to hand it back before acting."; export const TAKE_CONTROL_FIRST = "Take control before driving the computer yourself."; +type Options = { + store?: ControlStore; + invalidateSnapshot?: () => void; + drainTimeoutMs?: number; +}; -/** - * The wheel, as a state machine. - * - * A factory rather than a module-level `let` so a test can have its own, and so two of these cannot - * accidentally share state. `now` is injected for the same reason: `since` is part of the published - * state, and a test that cannot control the clock has to either skip it or match it loosely. - */ +/** Request identity is durable; secret values never enter this state machine. */ export function createControl( now: () => string = () => new Date().toISOString(), + options: Options = {}, ) { - let state: ControlState = { + const barrier = createActionBarrier(); + const restored = options.store?.load(); + const data: StoredControl = restored ?? { + version: 1, holder: "bot", since: now(), - requested: false, + resumeSnapshotRequired: false, + recoveryRequired: false, + requests: [], + aliases: {}, }; - - /** - * When the Bot asked for a secret, so an unanswered request can stop being shown. - * - * Held here rather than on the state because nothing outside needs it: the surface renders the - * label and the field, and a timestamp added to the published state would be one more thing on a - * screen that is asking somebody for a password. - */ - let secretRequestedAt: string | undefined; - - /** - * Drop a secret request the run that made it has outlived. - * - * The same argument as the ask above, missed for the other half of it. Control belongs to the - * computer rather than to a conversation, so a request nobody answered sat on it for ever: the run - * that asked had ended, and every later conversation with that Bot still showed a masked box - * wanting "the six-digit code from your authenticator", written for whoever asked and rendered to - * whoever looked. The surface makes no distinction — `useNeedsYou` lights the same "needs you" on - * `requested` and on `secretWanted` — so expiring one and not the other left the Bot flagged - * anyway. - * - * Expired on read for the same reason the ask is: there is nothing to wake, and the only thing - * that cares is whoever looks next. Read by `pendingSecret` too, because that is what decides - * whether a value typed now is accepted, and a prompt that has stopped being shown must not still - * be answerable. - */ - function dropStaleSecret(): void { - if (!state.secretWanted || !secretRequestedAt) return; + let secret: + | { label: string; ref: string; snapshotId?: number; requestedAt: string } + | undefined; + let storageError: Error | undefined; + const current = () => + data.requests.find((r) => r.id === data.currentRequestId); + const active = (request: HandoffRequest | undefined) => + request?.status === "waiting" || request?.status === "taken"; + function persist() { + // Keep the current identity and the latest 32 terminal records. Their aliases expire together. + const cutoff = Date.parse(now()) - 30 * 24 * 60 * 60 * 1000; + const keep = data.requests + .filter( + (r) => + r.id === data.currentRequestId || Date.parse(r.updatedAt) >= cutoff, + ) + .slice(-33); + const ids = new Set(keep.map((r) => r.id)); + data.requests = keep; + data.aliases = Object.fromEntries( + Object.entries(data.aliases) + .filter(([, id]) => ids.has(id)) + .slice(-256), + ); + if (storageError) throw storageError; + try { + options.store?.save(structuredClone(data)); + } catch (error) { + storageError = new Error( + "The handoff state could not be saved. Computer actions are disabled until storage is repaired.", + { cause: error }, + ); + barrier.close(); + throw storageError; + } + } + function finish( + request: HandoffRequest, + status: "completed" | "cancelled" | "expired" | "interrupted", + interruption?: string, + ) { + request.status = status; + request.updatedAt = now(); + request.finishedAt = request.updatedAt; + delete request.expiresAt; + if (interruption) request.interruption = interruption; + } + function syncAdmission() { + if (active(current()) || data.holder === "human" || data.recoveryRequired) + barrier.close(); + else barrier.open(); + } + function expire() { + if (storageError) throw storageError; + const request = current(); if ( - Date.parse(now()) - Date.parse(secretRequestedAt) <= - SECRET_REQUEST_TTL_MS + request?.status === "waiting" && + request.expiresAt && + Date.parse(now()) > Date.parse(request.expiresAt) ) { - return; + finish(request, "expired"); + persist(); + syncAdmission(); } - secretRequestedAt = undefined; - state = { - ...state, - secretWanted: undefined, - secretRef: undefined, - secretSnapshotId: undefined, + if ( + secret && + Date.parse(now()) - Date.parse(secret.requestedAt) > SECRET_REQUEST_TTL_MS + ) + secret = undefined; + } + function get(requestId?: string): ControlState { + expire(); + const request = requestId + ? data.requests.find((r) => r.id === requestId) + : current(); + if (requestId && !request) + throw new ControlRequestError("That handoff request was not found.", 404); + const live = current(); + return { + holder: data.holder, + since: data.since, + requested: live?.status === "waiting", + reason: active(live) ? live?.reason : undefined, + request: request ? { ...request } : undefined, + transitioning: + (active(live) || data.holder === "human" || data.recoveryRequired) && + barrier.pending() > 0, + resumeSnapshotRequired: data.resumeSnapshotRequired, + ...(secret + ? { + secretWanted: secret.label, + secretRef: secret.ref, + secretSnapshotId: secret.snapshotId, + } + : {}), }; } + function intended(id: string): HandoffRequest { + expire(); + const request = current(); + if (!request || request.id !== id) + throw new ControlRequestError( + "That request is no longer the current handoff.", + 409, + ); + return request; + } + function interrupt(reason: string): ControlState { + const request = current(); + if (request && active(request)) finish(request, "interrupted", reason); + if ( + request && + (active(request) || + request.status === "interrupted" || + data.holder === "human") + ) + data.recoveryRequired = true; + data.holder = "bot"; + data.since = now(); + data.resumeSnapshotRequired = true; + secret = undefined; + options.invalidateSnapshot?.(); + persist(); + syncAdmission(); + return get(); + } + // A persisted active request names a page in a previous process. Never call that completion. + if (restored && (active(current()) || data.holder === "human")) + interrupt( + "The computer process restarted; the previous page was interrupted. Request help again to continue.", + ); + if (restored && !active(current()) && data.holder === "bot") { + data.resumeSnapshotRequired = true; + persist(); + } + syncAdmission(); + + function rememberAlias(toolCallId: string, requestId: string) { + Object.defineProperty(data.aliases, toolCallId, { + value: requestId, + enumerable: true, + configurable: true, + writable: true, + }); + } return { - /** - * The current state, as the surface polls it. A copy, so a caller cannot mutate the machine. - * - * An unanswered request is dropped once it is older than {@link HELP_REQUEST_TTL_MS}. It is - * expired on read rather than on a timer because there is nothing to wake: the run that asked - * has ended, and the only thing that cares is whoever looks next. - * - * Only ever an ASK. A person actually holding the wheel is never timed out from under them: - * they may be halfway through typing a code, and taking the browser back mid-sign-in is worse - * than any stale prompt. - */ - get(): ControlState { + get, + requestHelp( + reason: unknown, + toolCallId?: string, + source: HandoffSource = "model", + ): ControlState { + expire(); if ( - state.requested && - state.holder === "bot" && - state.requestedAt && - Date.parse(now()) - Date.parse(state.requestedAt) > HELP_REQUEST_TTL_MS - ) { - const { reason: _reason, requestedAt: _at, ...rest } = state; - state = { ...rest, requested: false }; + toolCallId !== undefined && + (!toolCallId.trim() || toolCallId.length > 200) + ) + throw new ControlRequestError( + "The tool call id must be a nonempty string of at most 200 characters.", + ); + const previous = + toolCallId && Object.hasOwn(data.aliases, toolCallId) + ? data.aliases[toolCallId] + : undefined; + if (previous) return get(previous); + const existing = current(); + if (active(existing) || data.holder === "human") { + if (toolCallId && existing) { + rememberAlias(toolCallId, existing.id); + persist(); + } + return get(); } - dropStaleSecret(); - return { ...state }; - }, - - /** - * The Bot asking for help. - * - * It does not take control: it says it is stuck and why, and a person decides. A Bot that could - * hand itself to a human could also hand a human a page they never asked to see. - */ - requestHelp(reason: unknown): ControlState { - state = { - ...state, - requested: true, - requestedAt: now(), - // Polled ~1Hz by every viewer for HELP_REQUEST_TTL_MS: a model-generated megabyte reason - // would be retained and re-served the whole time. Capped like form fields are. + barrier.close(); // Before any persistence or awaited browser operation. + const at = now(); + const request: HandoffRequest = { + id: crypto.randomUUID(), + toolCallId, reason: typeof reason === "string" && reason.trim() ? reason.trim().slice(0, 500) : "The assistant needs a person to continue.", + source, + status: "waiting", + createdAt: at, + updatedAt: at, + expiresAt: new Date(Date.parse(at) + HELP_REQUEST_TTL_MS).toISOString(), }; - return this.get(); + data.currentRequestId = request.id; + data.requests.push(request); + if (toolCallId) rememberAlias(toolCallId, request.id); + persist(); + return get(); + }, + async take(requestId: string): Promise { + const request = intended(requestId); + if (request.status === "taken" && data.holder === "human") return get(); + if (request.status !== "waiting") + throw new ControlRequestError( + "That request is no longer waiting for control.", + 409, + ); + barrier.close(); + // Never await here from requestHelp: a navigation detector still owns its action lease. + await barrier.drain(options.drainTimeoutMs); + const after = intended(requestId); + if (after.status === "taken" && data.holder === "human") return get(); + if (after.status !== "waiting") + throw new ControlRequestError( + "That request is no longer waiting for control.", + 409, + ); + after.status = "taken"; + after.updatedAt = now(); + delete after.expiresAt; + data.holder = "human"; + data.since = now(); + secret = undefined; + persist(); + return get(); + }, + release(requestId: string): ControlState { + const request = intended(requestId); + if (request.status === "completed" && data.holder === "bot") return get(); + if ( + data.holder !== "human" || + (request.status !== "taken" && request.status !== "cancelled") + ) + throw new ControlRequestError( + "That request is no longer held by a person.", + 409, + ); + if (request.status === "taken") finish(request, "completed"); + data.holder = "bot"; + data.since = now(); + data.resumeSnapshotRequired = true; + data.recoveryRequired = false; + secret = undefined; + options.invalidateSnapshot?.(); + persist(); + syncAdmission(); + return get(); + }, + cancel(requestId: string): ControlState { + const request = intended(requestId); + if (request.status === "cancelled") return get(); + if (!active(request)) + throw new ControlRequestError("That request is no longer active.", 409); + finish(request, "cancelled"); + persist(); + syncAdmission(); + return get(); + }, + interrupt, + snapshotTaken() { + if ( + data.holder === "bot" && + data.resumeSnapshotRequired && + !active(current()) + ) { + data.resumeSnapshotRequired = false; + persist(); + } + }, + assertBotMayAct(browserMutation = false) { + expire(); + if (data.holder === "human" || active(current()) || data.recoveryRequired) + throw new ControlError(HUMAN_HAS_CONTROL); + if (browserMutation && data.resumeSnapshotRequired) + throw new SnapshotRequiredError(); + }, + admitBotAction(browserMutation = false): () => void { + this.assertBotMayAct(browserMutation); + return barrier.enter(); + }, + humanMayDrive(): boolean { + return data.holder === "human" && barrier.pending() === 0; }, - - /** The Bot asking for one value it must not be told, naming the field it goes in. */ requestSecret(input: { label?: unknown; ref?: unknown; snapshotId?: unknown; }): ControlState { - if (typeof input.ref !== "string" || !input.ref.trim()) { + if (typeof input.ref !== "string" || !input.ref.trim()) throw new ControlRequestError( "Say which field the value goes in, using a ref from your snapshot.", ); - } - secretRequestedAt = now(); - state = { - ...state, - secretWanted: + secret = { + label: typeof input.label === "string" && input.label.trim() ? input.label.trim().slice(0, 500) : "the value this page is asking for", - secretRef: input.ref.trim(), - secretSnapshotId: + ref: input.ref.trim(), + snapshotId: typeof input.snapshotId === "number" ? input.snapshotId : undefined, + requestedAt: now(), }; - return this.get(); + return get(); }, - - /** - * The pending secret request, or null. - * - * Read before typing so the caller can refuse when nothing asked for one: this is what keeps the - * masked box from being a general-purpose way to type into the page. - * - * Which is also why the staleness check is here and not only on `get`: a request that has stopped - * being shown must stop being answerable at the same moment, or a value typed into a box left - * open in an old tab still goes to a page whose run ended. - */ pendingSecret(): { ref: string; snapshotId?: number } | null { - dropStaleSecret(); - if (!state.secretWanted || !state.secretRef) return null; - return { ref: state.secretRef, snapshotId: state.secretSnapshotId }; - }, - - /** - * The secret landed, so the request is closed. - * - * Called only after the value reaches the field. A failure leaves the request open so the person - * can try again. - */ - secretSupplied(): void { - secretRequestedAt = undefined; - state = { - ...state, - secretWanted: undefined, - secretRef: undefined, - secretSnapshotId: undefined, - }; - }, - - /** - * A person taking the wheel. - * - * `reason` survives, because it is the thing they were just asked to do. Any pending secret is - * cleared: a person with full browser control can type the password into the page, and a masked - * box left open behind them no longer corresponds to an active request. - */ - take(): ControlState { - // With the pending secret, since the state below drops it: the timestamp is what says one is - // outstanding, and leaving it behind a request that is gone is how a stale one comes back. - secretRequestedAt = undefined; - state = { - holder: "human", - since: now(), - reason: state.reason, - requested: false, - }; - return this.get(); + expire(); + return secret ? { ref: secret.ref, snapshotId: secret.snapshotId } : null; }, - - /** - * A person handing back. - * - * `reason` is dropped: it described the thing the person was asked to do, and once they have done - * it, leaving it set would have the surface still showing the old request. Any pending secret goes - * with it, a person who took the whole wheel and handed it back has dealt with the login, and a - * secret box left open afterwards is asking for a password nothing is waiting for. - */ - release(): ControlState { - // As above: the request the state below drops takes its timestamp with it. - secretRequestedAt = undefined; - state = { - holder: "bot", - since: now(), - requested: false, - }; - return this.get(); - }, - - /** - * The Bot may not act while a person holds the wheel. - * - * Refused rather than queued. A queued click lands after the person has moved on and is worse than - * a refusal, which the Bot can explain and wait out. - */ - assertBotMayAct(): void { - if (state.holder === "human") throw new ControlError(HUMAN_HAS_CONTROL); - }, - - /** Whether a person's input should be applied. The socket being open is not permission. */ - humanMayDrive(): boolean { - return state.holder === "human"; + secretSupplied() { + secret = undefined; }, }; } - export type Control = ReturnType; diff --git a/agent-computer/src/index.ts b/agent-computer/src/index.ts index b1d8252dc..1eebdb851 100644 --- a/agent-computer/src/index.ts +++ b/agent-computer/src/index.ts @@ -9,19 +9,27 @@ import { } from "./aria-snapshot"; import { actsOnTheComputer, + mutatesBrowser, isOpenPath, matchesToken, offeredToken, } from "./authorisation"; import { isPlainBotId } from "./bot-id"; -import { browserModeFromEnv } from "./browser-mode"; +import { browserRuntimeFromEnv } from "./browser-runtime"; +import { detectChallenge } from "./challenge"; import { ControlError, ControlRequestError, + SnapshotRequiredError, NO_SECRET_PENDING, TAKE_CONTROL_FIRST, } from "./control"; import { identity } from "./identity"; +import { + assertPageAccess, + navigateWebPage, + type PagePurpose, +} from "./navigation"; import { createProfiles, numberFromEnv, VIEWPORT } from "./profiles"; import { parseExecTimeout, @@ -84,8 +92,11 @@ if (!COMPUTER_TOKEN) { process.exit(1); } -const BROWSER_MODE = browserModeFromEnv(process.env.COMPUTER_BROWSER_MODE); -const VIRTUAL_DISPLAY = await startVirtualDisplay(BROWSER_MODE); +const RUNTIME = browserRuntimeFromEnv(process.env); +const BROWSER_MODE = RUNTIME.mode; +const VIRTUAL_DISPLAY = await startVirtualDisplay( + RUNTIME.useVirtualDisplay ? "headed" : "headless", +); if (VIRTUAL_DISPLAY) process.env.DISPLAY = VIRTUAL_DISPLAY.name; console.info( JSON.stringify({ @@ -134,7 +145,10 @@ const TEXT_EXTRACT_LIMIT = 6000; * rules about when a run changes are testable there, and not here, because this file launches a * browser the moment it is imported. */ -const sessions = createSessions({ isLive: (botId) => profiles.isLive(botId) }); +const sessions = createSessions({ + isLive: (botId) => profiles.isLive(botId), + profilesDirectory: process.env.PROFILES_DIR?.trim() || "/profiles", +}); /** * Sent by the server as a header on every call. Absent means the caller does not know or does not @@ -232,7 +246,10 @@ const workspace = createWorkspace( */ const profiles = createProfiles( process.env.PROFILES_DIR?.trim() || "/profiles", - (botId) => sessions.get(botId)?.viewer.releaseAll(COMPUTER_STOPPED), + async (botId) => { + if (sessions.get(botId)) sessions.renewRun(botId); + await sessions.get(botId)?.viewer.releaseAll(COMPUTER_STOPPED); + }, ); // Rooted in the same workspace the file tools use, so a command and a written file see one // directory rather than two. @@ -255,9 +272,14 @@ const DEFAULT_BOT_ID = (() => { return configured; })(); -async function currentPage(botId: string): Promise { +async function currentPage( + botId: string, + purpose: PagePurpose = "read", +): Promise { const session = sessions.for(botId); const page = await profiles.page(botId); + sessions.observeBrowser(botId, page.context()); + assertPageAccess(RUNTIME.backend, page.url(), purpose); // A ref names an element on the page it was taken from, so moving to a window the site opened has // to retire the outstanding ones exactly as a navigation does, or a click lands on the wrong document. if (session.livePage && session.livePage !== page) session.snapshotId += 1; @@ -319,12 +341,20 @@ async function snapshotPage( elements: SnapshotElement[]; truncated: boolean; }> { - session.snapshotId += 1; + const snapshotId = ++session.snapshotId; + const before = session.control.get(); const yaml = await target.ariaSnapshot({ mode: "ai" }); + const title = await target.title(); + if ( + session.snapshotId === snapshotId && + before.holder === "bot" && + !before.requested + ) + session.control.snapshotTaken(); return { - snapshotId: session.snapshotId, + snapshotId, url: target.url(), - title: await target.title(), + title, ...parseAriaSnapshot(yaml), }; } @@ -413,6 +443,7 @@ type StreamData = { botId: string }; serve({ port: PORT, + ...(RUNTIME.hostname ? { hostname: RUNTIME.hostname } : {}), idleTimeout: 120, /** * The live screen, pushed by Chrome rather than polled. @@ -618,511 +649,596 @@ serve({ * the browser at a login wall. `actsOnTheComputer` is the list, and a new acting endpoint is * refused by being added to it rather than by remembering to repeat this. */ - if (actsOnTheComputer(url.pathname)) { - try { - session.control.assertBotMayAct(); - } catch (error) { - // A person holding the wheel is not a failure of the action; the Bot should wait and say so. - if (error instanceof ControlError) { - return json({ error: error.message, humanHasControl: true }, 409); + let releaseAction: (() => void) | undefined; + try { + if (actsOnTheComputer(url.pathname)) { + releaseAction = session.control.admitBotAction( + mutatesBrowser(url.pathname), + ); + } + + if (url.pathname === "/stream") { + /* + * The socket carries the Bot in the query because it cannot do it in a header. Every other call here names + * its Bot in `x-openbot-bot-id`, but a websocket client sends no custom headers on the upgrade, + * so the stream, and only the stream, also accepts the Bot as a query parameter. The header + * still wins where there is one. + */ + const streamBotId = botIdOf(request, url.searchParams.get("bot")); + if (!isPlainBotId(streamBotId)) { + return json({ error: "That is not a usable bot id." }, 400); } - throw error; + if (server.upgrade(request, { data: { botId: streamBotId } })) + return undefined as unknown as Response; + return json({ error: "Expected a WebSocket upgrade." }, 400); } - } - if (url.pathname === "/stream") { /* - * The socket carries the Bot in the query because it cannot do it in a header. Every other call here names - * its Bot in `x-openbot-bot-id`, but a websocket client sends no custom headers on the upgrade, - * so the stream, and only the stream, also accepts the Bot as a query parameter. The header - * still wins where there is one. + * `/live` stays absent. A page served by this process can only be opened by putting the secret in + * a URL, where it lands in history and logs. The React app is the guarded way to watch a Bot. */ - const streamBotId = botIdOf(request, url.searchParams.get("bot")); - if (!isPlainBotId(streamBotId)) { - return json({ error: "That is not a usable bot id." }, 400); - } - if (server.upgrade(request, { data: { botId: streamBotId } })) - return undefined as unknown as Response; - return json({ error: "Expected a WebSocket upgrade." }, 400); - } - /* - * `/live` stays absent. A page served by this process can only be opened by putting the secret in - * a URL, where it lands in history and logs. The React app is the guarded way to watch a Bot. - */ - - // Who has the wheel. Polled by the surface alongside the screen, so the person sees the Bot ask - // for help without having to reload anything. - if (url.pathname === "/control" && request.method === "GET") { - return json(session.control.get()); - } - - // The Bot asking for help. It does not take control: it says it is stuck and why, and a person - // decides. A Bot that could hand itself to a human could also hand a human a page they never - // asked to see. - if (url.pathname === "/control/request" && request.method === "POST") { - const body = (await request.json().catch(() => null)) as { - reason?: unknown; - } | null; - return json(session.control.requestHelp(body?.reason)); - } - - // The Bot asking for one value it must not be told. It has already focused the field. - if (url.pathname === "/control/secret" && request.method === "POST") { - const body = (await request.json().catch(() => null)) as { - label?: unknown; - ref?: unknown; - snapshotId?: unknown; - } | null; - try { - return json(session.control.requestSecret(body ?? {})); - } catch (error) { - if (error instanceof ControlRequestError) { - return json({ error: error.message }, 400); - } - throw error; + // Who has the wheel. Polled by the surface alongside the screen, so the person sees the Bot ask + // for help without having to reload anything. + if (url.pathname === "/control" && request.method === "GET") { + return json( + session.control.get(url.searchParams.get("requestId") ?? undefined), + ); } - } - /** - * A person supplying that value. - * - * Scoped by the pending request rather than by a control handover: it is usable only while the Bot - * has actually asked for a secret, and the request is cleared the moment it is answered, so this - * cannot be used as a general back door to type into the page. - * - * The value is typed and forgotten. Not stored on `control`, not returned in the response, not - * logged. The response says how many characters arrived, which is enough for the surface to - * confirm something was sent and useless to anybody reading it later. - * - * It types and does not submit. Committing a form is a separate action through the gateway and - * audit trail; secret entry only places the value in the named field. - */ - if (url.pathname === "/human/secret" && request.method === "POST") { - const pending = session.control.pendingSecret(); - if (!pending) { - return json({ error: NO_SECRET_PENDING }, 409); - } - const body = (await request.json().catch(() => null)) as { - text?: unknown; - } | null; - if (typeof body?.text !== "string" || !body.text) { - return json({ error: "A value is required." }, 400); - } - try { - const target = await currentPage(botId); - // Focus the field the Bot named, and let this throw if it cannot be found. A secret must not - // be reported as delivered unless a field receives it. - // - // No generation check here: a Bot may take another snapshot after asking for a secret, while - // the ref remains protected by Playwright's `aria-ref` rules. - // - // `aria-ref` resolves a ref only against the most recent snapshot, only while the element is - // still connected, and mints a - // new ref when an element's role or accessible name changes, so a recycled node cannot - // inherit an old one. If the ref resolves, it is the field the Bot meant. If it does not, - // nothing is typed, which is the outcome the generation check existed to guarantee. - const field = locateRef(session, target, pending.ref, undefined); - await field.click({ timeout: ACTION_TIMEOUT_MS }); - await field.fill(body.text, { timeout: ACTION_TIMEOUT_MS }); - const characters = body.text.length; - // Cleared only after it actually landed, so a failure leaves the request open and the person - // can try again rather than being told to start over. - session.control.secretSupplied(); - return json({ supplied: true, characters, url: target.url() }); - } catch (error) { - if (error instanceof StaleSnapshotError) { - return json({ error: error.message, stale: true }, 409); - } - // The field is gone, which is unretryable, so the request is closed rather than left open. - // Keeping it open is right for a mistyped value and wrong here: the person would retype their - // password into the same dead ref for ever. Clearing it also unblocks the Bot, which can see - // on its next turn that nothing is pending and ask again against a fresh snapshot. - session.control.secretSupplied(); + // The Bot asking for help. It does not take control: it says it is stuck and why, and a person + // decides. A Bot that could hand itself to a human could also hand a human a page they never + // asked to see. + if (url.pathname === "/control/request" && request.method === "POST") { + const body = (await request.json().catch(() => null)) as { + reason?: unknown; + toolCallId?: unknown; + } | null; + if ( + body?.toolCallId !== undefined && + typeof body.toolCallId !== "string" + ) + return json({ error: "toolCallId must be a string." }, 400); return json( - { - error: describe( - error, - "That value could not be entered: the field is no longer on the page. Ask the assistant to request it again.", - ), - }, - 502, + session.control.requestHelp(body?.reason, body?.toolCallId), ); } - } - - if (url.pathname === "/control/take" && request.method === "POST") { - return json(session.control.take()); - } - - if (url.pathname === "/control/release" && request.method === "POST") { - // `reason` is dropped on release: it described the thing the person was asked to do, and once - // they have done it, leaving it set would have the surface still showing the old request. - return json(session.control.release()); - } - // A person's input, by pixel. The Bot addresses elements by reference because it reads a list; a - // person addresses them by pointing, because they are looking at a picture. Different problem, - // different endpoint, and only usable while they hold the wheel. - if (HUMAN_INPUT.has(url.pathname) && request.method === "POST") { - if (!session.control.humanMayDrive()) { - return json({ error: TAKE_CONTROL_FIRST }, 409); - } - const body = (await request.json().catch(() => null)) as Record< - string, - unknown - > | null; - if (url.pathname === "/human/scroll") { - const parsed = parseScrollDelta(body?.deltaY); - if (!parsed.ok) { - return json({ error: parsed.error }, 400); + // The Bot asking for one value it must not be told. It has already focused the field. + if (url.pathname === "/control/secret" && request.method === "POST") { + const body = (await request.json().catch(() => null)) as { + label?: unknown; + ref?: unknown; + snapshotId?: unknown; + } | null; + try { + return json(session.control.requestSecret(body ?? {})); + } catch (error) { + if (error instanceof ControlRequestError) { + return json({ error: error.message }, 400); + } + throw error; } } - try { - const target = await currentPage(botId); - return json(await performHumanInput(target, url.pathname, body ?? {})); - } catch (error) { - return json({ error: describe(error, "That did not work.") }, 502); - } - } - - if (url.pathname === "/health") { - const [profile] = profiles.summary([botId]); - return json({ - status: "ok", - // `browser` kept as it was: it is in the published contract and start.sh reads it. - browser: profile?.running ?? false, - profile, - // Which Bot this computer can prove it is, when the deployment runs SPIRE. Null is a - // deployment without it, not a failure, and it is reported rather than omitted so the - // difference between "no identity here" and "identity broken" is visible. - identity: await identity(), - browserMode: BROWSER_MODE, - }); - } - - /** - * Which run of this Bot's browser the caller is looking at. - * - * The server orders snapshots on `(run, generation)`, and the generation alone cannot carry it: - * this process mints one at zero for every session that is new, so a restart, a redeploy, an - * eviction from the idle sweep and a reset all produce a page at generation one that looks older - * than the page the server still has. Ordering on the run as well is what lets the fresh one land - * and the dead one stop resolving. - * - * A read, and deliberately not on the acting list: a person holding the wheel must not turn every - * ref the server holds into an unanswerable question. - * - * Its own endpoint rather than a field on `/computers`, because that one reads the profile - * directory and this is asked on the path of every governed action. - */ - if (url.pathname === "/run" && request.method === "GET") { - return json({ run: session.run }); - } - /** - * The computers this process holds. The shape is a list because the admin surface is a - * list, and because a Bot that has a profile has a computer whether or not a browser is running - * for it this second. - */ - if (url.pathname === "/computers" && request.method === "GET") { - return json({ computers: profiles.summary(await profiles.known()) }); - } - - /** - * Stop the browser, keep what it knows. - * - * Closed gracefully so Chromium flushes its profile, and deliberately - * not restarted here: the next request starts it again, which is the same path as a first ever - * start, so there is no second way for a browser to come into existence. - */ - if (url.pathname === "/computers/stop" && request.method === "POST") { - const wasRunning = await profiles.stop(botId); - // The wheel goes back to the Bot because the controlled browser no longer exists. - session.control.release(); - return json({ stopped: true, wasRunning }); - } - - /** - * Forget everything and start over. - * - * Signs the computer out of everything by deleting the profile. Irreversible, which is why it is - * its own endpoint rather than a flag on the one above: a person clicking "stop" must not be able - * to discard a login by mistyping a parameter. - */ - if (url.pathname === "/computers/reset" && request.method === "POST") { - await profiles.reset(botId); - // Reset releases control because any previous browser session and pending secret request are gone. - session.control.release(); - /* - * And a new run, because the browser this session described is gone. + /** + * A person supplying that value. + * + * Scoped by the pending request rather than by a control handover: it is usable only while the Bot + * has actually asked for a secret, and the request is cleared the moment it is answered, so this + * cannot be used as a general back door to type into the page. * - * Nothing else here says so. The entry stays in the map and the generation counter carries on, - * so a snapshot that was in flight when the wipe landed arrives at the server carrying the same - * run and the same generation as the fresh browser's would: the server deletes its row on - * reset, the late save inserts the wiped page straight back, and every ref on it goes on - * resolving. A new run is what makes those two distinguishable at the far end. + * The value is typed and forgotten. Not stored on `control`, not returned in the response, not + * logged. The response says how many characters arrived, which is enough for the surface to + * confirm something was sent and useless to anybody reading it later. + * + * It types and does not submit. Committing a form is a separate action through the gateway and + * audit trail; secret entry only places the value in the named field. */ - sessions.renewRun(botId); - return json({ reset: true, botId }); - } + if (url.pathname === "/human/secret" && request.method === "POST") { + const pending = session.control.pendingSecret(); + if (!pending) { + return json({ error: NO_SECRET_PENDING }, 409); + } + const body = (await request.json().catch(() => null)) as { + text?: unknown; + } | null; + if (typeof body?.text !== "string" || !body.text) { + return json({ error: "A value is required." }, 400); + } + try { + const target = await currentPage(botId); + // Focus the field the Bot named, and let this throw if it cannot be found. A secret must not + // be reported as delivered unless a field receives it. + // + // No generation check here: a Bot may take another snapshot after asking for a secret, while + // the ref remains protected by Playwright's `aria-ref` rules. + // + // `aria-ref` resolves a ref only against the most recent snapshot, only while the element is + // still connected, and mints a + // new ref when an element's role or accessible name changes, so a recycled node cannot + // inherit an old one. If the ref resolves, it is the field the Bot meant. If it does not, + // nothing is typed, which is the outcome the generation check existed to guarantee. + const field = locateRef(session, target, pending.ref, undefined); + await field.click({ timeout: ACTION_TIMEOUT_MS }); + await field.fill(body.text, { timeout: ACTION_TIMEOUT_MS }); + const characters = body.text.length; + // Cleared only after it actually landed, so a failure leaves the request open and the person + // can try again rather than being told to start over. + session.control.secretSupplied(); + return json({ supplied: true, characters, url: target.url() }); + } catch (error) { + if (error instanceof StaleSnapshotError) { + return json({ error: error.message, stale: true }, 409); + } + // The field is gone, which is unretryable, so the request is closed rather than left open. + // Keeping it open is right for a mistyped value and wrong here: the person would retype their + // password into the same dead ref for ever. Clearing it also unblocks the Bot, which can see + // on its next turn that nothing is pending and ask again against a fresh snapshot. + session.control.secretSupplied(); + return json( + { + error: describe( + error, + "That value could not be entered: the field is no longer on the page. Ask the assistant to request it again.", + ), + }, + 502, + ); + } + } - if (url.pathname === "/navigate" && request.method === "POST") { - const body = (await request.json().catch(() => null)) as { - url?: unknown; - } | null; - const parsed = parseNavigateUrl(body?.url); - if (!parsed.ok) { - return json({ error: parsed.error }, 400); + if ( + ["/control/take", "/control/release", "/control/cancel"].includes( + url.pathname, + ) && + request.method === "POST" + ) { + const body = (await request.json().catch(() => null)) as { + requestId?: unknown; + } | null; + if (typeof body?.requestId !== "string" || !body.requestId.trim()) + return json({ error: "A requestId is required." }, 400); + if (url.pathname === "/control/take") + return json(await session.control.take(body.requestId)); + if (url.pathname === "/control/cancel") + return json(session.control.cancel(body.requestId)); + return json(session.control.release(body.requestId)); } - const startedAt = Date.now(); - try { - const target = await currentPage(botId); - await target.goto(parsed.url, { - waitUntil: "domcontentloaded", - timeout: NAVIGATION_TIMEOUT_MS, - }); - // A new document wipes every stamp, so every ref handed out before now is meaningless. - // Bumping the generation makes an action carrying one fail with "take a new snapshot" rather - // than fall through to a selector that matches nothing and read as a missing element. - session.snapshotId += 1; - const extract = await readablePageText(target); - return json({ - url: target.url(), - title: await target.title(), - text: extract.text, - truncated: extract.truncated, - elapsedMs: Date.now() - startedAt, - }); - } catch (error) { - // The page is the Bot's working surface, so a failed navigation is reported rather than - // thrown: the transcript needs to say what happened, and the browser stays usable. - return json( - { - error: - error instanceof Error ? error.message : "Navigation failed.", - }, - 502, - ); + // A person's input, by pixel. The Bot addresses elements by reference because it reads a list; a + // person addresses them by pointing, because they are looking at a picture. Different problem, + // different endpoint, and only usable while they hold the wheel. + if (HUMAN_INPUT.has(url.pathname) && request.method === "POST") { + if (!session.control.humanMayDrive()) { + return json({ error: TAKE_CONTROL_FIRST }, 409); + } + const body = (await request.json().catch(() => null)) as Record< + string, + unknown + > | null; + if (url.pathname === "/human/scroll") { + const parsed = parseScrollDelta(body?.deltaY); + if (!parsed.ok) { + return json({ error: parsed.error }, 400); + } + } + try { + const target = await currentPage(botId); + return json( + await performHumanInput(target, url.pathname, body ?? {}), + ); + } catch (error) { + return json({ error: describe(error, "That did not work.") }, 502); + } } - } - if (url.pathname === "/screenshot" && request.method === "GET") { - try { - const target = await currentPage(botId); - const buffer = await target.screenshot({ type: "png" }); - const size = target.viewportSize() ?? { width: 1280, height: 800 }; + if (url.pathname === "/health") { + const [profile] = profiles.summary([botId]); return json({ - base64: buffer.toString("base64"), - width: size.width, - height: size.height, - capturedAt: new Date().toISOString(), - // Which page this is a picture of. A browser that has not been sent anywhere sits on - // `about:blank`, and a screenshot of that is a valid, entirely white PNG, indistinguishable - // from a real page to anything looking only at the bytes. The transcript needs to tell - // those apart to avoid presenting a blank browser as though it were a loaded page. - url: target.url(), + status: "ok", + // `browser` kept as it was: it is in the published contract and start.sh reads it. + browser: profile?.running ?? false, + profile, + // Which Bot this computer can prove it is, when the deployment runs SPIRE. Null is a + // deployment without it, not a failure, and it is reported rather than omitted so the + // difference between "no identity here" and "identity broken" is visible. + identity: await identity(), + browserMode: BROWSER_MODE, + browserBackend: RUNTIME.backend, + browserChannel: RUNTIME.channel, }); - } catch (error) { - return json( - { - error: - error instanceof Error ? error.message : "Screenshot failed.", - }, - 502, - ); } - } - // The Bot's files. Confined to the workspace by workspace.ts. Nothing here decides whether a Bot - // MAY touch a path: the gateway in front of this process does that. - if (url.pathname === "/files/read" && request.method === "POST") { - const body = (await request.json().catch(() => null)) as { - path?: unknown; - } | null; - try { - return json(await workspace.read(String(body?.path ?? ""))); - } catch (error) { - return json( - { error: describe(error, "The file could not be read.") }, - fileStatus(error), - ); + /** + * Which run of this Bot's browser the caller is looking at. + * + * The server orders snapshots on `(run, generation)`, and the generation alone cannot carry it: + * this process mints one at zero for every session that is new, so a restart, a redeploy, an + * eviction from the idle sweep and a reset all produce a page at generation one that looks older + * than the page the server still has. Ordering on the run as well is what lets the fresh one land + * and the dead one stop resolving. + * + * A read, and deliberately not on the acting list: a person holding the wheel must not turn every + * ref the server holds into an unanswerable question. + * + * Its own endpoint rather than a field on `/computers`, because that one reads the profile + * directory and this is asked on the path of every governed action. + */ + if (url.pathname === "/run" && request.method === "GET") { + return json({ run: session.run }); } - } - if (url.pathname === "/files/list" && request.method === "POST") { - const body = (await request.json().catch(() => null)) as { - path?: unknown; - } | null; - try { - return json( - await workspace.list( - typeof body?.path === "string" ? body.path : undefined, - ), - ); - } catch (error) { - return json( - { error: describe(error, "The folder could not be listed.") }, - fileStatus(error), - ); + /** + * The computers this process holds. The shape is a list because the admin surface is a + * list, and because a Bot that has a profile has a computer whether or not a browser is running + * for it this second. + */ + if (url.pathname === "/computers" && request.method === "GET") { + return json({ computers: profiles.summary(await profiles.known()) }); } - } - /* - * A command on this computer. - * - * Nothing here decides whether it may run: the gateway already asked the deployment's policy and - * wrote the audit row before this was called. Refusing again here would be a second, quieter - * policy nobody configured. - */ - if (url.pathname === "/exec" && request.method === "POST") { - const body = (await request.json().catch(() => null)) as { - command?: unknown; - timeoutMs?: unknown; - } | null; - if (typeof body?.command !== "string" || !body.command.trim()) { - return json({ error: "A command is required." }, 400); - } - const timeout = parseExecTimeout(body.timeoutMs); - if (!timeout.ok) { - return json({ error: timeout.error }, 400); - } - try { - return json( - await shell.run({ - command: body.command, - ...(timeout.timeoutMs !== undefined - ? { timeoutMs: timeout.timeoutMs } - : {}), - signal: request.signal, - }), - ); - } catch (error) { - return json( - { error: describe(error, "The command could not be run.") }, - 500, + /** + * Stop the browser, keep what it knows. + * + * Closed gracefully so Chromium flushes its profile, and deliberately + * not restarted here: the next request starts it again, which is the same path as a first ever + * start, so there is no second way for a browser to come into existence. + */ + if (url.pathname === "/computers/stop" && request.method === "POST") { + const wasRunning = await profiles.stop(botId); + session.control.interrupt( + "The computer stopped; the previous task was interrupted.", ); + return json({ stopped: true, wasRunning }); } - } - if (url.pathname === "/files/write" && request.method === "POST") { - const body = (await request.json().catch(() => null)) as { - path?: unknown; - contents?: unknown; - append?: unknown; - } | null; - if (typeof body?.contents !== "string") { - return json({ error: "The contents to write are required." }, 400); - } - try { - return json( - await workspace.write(String(body?.path ?? ""), body.contents, { - append: body.append === true, - }), - ); - } catch (error) { - return json( - { error: describe(error, "The file could not be written.") }, - fileStatus(error), + /** + * Forget everything and start over. + * + * Signs the computer out of everything by deleting the profile. Irreversible, which is why it is + * its own endpoint rather than a flag on the one above: a person clicking "stop" must not be able + * to discard a login by mistyping a parameter. + */ + if (url.pathname === "/computers/reset" && request.method === "POST") { + await profiles.reset(botId); + session.control.interrupt( + "The computer was reset; the previous task was interrupted.", ); + /* + * And a new run, because the browser this session described is gone. + * + * Nothing else here says so. The entry stays in the map and the generation counter carries on, + * so a snapshot that was in flight when the wipe landed arrives at the server carrying the same + * run and the same generation as the fresh browser's would: the server deletes its row on + * reset, the late save inserts the wiped page straight back, and every ref on it goes on + * resolving. A new run is what makes those two distinguishable at the far end. + */ + sessions.renewRun(botId); + return json({ reset: true, botId }); } - } - // The current page as text, without navigating anywhere. - // - // Reading must be available after actions too. Returning page text only from `/navigate` would be enough if - // opening a page were the only way to change what is on screen. It is not: the Bot presses - // "Submit order", the page becomes a confirmation, and it has no way to find out what the - // confirmation said. "I clicked the button" is not an answer to what happened. - if (url.pathname === "/read" && request.method === "GET") { - try { - const target = await currentPage(botId); - const extract = await readablePageText(target); - return json({ - url: target.url(), - title: await target.title(), - text: extract.text, - truncated: extract.truncated, - }); - } catch (error) { - return json( - { error: describe(error, "Reading the page failed.") }, - 502, - ); + if (url.pathname === "/navigate" && request.method === "POST") { + const body = (await request.json().catch(() => null)) as { + url?: unknown; + toolCallId?: unknown; + } | null; + const parsed = parseNavigateUrl(body?.url); + if (!parsed.ok) { + return json({ error: parsed.error }, 400); + } + + const startedAt = Date.now(); + try { + const target = await currentPage(botId, "navigate"); + session.control.assertBotMayAct(true); + const response = await navigateWebPage( + target, + parsed.url, + RUNTIME.backend, + NAVIGATION_TIMEOUT_MS, + ); + // A new document wipes every stamp, so every ref handed out before now is meaningless. + // Bumping the generation makes an action carrying one fail with "take a new snapshot" rather + // than fall through to a selector that matches nothing and read as a missing element. + session.snapshotId += 1; + const extract = await readablePageText(target); + return json({ + url: target.url(), + title: await target.title(), + text: extract.text, + truncated: extract.truncated, + elapsedMs: Date.now() - startedAt, + challenge: await detectChallenge(target, session.control, { + cfMitigated: response + ? await response.headerValue("cf-mitigated") + : undefined, + toolCallId: + typeof body?.toolCallId === "string" + ? body.toolCallId + : undefined, + }), + }); + } catch (error) { + if ( + error instanceof ControlError || + error instanceof SnapshotRequiredError + ) + throw error; + // The page is the Bot's working surface, so a failed navigation is reported rather than + // thrown: the transcript needs to say what happened, and the browser stays usable. + return json( + { + error: + error instanceof Error ? error.message : "Navigation failed.", + }, + 502, + ); + } } - } - // The list of things on the page a Bot can act on. POST rather than GET because it mutates the - // page, stamping every element it describes, and a GET that changes the document is a lie that - // caches and prefetchers eventually punish. - if (url.pathname === "/snapshot" && request.method === "POST") { - try { - return json(await snapshotPage(session, await currentPage(botId))); - } catch (error) { - return json({ error: describe(error, "Snapshot failed.") }, 502); + if (url.pathname === "/screenshot" && request.method === "GET") { + try { + const target = await currentPage(botId); + const buffer = await target.screenshot({ type: "png" }); + const size = target.viewportSize() ?? { width: 1280, height: 800 }; + return json({ + base64: buffer.toString("base64"), + width: size.width, + height: size.height, + capturedAt: new Date().toISOString(), + // Which page this is a picture of. A browser that has not been sent anywhere sits on + // `about:blank`, and a screenshot of that is a valid, entirely white PNG, indistinguishable + // from a real page to anything looking only at the bytes. The transcript needs to tell + // those apart to avoid presenting a blank browser as though it were a loaded page. + url: target.url(), + }); + } catch (error) { + return json( + { + error: + error instanceof Error ? error.message : "Screenshot failed.", + }, + 502, + ); + } } - } - if (ACTIONS.has(url.pathname) && request.method === "POST") { - const body = (await request - .json() - .catch(() => null)) as ActionBody | null; - if (!body) { - return json({ error: "An action needs a JSON body." }, 400); + // The Bot's files. Confined to the workspace by workspace.ts. Nothing here decides whether a Bot + // MAY touch a path: the gateway in front of this process does that. + if (url.pathname === "/files/read" && request.method === "POST") { + const body = (await request.json().catch(() => null)) as { + path?: unknown; + } | null; + try { + return json(await workspace.read(String(body?.path ?? ""))); + } catch (error) { + return json( + { error: describe(error, "The file could not be read.") }, + fileStatus(error), + ); + } } - if (url.pathname === "/scroll") { - const parsed = parseScrollDelta(body.deltaY); - if (!parsed.ok) { - return json({ error: parsed.error }, 400); + + if (url.pathname === "/files/list" && request.method === "POST") { + const body = (await request.json().catch(() => null)) as { + path?: unknown; + } | null; + try { + return json( + await workspace.list( + typeof body?.path === "string" ? body.path : undefined, + ), + ); + } catch (error) { + return json( + { error: describe(error, "The folder could not be listed.") }, + fileStatus(error), + ); } } - const startedAt = Date.now(); - try { - const target = await currentPage(botId); - const detail = await performAction( - session, - target, - url.pathname, - body, - // The caller going away is the stop signal: the surface aborts its request, the server - // aborts the one it made to this computer, and Bun aborts this one in turn. - request.signal, - ); - return json({ ...detail, elapsedMs: Date.now() - startedAt }); - } catch (error) { - /* - * Stopped, not failed. The signal is checked rather than the error text: Playwright words an - * abort differently per call, and the caller's own request going away is the fact that - * matters either way. - * - * Logged because the response is not observed after the caller aborts. The log distinguishes - * "stopped in time" from "ran to completion after cancellation". - */ - if (request.signal.aborted) { - console.info( - JSON.stringify({ - type: "action-stopped", - action: url.pathname, - ref: typeof body.ref === "string" ? body.ref : undefined, - elapsedMs: Date.now() - startedAt, + /* + * A command on this computer. + * + * Nothing here decides whether it may run: the gateway already asked the deployment's policy and + * wrote the audit row before this was called. Refusing again here would be a second, quieter + * policy nobody configured. + */ + if (url.pathname === "/exec" && request.method === "POST") { + if (!RUNTIME.allowExec) + return json( + { + error: + "Shell execution is disabled for local Chrome. Use the approved host-access tools to run commands on this computer.", + }, + 403, + ); + const body = (await request.json().catch(() => null)) as { + command?: unknown; + timeoutMs?: unknown; + } | null; + if (typeof body?.command !== "string" || !body.command.trim()) { + return json({ error: "A command is required." }, 400); + } + const timeout = parseExecTimeout(body.timeoutMs); + if (!timeout.ok) { + return json({ error: timeout.error }, 400); + } + try { + return json( + await shell.run({ + command: body.command, + ...(timeout.timeoutMs !== undefined + ? { timeoutMs: timeout.timeoutMs } + : {}), + signal: request.signal, }), ); - // 499, the convention for a client that closed the request: this is not the computer - // failing, and a 502 here would be counted as one. - return json({ error: "Stopped.", stopped: true }, 499); + } catch (error) { + return json( + { error: describe(error, "The command could not be run.") }, + 500, + ); } - // A stale ref is the caller's mistake and is fixable by taking a new snapshot, so it is a 409 - // rather than a 502: the computer is fine and retrying the same call unchanged will not help. - if (error instanceof StaleSnapshotError) { - return json({ error: error.message, stale: true }, 409); + } + + if (url.pathname === "/files/write" && request.method === "POST") { + const body = (await request.json().catch(() => null)) as { + path?: unknown; + contents?: unknown; + append?: unknown; + } | null; + if (typeof body?.contents !== "string") { + return json({ error: "The contents to write are required." }, 400); + } + try { + return json( + await workspace.write(String(body?.path ?? ""), body.contents, { + append: body.append === true, + }), + ); + } catch (error) { + return json( + { error: describe(error, "The file could not be written.") }, + fileStatus(error), + ); + } + } + + // The current page as text, without navigating anywhere. + // + // Reading must be available after actions too. Returning page text only from `/navigate` would be enough if + // opening a page were the only way to change what is on screen. It is not: the Bot presses + // "Submit order", the page becomes a confirmation, and it has no way to find out what the + // confirmation said. "I clicked the button" is not an answer to what happened. + if (url.pathname === "/read" && request.method === "GET") { + try { + const target = await currentPage(botId); + const extract = await readablePageText(target); + return json({ + url: target.url(), + title: await target.title(), + text: extract.text, + truncated: extract.truncated, + challenge: await detectChallenge(target, session.control), + }); + } catch (error) { + return json( + { error: describe(error, "Reading the page failed.") }, + 502, + ); + } + } + + // The list of things on the page a Bot can act on. POST rather than GET because it mutates the + // page, stamping every element it describes, and a GET that changes the document is a lie that + // caches and prefetchers eventually punish. + if (url.pathname === "/snapshot" && request.method === "POST") { + try { + const target = await currentPage(botId); + const snapshot = await snapshotPage(session, target); + return json({ + ...snapshot, + challenge: await detectChallenge(target, session.control), + }); + } catch (error) { + return json({ error: describe(error, "Snapshot failed.") }, 502); + } + } + + if (ACTIONS.has(url.pathname) && request.method === "POST") { + const body = (await request + .json() + .catch(() => null)) as ActionBody | null; + if (!body) { + return json({ error: "An action needs a JSON body." }, 400); + } + if (url.pathname === "/scroll") { + const parsed = parseScrollDelta(body.deltaY); + if (!parsed.ok) { + return json({ error: parsed.error }, 400); + } + } + + const startedAt = Date.now(); + try { + const target = await currentPage(botId); + session.control.assertBotMayAct(true); + const detail = await performAction( + session, + target, + url.pathname, + body, + // The caller going away is the stop signal: the surface aborts its request, the server + // aborts the one it made to this computer, and Bun aborts this one in turn. + request.signal, + ); + return json({ ...detail, elapsedMs: Date.now() - startedAt }); + } catch (error) { + /* + * Stopped, not failed. The signal is checked rather than the error text: Playwright words an + * abort differently per call, and the caller's own request going away is the fact that + * matters either way. + * + * Logged because the response is not observed after the caller aborts. The log distinguishes + * "stopped in time" from "ran to completion after cancellation". + */ + if ( + error instanceof ControlError || + error instanceof SnapshotRequiredError + ) + throw error; + if (request.signal.aborted) { + console.info( + JSON.stringify({ + type: "action-stopped", + action: url.pathname, + ref: typeof body.ref === "string" ? body.ref : undefined, + elapsedMs: Date.now() - startedAt, + }), + ); + // 499, the convention for a client that closed the request: this is not the computer + // failing, and a 502 here would be counted as one. + return json({ error: "Stopped.", stopped: true }, 499); + } + // A stale ref is the caller's mistake and is fixable by taking a new snapshot, so it is a 409 + // rather than a 502: the computer is fine and retrying the same call unchanged will not help. + if (error instanceof StaleSnapshotError) { + return json({ error: error.message, stale: true }, 409); + } + return json({ error: describe(error, "The action failed.") }, 502); } - return json({ error: describe(error, "The action failed.") }, 502); } - } - return json({ error: "Not found." }, 404); + return json({ error: "Not found." }, 404); + } catch (error) { + if (error instanceof ControlError) { + const state = session.control.get(); + return json( + { + error: error.message, + humanHasControl: true, + requestId: state.request?.id, + handoff: state.request, + }, + 409, + ); + } + if (error instanceof SnapshotRequiredError) + return json( + { error: error.message, stale: true, snapshotRequired: true }, + 409, + ); + if (error instanceof ControlRequestError) + return json( + { error: error.message, controlRequestError: true }, + error.status, + ); + throw error; + } finally { + releaseAction?.(); + } }, }); diff --git a/agent-computer/src/navigation.ts b/agent-computer/src/navigation.ts new file mode 100644 index 000000000..d3ad70adf --- /dev/null +++ b/agent-computer/src/navigation.ts @@ -0,0 +1,41 @@ +import type { Page } from "playwright"; +import type { BrowserRuntime } from "./browser-runtime"; +import { parseNavigateUrl } from "./request-validation"; + +type Backend = BrowserRuntime["backend"]; +export type PagePurpose = "read" | "navigate"; + +/** Navigation can leave an unreadable Chrome error/local page, but cannot inspect it. */ +export function assertPageAccess( + backend: Backend, + url: string, + purpose: PagePurpose = "read", +): void { + if (purpose === "navigate") return; + if ( + backend === "local-chrome" && + url !== "about:blank" && + !/^https?:\/\//i.test(url) + ) { + throw new Error( + "Local Chrome computer tools can access only web pages. Use the approved host-access tools for local files.", + ); + } +} + +/** Validate the destination before goto and the resulting page before callers read its content. */ +export async function navigateWebPage( + page: Pick, + url: string, + backend: Backend, + timeoutMs: number, +) { + const parsed = parseNavigateUrl(url); + if (!parsed.ok) throw new Error(parsed.error); + const response = await page.goto(parsed.url, { + waitUntil: "domcontentloaded", + timeout: timeoutMs, + }); + assertPageAccess(backend, page.url()); + return response; +} diff --git a/agent-computer/src/profiles.ts b/agent-computer/src/profiles.ts index 800a9a599..7bbd033ec 100644 --- a/agent-computer/src/profiles.ts +++ b/agent-computer/src/profiles.ts @@ -37,7 +37,7 @@ import { readdir, rm } from "node:fs/promises"; import { join } from "node:path"; import { type BrowserContext, chromium, type Page } from "playwright"; import { profileDirectoryFor } from "./bot-id"; -import { browserModeFromEnv } from "./browser-mode"; +import { browserRuntimeFromEnv } from "./browser-runtime"; import { chooseEvictions, chooseIdle } from "./browser-eviction"; import { egressFor, egressLabel } from "./egress"; import { numberFromEnv, settleWithin } from "./env"; @@ -91,21 +91,21 @@ const SINGLETON_FILES = ["SingletonLock", "SingletonSocket", "SingletonCookie"]; * Said out loud at start-up either way. An operator should not have to read this file to find out * whether the browser rendering the open internet is sandboxed. */ -const SANDBOX_ENABLED = process.env.COMPUTER_SANDBOX === "on"; -const BROWSER_MODE = browserModeFromEnv(process.env.COMPUTER_BROWSER_MODE); +const BROWSER_RUNTIME = browserRuntimeFromEnv(process.env); +const LOCAL_CHROME = BROWSER_RUNTIME.backend === "local-chrome"; +// Native Chrome has no container boundary. Always retain its own process sandbox and OS keychain. +const SANDBOX_ENABLED = LOCAL_CHROME || process.env.COMPUTER_SANDBOX === "on"; const LAUNCH_ARGS = [ ...(SANDBOX_ENABLED ? [] : ["--no-sandbox"]), "--disable-dev-shm-usage", - "--password-store=basic", + ...(LOCAL_CHROME ? [] : ["--password-store=basic"]), // Drop the automation signals Chromium sets for itself, so a real person who takes the wheel can // sign in to a site that refuses obvious automation (Google among them). This is the flag, not a // JS patch of `navigator.webdriver`: the flag turns the property off at the source, where spoofing // it from a script leaves the other tells a detector cross-checks. It does not change what the Bot - // may do; the governed path is unchanged. The larger tell — a headless build reporting - // `HeadlessChrome` in its user agent — is only removed by running headed under a virtual display, - // which is a heavier image change tracked separately; this reduces the signals it can reduce - // without one. + // may do; the governed path is unchanged. Full Chromium is selected explicitly in both modes; + // headed mode also provides a window a person can use on the native desktop or virtual display. "--disable-blink-features=AutomationControlled", ]; @@ -409,10 +409,13 @@ export function createProfiles(root: string, onClosed: BrowserClosed) { const launch = (async () => { const dir = directoryFor(botId); - await sweepLocks(dir); + // A second native helper can target the same data root on another port. Do not remove an + // active Chrome profile's lock; Chrome reports contention and handles its own stale locks. + if (!LOCAL_CHROME) await sweepLocks(dir); const proxy = egressFor(botId, process.env); const context = await chromium.launchPersistentContext(dir, { - headless: BROWSER_MODE === "headless", + channel: BROWSER_RUNTIME.channel, + headless: BROWSER_RUNTIME.mode === "headless", args: LAUNCH_ARGS, // Playwright launches with `--enable-automation`, which sets `navigator.webdriver` and the // "controlled by automated software" banner. Dropped for the same reason as the flag above: diff --git a/agent-computer/src/sessions.ts b/agent-computer/src/sessions.ts index bd2d3a700..5716c054e 100644 --- a/agent-computer/src/sessions.ts +++ b/agent-computer/src/sessions.ts @@ -21,11 +21,12 @@ * * The `Map` below is per-process state, and is allowed to be: it is one container's view of the * browsers it is running, not state that has to survive a replica or agree with one. A restart - * dropping it is precisely the case the run exists for, since nothing else would say the browsers - * were replaced. + * dropping it is precisely the case the run exists for. Request identities are the exception: the + * separate control store retains their outcomes and marks active handoffs interrupted on reload. */ import type { Page } from "playwright"; import { type Control, createControl } from "./control"; +import { createControlStore } from "./control-store"; import { createViewerSlot, type ViewerSlot } from "./viewer"; /** Per-Bot browser-control state. Profiles are isolated, but this process is not a security boundary. */ @@ -43,6 +44,8 @@ export type BotSession = { run: string; /** The page this Bot was last handed, so a change of page can retire its refs. */ livePage?: Page; + /** Context identity separates popup changes from actual browser replacement. */ + browserContext?: object; /** * This Bot's live screen, and who owns it. * @@ -57,6 +60,8 @@ export type SessionsOptions = { isLive: (botId: string) => boolean; /** How many sessions may accumulate before the sweep runs. */ cap?: number; + /** Atomic handoff history outside each Chromium profile. */ + profilesDirectory?: string; /** Injected so a test can name the run it expects rather than match a uuid. */ mintRun?: () => string; }; @@ -92,6 +97,11 @@ export function createSessions(options: SessionsOptions) { for (const [botId, session] of [...sessions.entries()]) { if (session.viewer.occupied()) continue; if (options.isLive(botId)) continue; + if ( + session.control.get().request?.status === "waiting" || + session.control.get().holder === "human" + ) + continue; sessions.delete(botId); } } @@ -99,9 +109,22 @@ export function createSessions(options: SessionsOptions) { function sessionFor(botId: string): BotSession { const existing = sessions.get(botId); if (existing) return existing; + let generation = 0; const created: BotSession = { - control: createControl(), - snapshotId: 0, + control: createControl(undefined, { + ...(options.profilesDirectory + ? { store: createControlStore(options.profilesDirectory, botId) } + : {}), + invalidateSnapshot: () => { + generation += 1; + }, + }), + get snapshotId() { + return generation; + }, + set snapshotId(value: number) { + generation = value; + }, run: mintRun(), viewer: createViewerSlot(), }; @@ -114,6 +137,16 @@ export function createSessions(options: SessionsOptions) { return { for: sessionFor, + observeBrowser(botId: string, context: object): boolean { + const session = sessionFor(botId); + const replaced = + session.browserContext !== undefined && + session.browserContext !== context; + if (replaced) this.renewRun(botId); + session.browserContext = context; + return replaced; + }, + /** * The session this Bot already has, or nothing. * @@ -141,6 +174,11 @@ export function createSessions(options: SessionsOptions) { const existing = sessions.get(botId); // Nothing to renew: a Bot nobody has touched yet gets a session, and its run is already new. if (!existing) return sessionFor(botId).run; + existing.control.interrupt( + "The browser was stopped or replaced; request help again to continue.", + ); + existing.livePage = undefined; + existing.browserContext = undefined; existing.run = mintRun(); return existing.run; }, diff --git a/agent-computer/tests/action-barrier.test.ts b/agent-computer/tests/action-barrier.test.ts new file mode 100644 index 000000000..101f4d367 --- /dev/null +++ b/agent-computer/tests/action-barrier.test.ts @@ -0,0 +1,31 @@ +import { expect, test } from "bun:test"; +import { createActionBarrier } from "../src/action-barrier"; + +test("closing admission refuses new actions and drains admitted work exactly once", async () => { + const barrier = createActionBarrier(); + const release = barrier.enter(); + barrier.close(); + expect(() => barrier.enter()).toThrow(/closed/); + let drained = false; + const waiting = barrier.drain().then(() => { + drained = true; + }); + await Promise.resolve(); + expect(drained).toBe(false); + release(); + release(); + await waiting; + expect(barrier.pending()).toBe(0); + barrier.open(); + barrier.enter()(); +}); + +test("a drain timeout never grants admission", async () => { + const barrier = createActionBarrier(); + const release = barrier.enter(); + barrier.close(); + await expect(barrier.drain(1)).rejects.toThrow(/still finishing/); + expect(() => barrier.enter()).toThrow(); + release(); + await barrier.drain(); +}); diff --git a/agent-computer/tests/browser-mode.test.ts b/agent-computer/tests/browser-mode.test.ts index 12ef2da80..76658e91f 100644 --- a/agent-computer/tests/browser-mode.test.ts +++ b/agent-computer/tests/browser-mode.test.ts @@ -7,7 +7,7 @@ describe("choosing the browser people take over", () => { expect(browserModeFromEnv("")).toBe("headless"); }); - test("runs the full browser only when headed is requested", () => { + test("preserves an explicit headed or headless mode", () => { expect(browserModeFromEnv("headed")).toBe("headed"); expect(browserModeFromEnv("headless")).toBe("headless"); }); diff --git a/agent-computer/tests/browser-runtime.test.ts b/agent-computer/tests/browser-runtime.test.ts new file mode 100644 index 000000000..5b85eeeee --- /dev/null +++ b/agent-computer/tests/browser-runtime.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { browserRuntimeFromEnv } from "../src/browser-runtime"; + +describe("the configured browser runtime", () => { + test.each(["linux", "darwin", "win32"])( + "managed %s uses full headless Chromium", + (platform) => { + expect(browserRuntimeFromEnv({}, platform)).toEqual({ + backend: "managed", + channel: "chromium", + mode: "headless", + useVirtualDisplay: false, + allowExec: true, + }); + }, + ); + + test.each(["linux", "darwin", "win32"])( + "headed %s needs Xvfb only on Linux", + (platform) => { + expect( + browserRuntimeFromEnv({ COMPUTER_BROWSER_MODE: "headed" }, platform) + .useVirtualDisplay, + ).toBe(platform === "linux"); + }, + ); + + test.each(["linux", "darwin", "win32"])( + "local Chrome on %s is headed and confined", + (platform) => { + expect( + browserRuntimeFromEnv( + { COMPUTER_BROWSER_BACKEND: "local-chrome" }, + platform, + ), + ).toEqual({ + backend: "local-chrome", + channel: "chrome", + mode: "headed", + useVirtualDisplay: platform === "linux", + hostname: "127.0.0.1", + allowExec: false, + }); + }, + ); + + test("rejects unknown backends and incompatible local modes", () => { + expect(() => + browserRuntimeFromEnv({ COMPUTER_BROWSER_BACKEND: "cdp" }), + ).toThrow("COMPUTER_BROWSER_BACKEND"); + expect(() => + browserRuntimeFromEnv({ COMPUTER_BROWSER_MODE: "visible" }), + ).toThrow("COMPUTER_BROWSER_MODE"); + expect(() => + browserRuntimeFromEnv({ + COMPUTER_BROWSER_BACKEND: "local-chrome", + COMPUTER_BROWSER_MODE: "headless", + }), + ).toThrow("headed"); + }); +}); diff --git a/agent-computer/tests/challenge.test.ts b/agent-computer/tests/challenge.test.ts new file mode 100644 index 000000000..51aaf749c --- /dev/null +++ b/agent-computer/tests/challenge.test.ts @@ -0,0 +1,48 @@ +import { expect, test } from "bun:test"; +import { classifyChallenge } from "../src/challenge"; + +test("Cloudflare's explicit response signal requests help, generic 403 does not", () => { + expect( + classifyChallenge({ + cfMitigated: "challenge", + text: "", + visibleChallengeControl: false, + })?.kind, + ).toBe("cloudflare"); + expect( + classifyChallenge({ + text: "403 Forbidden", + visibleChallengeControl: false, + }), + ).toBeUndefined(); +}); +test("visible challenges require specific language and visible interactive evidence", () => { + expect( + classifyChallenge({ + text: "Verify you are human", + visibleChallengeControl: true, + })?.kind, + ).toBe("visible-challenge"); + for (const text of [ + "An article about CAPTCHA", + "Sign in to your account", + "Access denied", + "Unusual traffic detected", + ]) { + expect( + classifyChallenge({ text, visibleChallengeControl: false }), + ).toBeUndefined(); + } + expect( + classifyChallenge({ + text: "Verify you are human", + visibleChallengeControl: false, + }), + ).toBeUndefined(); + expect( + classifyChallenge({ + text: "An article about CAPTCHA", + visibleChallengeControl: true, + }), + ).toBeUndefined(); +}); diff --git a/agent-computer/tests/control-http.test.ts b/agent-computer/tests/control-http.test.ts new file mode 100644 index 000000000..3b092d0b9 --- /dev/null +++ b/agent-computer/tests/control-http.test.ts @@ -0,0 +1,150 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ComputerControlState } from "../../shared/computer-control"; + +// A real HTTP dispatcher and shell lease, without launching Chromium. Opt in like live-screen tests +// because agent-computer's own Playwright dependency is not installed by the root unit-test lane. +const asked = process.env.OPENBOT_CONTROL_HTTP === "1"; +let root = ""; +let base = ""; +let child: ReturnType | undefined; +const token = "handoff-http-test-token"; +async function post(path: string, body: unknown) { + return fetch(`${base}${path}`, { + method: "POST", + headers: { + "x-openbot-computer-token": token, + "x-openbot-bot-id": "bot-http", + "content-type": "application/json", + }, + body: JSON.stringify(body), + }); +} +async function control(requestId?: string): Promise { + const response = await fetch( + `${base}/control${requestId ? `?requestId=${requestId}` : ""}`, + { + headers: { + "x-openbot-computer-token": token, + "x-openbot-bot-id": "bot-http", + }, + }, + ); + expect(response.status).toBe(200); + return response.json(); +} +async function until(check: () => Promise) { + const end = Date.now() + 5_000; + while (!(await check())) { + if (Date.now() > end) + throw new Error( + "Timed out waiting for the HTTP fixture's observable condition.", + ); + await Bun.sleep(5); + } +} +beforeAll(async () => { + if (!asked) return; + root = await mkdtemp(join(tmpdir(), "control-http-")); + const probe = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: () => new Response(), + }); + const port = probe.port; + await probe.stop(true); + base = `http://127.0.0.1:${port}`; + child = Bun.spawn([process.execPath, "src/index.ts"], { + cwd: join(import.meta.dir, ".."), + env: { + ...process.env, + COMPUTER_TOKEN: token, + COMPUTER_BROWSER_BACKEND: "managed", + COMPUTER_BROWSER_MODE: "headless", + PORT: String(port), + PROFILES_DIR: join(root, "profiles"), + WORKSPACE_DIR: join(root, "workspace"), + }, + stdout: "ignore", + stderr: "inherit", + }); + await until(async () => { + if (child?.exitCode !== null) + throw new Error(`Computer fixture exited with ${child?.exitCode}.`); + try { + return (await fetch(`${base}/health`)).ok; + } catch { + return false; + } // Explicit readiness probe, bounded above. + }); +}); +afterAll(async () => { + if (!asked) return; + child?.kill(); + if (child) await child.exited; + await rm(root, { recursive: true, force: true }); +}); + +describe.skipIf(!asked)( + "request handoff through the actual computer HTTP dispatcher", + () => { + test("take cannot overtake admitted work; later mutations are refused, then freshness is mandatory", async () => { + const running = post("/exec", { + command: "printf admitted > admitted; sleep 0.3", + timeoutMs: 3000, + }); + await until(() => Bun.file(join(root, "workspace", "admitted")).exists()); + const requested = await post("/control/request", { + reason: "A human step", + toolCallId: "http-call", + }); + const waiting: ComputerControlState = await requested.json(); + expect(waiting.transitioning).toBe(true); + expect(waiting.holder).toBe("bot"); + const requestId = waiting.request!.id; + const taking = post("/control/take", { requestId }); + const refused = await post("/files/write", { + path: "too-late", + contents: "wrong", + }); + expect(refused.status).toBe(409); + expect(await refused.json()).toMatchObject({ + humanHasControl: true, + requestId, + }); + expect((await control(requestId)).holder).toBe("bot"); + expect((await running).status).toBe(200); + expect((await (await taking).json()).holder).toBe("human"); + expect( + (await post("/control/release", { requestId: "wrong" })).status, + ).toBe(409); + expect((await post("/control/release", { requestId })).status).toBe(200); + for (const path of ["/navigate", "/click", "/type", "/key", "/scroll"]) { + const response = await post(path, {}); + expect(response.status).toBe(409); + expect(await response.json()).toMatchObject({ + stale: true, + snapshotRequired: true, + }); + } + expect( + (await post("/files/write", { path: "after", contents: "allowed" })) + .status, + ).toBe(200); + expect((await control(requestId)).request?.status).toBe("completed"); + const next: ComputerControlState = await ( + await post("/control/request", { + reason: "Next", + toolCallId: "next-http-call", + }) + ).json(); + expect((await post("/control/take", { requestId })).status).toBe(409); + expect((await control(requestId)).request?.status).toBe("completed"); + expect( + (await post("/control/cancel", { requestId: next.request!.id })).status, + ).toBe(200); + }); + }, +); diff --git a/agent-computer/tests/control-store.test.ts b/agent-computer/tests/control-store.test.ts new file mode 100644 index 000000000..d75699c1b --- /dev/null +++ b/agent-computer/tests/control-store.test.ts @@ -0,0 +1,103 @@ +import { afterEach, expect, test } from "bun:test"; +import { + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createControl } from "../src/control"; +import { createControlStore } from "../src/control-store"; +const directories: string[] = []; +function fixture() { + const directory = mkdtempSync(join(tmpdir(), "control-store-")); + directories.push(directory); + const store = createControlStore(directory, "bot-1"); + const control = createControl(undefined, { store }); + return { directory, store, control }; +} +afterEach(() => { + for (const directory of directories.splice(0)) + rmSync(directory, { recursive: true, force: true }); +}); +test("atomic persisted completion remains exact across reload and later request", async () => { + const { store, control, directory } = fixture(); + const first = control.requestHelp("Sign in", "call-1").request!; + await control.take(first.id); + control.release(first.id); + control.requestSecret({ ref: "e1", label: "private label" }); + const restored = createControl(undefined, { store }); + expect(restored.get(first.id).request?.status).toBe("completed"); + expect(restored.get().resumeSnapshotRequired).toBe(true); + expect(restored.requestHelp("Replay", "call-1").request?.id).toBe(first.id); + restored.requestHelp("Next", "call-2"); + expect(restored.get(first.id).request?.status).toBe("completed"); + expect(restored.pendingSecret()).toBeNull(); + expect( + readFileSync(join(directory, ".control", "bot-1.json"), "utf8"), + ).not.toContain("private label"); + expect(readdirSync(join(directory, ".control"))).toEqual(["bot-1.json"]); +}); +test("reload interrupts active requests and retains idempotent identity", async () => { + for (const take of [false, true]) { + const { store, control } = fixture(); + const first = control.requestHelp("Sign in", "call-1").request!; + if (take) await control.take(first.id); + const restored = createControl(undefined, { store }); + expect(restored.get(first.id).request?.status).toBe("interrupted"); + expect(restored.get().holder).toBe("bot"); + expect(restored.requestHelp("Replay", "call-1").request?.id).toBe(first.id); + expect(() => restored.admitBotAction(true)).toThrow(); + expect( + createControl(undefined, { store }).get(first.id).request?.status, + ).toBe("interrupted"); + } +}); +test("malformed store fails loudly; Bot ids cannot escape the control directory", () => { + const { directory, control, store } = fixture(); + control.requestHelp("Sign in"); + const path = join(directory, ".control", "bot-1.json"); + for (const malformed of ["{", JSON.stringify({ version: 1, requests: [] })]) { + writeFileSync(path, malformed); + expect(() => createControl(undefined, { store })).toThrow(); + } + expect(() => createControlStore(directory, "../other")).toThrow(); +}); +test("history remains bounded and keeps recent terminal requests", () => { + const { control, store } = fixture(); + let latest = ""; + for (let index = 0; index < 50; index++) { + latest = control.requestHelp("Sign in", `call-${index}`).request!.id; + control.cancel(latest); + } + expect(store.load()!.requests.length).toBeLessThanOrEqual(33); + expect(store.load()!.aliases["call-49"]).toBe(latest); +}); + +test("a write failure never becomes an in-memory success or reopens actions", () => { + const failure = new Error("disk full"); + const control = createControl(undefined, { + store: { + load: () => undefined, + save: () => { + throw failure; + }, + }, + }); + expect(() => control.requestHelp("Sign in")).toThrow(/could not be saved/); + expect(() => control.get()).toThrow(/could not be saved/); + expect(() => control.admitBotAction()).toThrow(/could not be saved/); +}); + +test("opaque tool call ids cannot collide with object prototype names", () => { + const { control } = fixture(); + for (const toolCallId of ["__proto__", "constructor", "toString"]) { + const request = control.requestHelp("Sign in", toolCallId).request!; + control.cancel(request.id); + expect(control.requestHelp("Replay", toolCallId).request?.id).toBe( + request.id, + ); + } +}); diff --git a/agent-computer/tests/control.test.ts b/agent-computer/tests/control.test.ts index afaacbdf8..a4ec16cf8 100644 --- a/agent-computer/tests/control.test.ts +++ b/agent-computer/tests/control.test.ts @@ -1,367 +1,174 @@ import { describe, expect, test } from "bun:test"; import { + createControl, ControlError, ControlRequestError, - createControl, + HELP_REQUEST_TTL_MS, + SnapshotRequiredError, } from "../src/control"; -/** - * The wheel, tested on both paths. - * - * This is the piece standing between two drivers and one page, and until now it had no tests at all, * it lived as a `let` inside the file that imports playwright, so a test could not reach it without - * launching Chrome. What is checked here is mostly the refusal path, because that is where this - * component earns its keep: a Bot clicking while a person types, a secret typed when nothing asked for - * one, a request answered twice, a handover that leaves a password box open behind it. - * - * A fake clock is injected so `since` can be asserted rather than shrugged at. - */ function fixture() { - let tick = 0; - const at = () => `2026-08-14T00:00:0${tick}.000Z`; - const control = createControl(() => { - tick += 1; - return at(); - }); - return { control }; + let tick = Date.parse("2026-09-26T00:00:00Z"); + const control = createControl(() => new Date(tick).toISOString()); + const advance = (ms: number) => { + tick += ms; + }; + const request = (toolCallId = "call-1") => + control.requestHelp("Sign in to continue.", toolCallId).request!; + return { control, advance, request }; } -describe("the happy path: ask, hand over, hand back", () => { - test("starts with the Bot driving and nothing pending", () => { - const { control } = fixture(); - const state = control.get(); - expect(state.holder).toBe("bot"); - expect(state.requested).toBe(false); - expect(state.reason).toBeUndefined(); - expect(control.pendingSecret()).toBeNull(); - // Nothing to refuse yet. - expect(() => control.assertBotMayAct()).not.toThrow(); - }); - - test("the Bot asking for help does NOT hand itself the human's authority", () => { - const { control } = fixture(); - const state = control.requestHelp("There is a login wall."); - // The flag is raised and a person decides. A Bot that could take control on its own behalf could - // also hand a person a page they never asked to see. - expect(state.requested).toBe(true); - expect(state.reason).toBe("There is a login wall."); - expect(state.holder).toBe("bot"); - // And it may still act while it waits: asking is not being blocked. - expect(() => control.assertBotMayAct()).not.toThrow(); +describe("request-aware handoff", () => { + test("asking closes admission without granting human input", () => { + const { control, request } = fixture(); + expect(control.get()).toMatchObject({ + holder: "bot", + requested: false, + transitioning: false, + resumeSnapshotRequired: false, + }); + const help = request(); + expect(help.status).toBe("waiting"); + expect(control.get().requested).toBe(true); + expect(control.humanMayDrive()).toBe(false); + expect(() => control.admitBotAction()).toThrow(ControlError); }); - test("taking the wheel keeps the reason and lowers the flag", () => { - const { control } = fixture(); - control.requestHelp("Sign in to continue."); - const state = control.take(); - expect(state.holder).toBe("human"); - // The reason survives, because it is the thing the person was just asked to do. - expect(state.reason).toBe("Sign in to continue."); - // The request is answered, so the surface stops asking. - expect(state.requested).toBe(false); + test("take drains an old action; detector can request help inside its own lease", async () => { + const { control, request } = fixture(); + const finish = control.admitBotAction(true); + const help = request(); + expect(control.get().transitioning).toBe(true); + const taking = control.take(help.id); + await Promise.resolve(); + expect(control.humanMayDrive()).toBe(false); + expect(() => control.admitBotAction()).toThrow(ControlError); + finish(); + expect((await taking).holder).toBe("human"); + expect(control.get().transitioning).toBe(false); + }); + + test("only waiting expires; taken control has no ten-minute limit", async () => { + const waiting = fixture(); + const help = waiting.request(); + waiting.advance(HELP_REQUEST_TTL_MS + 1); + expect(waiting.control.get(help.id).request?.status).toBe("expired"); + expect(waiting.control.get().requested).toBe(false); + expect(() => waiting.control.admitBotAction()()).not.toThrow(); + await expect(waiting.control.take(help.id)).rejects.toThrow(/no longer/); + const taken = fixture(); + const held = taken.request(); + await taken.control.take(held.id); + taken.advance(HELP_REQUEST_TTL_MS * 10); + expect(taken.control.get(held.id).request?.status).toBe("taken"); + expect(taken.control.get().request?.expiresAt).toBeUndefined(); + expect(taken.control.humanMayDrive()).toBe(true); + }); + + test("completion requires fresh snapshot even on the same page; files need only ownership", async () => { + const { control, request } = fixture(); + const help = request(); + await control.take(help.id); + control.snapshotTaken(); + control.release(help.id); + expect(control.get().request?.status).toBe("completed"); + expect(() => control.admitBotAction(true)).toThrow(SnapshotRequiredError); + control.admitBotAction(false)(); + control.snapshotTaken(); + control.admitBotAction(true)(); + expect(control.release(help.id).request?.status).toBe("completed"); + }); + + test("idempotency and exact historical lookup survive later requests", async () => { + const { control, request } = fixture(); + const first = request(); + expect(request().id).toBe(first.id); + expect(request("parallel-call").id).toBe(first.id); + await control.take(first.id); + await control.take(first.id); + control.release(first.id); + const next = request("call-2"); + expect(next.id).not.toBe(first.id); + expect(control.get(first.id).request?.status).toBe("completed"); + expect(request().id).toBe(first.id); + expect(request("parallel-call").id).toBe(first.id); + expect(control.get().request?.id).toBe(next.id); + expect(() => control.release(first.id)).toThrow(/no longer/); + await expect(control.take(first.id)).rejects.toThrow(/no longer/); + expect(() => control.get("unknown")).toThrow(/not found/); + }); + + test("cancel while taken preserves ownership and cannot later become completion", async () => { + const { control, request } = fixture(); + const help = request(); + await control.take(help.id); + expect(control.cancel(help.id).request?.status).toBe("cancelled"); expect(control.humanMayDrive()).toBe(true); - }); - - test("handing back returns the wheel and clears the old request", () => { - const { control } = fixture(); - control.requestHelp("Sign in to continue."); - control.take(); - const state = control.release(); - expect(state.holder).toBe("bot"); - // Dropped on purpose: leaving it set has the surface still showing a request that was dealt with. - expect(state.reason).toBeUndefined(); - expect(state.requested).toBe(false); + expect(control.release(help.id).request?.status).toBe("cancelled"); expect(control.humanMayDrive()).toBe(false); - expect(() => control.assertBotMayAct()).not.toThrow(); - }); - - test("`since` moves on a handover and not on a request", () => { - const { control } = fixture(); - const created = control.get().since; - control.requestHelp("Stuck."); - // Asking for help is not a change of driver, so the clock does not restart. - expect(control.get().since).toBe(created); - expect(control.take().since).not.toBe(created); - }); -}); - -describe("the crappy paths: two drivers, one page", () => { - test("the Bot is refused while a person holds the wheel", () => { - const { control } = fixture(); - control.take(); - expect(() => control.assertBotMayAct()).toThrow(ControlError); - // Refused with a reason the Bot can act on, wait, rather than a bare failure. - expect(() => control.assertBotMayAct()).toThrow(/hand it back/); - }); - - test("the refusal lifts the moment the person hands back", () => { - const { control } = fixture(); - control.take(); - control.release(); - expect(() => control.assertBotMayAct()).not.toThrow(); - }); - - test("a person's input is not applied merely because they asked", () => { - const { control } = fixture(); - control.requestHelp("Sign in."); - // The Bot asked for help and no person has taken the wheel. An open socket is not permission: this is - // what stops anything that can reach the port from driving the browser mid-task. + expect(control.get().resumeSnapshotRequired).toBe(true); + }); + + test("cancel waiting and reset report terminal outcomes, never success", async () => { + const { control, request } = fixture(); + const cancelled = request(); + control.cancel(cancelled.id); + expect(() => control.release(cancelled.id)).toThrow(); + const interrupted = request("call-2"); + await control.take(interrupted.id); + control.interrupt("The browser restarted."); + expect(control.get(interrupted.id).request).toMatchObject({ + status: "interrupted", + interruption: "The browser restarted.", + }); expect(control.humanMayDrive()).toBe(false); + expect(() => control.admitBotAction(true)).toThrow(ControlError); + expect(() => control.release(interrupted.id)).toThrow(); }); - test("taking the wheel twice is not a way to lose the reason", () => { - const { control } = fixture(); - control.requestHelp("Sign in."); - control.take(); - const state = control.take(); - expect(state.holder).toBe("human"); - expect(state.reason).toBe("Sign in."); - }); - - test("handing back when the Bot already has it is harmless", () => { - const { control } = fixture(); - const state = control.release(); - expect(state.holder).toBe("bot"); - expect(() => control.assertBotMayAct()).not.toThrow(); - }); - - test("the caller cannot reach in and change the state it was handed", () => { + test("reads are copies and reasons are bounded", () => { const { control } = fixture(); - const state = control.get(); - state.holder = "human"; - // A copy, so reading the state is not a way to take the wheel. - expect(control.get().holder).toBe("bot"); - }); - - test("junk reasons fall back to something a person can read", () => { - const { control } = fixture(); - // The wire carries whatever the caller sent. An empty or non-string reason must not leave the - // person staring at a blank explanation of why they have just been handed a browser. - for (const junk of ["", " ", null, undefined, 42, {}]) { - const { control: fresh } = fixture(); - expect(fresh.requestHelp(junk).reason).toBe( - "The assistant needs a person to continue.", - ); - } - expect(control.requestHelp(" Trimmed. ").reason).toBe("Trimmed."); + const state = control.requestHelp("x".repeat(900)); + expect(state.reason).toHaveLength(500); + state.request!.status = "completed"; + expect(control.get().request?.status).toBe("waiting"); + expect(createControl().requestHelp(null).reason).toBe( + "The assistant needs a person to continue.", + ); }); }); -describe("the crappy paths: secrets", () => { - test("a secret request must name the field it goes in", () => { +describe("secret entry remains scoped and ephemeral", () => { + test("requires a field, records only label/ref/snapshot, and clears after delivery", () => { const { control } = fixture(); - // The version without this typed the value into whatever happened to have focus, and reported - // success when that was nothing at all. - for (const bad of [{}, { ref: "" }, { ref: " " }, { ref: 7 }]) { - expect(() => control.requestSecret(bad)).toThrow(ControlRequestError); - } - // A request error, not a control refusal: the caller asked wrongly and no driver changed. - expect(() => control.requestSecret({})).toThrow(/which field/); - expect(control.pendingSecret()).toBeNull(); - }); - - test("a secret request records the label and the field, and nothing else", () => { - const { control } = fixture(); - const state = control.requestSecret({ - label: " the six-digit code ", - ref: "e12", - snapshotId: 3, - }); - expect(state.secretWanted).toBe("the six-digit code"); - expect(state.secretRef).toBe("e12"); - expect(state.secretSnapshotId).toBe(3); + for (const ref of [undefined, "", " ", 7]) + expect(() => control.requestSecret({ ref })).toThrow(ControlRequestError); + control.requestSecret({ label: " code ", ref: "e12", snapshotId: 3 }); expect(control.pendingSecret()).toEqual({ ref: "e12", snapshotId: 3 }); - }); - - test("an unlabelled request still says something honest", () => { - const { control } = fixture(); - expect(control.requestSecret({ ref: "e1" }).secretWanted).toBe( - "the value this page is asking for", - ); - }); - - test("a non-numeric snapshotId is dropped rather than carried as junk", () => { - const { control } = fixture(); - const state = control.requestSecret({ ref: "e1", snapshotId: "3" }); - // Carried through to `locateRef`, where a string would prevent the numeric staleness check from - // matching and could let a stale field accept the secret. - expect(state.secretSnapshotId).toBeUndefined(); - }); - - test("nothing is pending until the Bot asks", () => { - const { control } = fixture(); - // What makes the masked box scoped rather than a general-purpose way to type into the page. + expect(control.get().secretWanted).toBe("code"); + control.secretSupplied(); expect(control.pendingSecret()).toBeNull(); }); - - test("a supplied secret closes the request, so it cannot be answered twice", () => { - const { control } = fixture(); - control.requestSecret({ ref: "e12", label: "code" }); - control.secretSupplied(); + test("expires unanswered secrets and admits a fresh request afterward", () => { + const { control, advance } = fixture(); + control.requestSecret({ ref: "e1" }); + advance(HELP_REQUEST_TTL_MS + 1); expect(control.pendingSecret()).toBeNull(); - expect(control.get().secretWanted).toBeUndefined(); expect(control.get().secretRef).toBeUndefined(); - }); - - test("a FAILED attempt leaves the request open", () => { - const { control } = fixture(); - control.requestSecret({ ref: "e12", label: "code" }); - // `secretSupplied` is called only after the value reached the field, so a field that could not be - // found leaves this pending and the person can try again instead of starting over. - expect(control.pendingSecret()).not.toBeNull(); - }); - - test("handing the wheel over or back closes any pending secret", () => { - for (const handover of ["take", "release"] as const) { - const { control } = fixture(); - control.requestSecret({ ref: "e12", label: "password" }); - control[handover](); - // A person who drove the browser themselves has dealt with the login. A masked box still asking - // for a password afterwards is asking for a secret nothing is waiting for. - expect(control.pendingSecret()).toBeNull(); - expect(control.get().secretWanted).toBeUndefined(); - } - }); - - test("the secret VALUE is never anywhere in the state", () => { - const { control } = fixture(); - control.requestSecret({ ref: "e12", label: "one-time code" }); - // The machine has no field that could hold it, and this test exists to fail if one is ever added. - // The value passes through a single request, into the page, and is not kept. - const serialised = JSON.stringify(control.get()); - expect(serialised).not.toContain("value:"); - expect( - Object.keys(control.get()) - .filter((k) => /secret/i.test(k)) - .sort(), - ).toEqual(["secretRef", "secretSnapshotId", "secretWanted"]); - }); -}); - -/** - * A request nobody answered does not outlive the run that made it. - * - * Control belongs to the computer, not to a conversation, and an unanswered request used to sit on - * it forever. The run that asked had ended, but every later conversation with that Bot showed a live - * "Take control" for work it was not doing — and showed the reason the Bot gave, which is written - * for whoever asked and was being rendered to whoever looked. - * - * Seen in the product: a brand new channel, on an unrelated question, displaying "Google Docs is - * asking for sign-in before I can read the PRD document" from a conversation minutes earlier. - */ -describe("an unanswered request to take the wheel", () => { - test("is still shown inside the window", () => { - let clock = "2026-08-22T03:00:00.000Z"; - const control = createControl(() => clock); - control.requestHelp("sign in to Drive"); - - clock = "2026-08-22T03:05:00.000Z"; - const state = control.get(); - expect(state.requested).toBe(true); - expect(state.reason).toBe("sign in to Drive"); - }); - - test("stops being shown once it is stale, and takes its reason with it", () => { - let clock = "2026-08-22T03:00:00.000Z"; - const control = createControl(() => clock); - control.requestHelp("sign in to Drive"); - - clock = "2026-08-22T03:20:00.000Z"; - const state = control.get(); - expect(state.requested).toBe(false); - // The reason is the part that leaked between conversations, so it goes too. - expect(state.reason).toBeUndefined(); - }); - - test("never takes the wheel back off a person who holds it", () => { - /* - * The one case that must not expire. Somebody may be halfway through typing a code, and pulling - * the browser back mid-sign-in is worse than any stale prompt. Only the ASK times out. - */ - let clock = "2026-08-22T03:00:00.000Z"; - const control = createControl(() => clock); - control.requestHelp("sign in to Drive"); - control.take(); - - clock = "2026-08-22T04:00:00.000Z"; - expect(control.get().holder).toBe("human"); - }); -}); - -/** - * The other half of the same request, which did not expire at all. - * - * A request for a secret is an ask like the one above and outlived its run the same way: the label - * the Bot wrote is rendered to whoever looks next, and the surface makes no distinction between the - * two — `useNeedsYou` lights the same "needs you" on `requested` and on `secretWanted` — so timing - * one out and not the other left the Bot flagged for a conversation that ended anyway, now asking - * for a password rather than for a hand. - * - * It is also the prompt where being stale matters more. Answering it types a value into a field - * named by a ref from a snapshot the browser has long since moved past, so the person is being asked - * for their password by a request nothing is waiting for. - */ -describe("an unanswered request for a secret", () => { - test("is still shown, and still answerable, inside the window", () => { - let clock = "2026-08-22T03:00:00.000Z"; - const control = createControl(() => clock); - control.requestSecret({ ref: "e12", label: "the six-digit code" }); - - clock = "2026-08-22T03:05:00.000Z"; - expect(control.get().secretWanted).toBe("the six-digit code"); + control.requestSecret({ ref: "e2", snapshotId: "junk" }); expect(control.pendingSecret()).toEqual({ - ref: "e12", + ref: "e2", snapshotId: undefined, }); }); - - test("stops being shown once it is stale, and takes the field it named with it", () => { - let clock = "2026-08-22T03:00:00.000Z"; - const control = createControl(() => clock); - control.requestSecret({ - ref: "e12", - label: "the six-digit code", - snapshotId: 4, - }); - - clock = "2026-08-22T03:20:00.000Z"; - const state = control.get(); - // The label is the part that was being rendered to whoever looked, so it goes, and the field it - // named goes with it: half a request is not a thing anything downstream knows how to read. - expect(state.secretWanted).toBeUndefined(); - expect(state.secretRef).toBeUndefined(); - expect(state.secretSnapshotId).toBeUndefined(); - }); - - test("stops being answerable at the same moment it stops being shown", () => { - /* - * Asked through `pendingSecret` alone, without a `get` first. That is the call `/human/secret` - * makes before it types, and it is the one that decides whether a value supplied now reaches the - * page: expiring only on the path the surface polls would leave a prompt that is no longer - * displayed still able to accept a password. - */ - let clock = "2026-08-22T03:00:00.000Z"; - const control = createControl(() => clock); - control.requestSecret({ ref: "e12", label: "the six-digit code" }); - - clock = "2026-08-22T03:20:00.000Z"; + test("take and release clear outstanding secret fields", async () => { + const { control, request } = fixture(); + const help = request(); + control.requestSecret({ ref: "e1" }); + await control.take(help.id); expect(control.pendingSecret()).toBeNull(); - }); - - test("a fresh request after a stale one is shown, not swallowed by it", () => { - // The expiry must clear its own bookkeeping, or the next request inherits the old timestamp and - // is stale on arrival: a Bot that asked twice would be answerable neither time. - let clock = "2026-08-22T03:00:00.000Z"; - const control = createControl(() => clock); - control.requestSecret({ ref: "e12", label: "the six-digit code" }); - - clock = "2026-08-22T03:20:00.000Z"; + control.requestSecret({ ref: "e2" }); + control.release(help.id); expect(control.pendingSecret()).toBeNull(); - - control.requestSecret({ ref: "e40", label: "the code, again" }); - expect(control.get().secretWanted).toBe("the code, again"); - expect(control.pendingSecret()).toEqual({ - ref: "e40", - snapshotId: undefined, - }); }); }); diff --git a/agent-computer/tests/live-screen.test.ts b/agent-computer/tests/live-screen.test.ts index c3bc40c7a..87f1159f1 100644 --- a/agent-computer/tests/live-screen.test.ts +++ b/agent-computer/tests/live-screen.test.ts @@ -97,6 +97,19 @@ function api(path: string, botId: string, init?: RequestInit) { }); } +async function takeControl(botId: string) { + const requested = await api("/control/request", botId, { + method: "POST", + body: JSON.stringify({ reason: "Manual control for the screen test" }), + }); + const state: { request: { id: string } } = await requested.json(); + const taken = await api("/control/take", botId, { + method: "POST", + body: JSON.stringify({ requestId: state.request.id }), + }); + expect(taken.status).toBe(200); +} + type Frames = { socket: WebSocket; /** Every error the server sent this socket, in order. */ @@ -264,7 +277,7 @@ describe.skipIf(!asked)("a socket that another connection replaced", () => { const second = watch(botId); await second.casting; - await api("/control/take", botId, { method: "POST" }); + await takeControl(botId); first.socket.send(JSON.stringify({ type: "key", key: "z" })); // The exact refusal, not merely some error. Dispatching through a cast the sender does not own @@ -306,7 +319,7 @@ describe.skipIf(!asked)("a superseded socket closing later", () => { // The survivor still owns the screen, and the proof is that its typing arrives: a cast that was // stopped underneath it, or an ownership it quietly lost, would refuse this instead. - await api("/control/take", botId, { method: "POST" }); + await takeControl(botId); second.socket.send(JSON.stringify({ type: "key", key: "k" })); let landed = ""; @@ -333,7 +346,7 @@ describe.skipIf(!asked)("printable punctuation from the live screen", () => { }); const viewer = watch(botId); await viewer.casting; - await api("/control/take", botId, { method: "POST" }); + await takeControl(botId); viewer.socket.send( JSON.stringify({ @@ -406,7 +419,7 @@ describe.skipIf(!asked)( method: "POST", body: JSON.stringify({ url: TYPING_PAGE }), }); - await api("/control/release", botId, { method: "POST" }); + // A fresh Bot starts with Bot ownership; no handoff exists to release. const viewer = watch(botId); await viewer.casting; diff --git a/agent-computer/tests/local-chrome-startup.test.ts b/agent-computer/tests/local-chrome-startup.test.ts new file mode 100644 index 000000000..9944d8e5b --- /dev/null +++ b/agent-computer/tests/local-chrome-startup.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { createServer } from "node:net"; +import { + localChromeConfiguration, + requireAvailablePort, + requireInstalledChrome, +} from "../../scripts/start-local-chrome-computer"; + +describe("starting the local computer", () => { + test("requires the existing API token and never includes it in an error", () => { + expect(() => + localChromeConfiguration({}, "darwin", "/Users/operator"), + ).toThrow("COMPUTER_TOKEN"); + }); + + test("isolates profiles and files from inherited paths", () => { + const config = localChromeConfiguration( + { + COMPUTER_TOKEN: "test-token", + OPENBOT_LOCAL_COMPUTER_DIR: "/tmp/openbot-local-test", + PROFILES_DIR: + "/Users/operator/Library/Application Support/Google/Chrome", + WORKSPACE_DIR: "/Users/operator", + }, + "darwin", + "/Users/operator", + ); + expect(config.env.PROFILES_DIR).toBe("/tmp/openbot-local-test/profiles"); + expect(config.env.WORKSPACE_DIR).toBe("/tmp/openbot-local-test/workspace"); + expect(config.env.COMPUTER_BROWSER_BACKEND).toBe("local-chrome"); + expect(config.env.COMPUTER_BROWSER_MODE).toBe("headed"); + expect(config.env.COMPUTER_SANDBOX).toBe("on"); + expect(config.env.PORT).toBe("4101"); + expect(config.env.COMPUTER_TOKEN).toBe("test-token"); + }); + + test("uses platform user data roots", () => { + const env = { COMPUTER_TOKEN: "test-token" }; + expect( + localChromeConfiguration(env, "darwin", "/Users/operator").root, + ).toBe( + "/Users/operator/Library/Application Support/OpenBot/local-computer", + ); + expect( + localChromeConfiguration( + { ...env, XDG_DATA_HOME: "/data" }, + "linux", + "/home/operator", + ).root, + ).toBe("/data/openbot/local-computer"); + expect( + localChromeConfiguration( + { ...env, LOCALAPPDATA: "C:\\Users\\Operator\\AppData\\Local" }, + "win32", + "C:\\Users\\Operator", + ).root, + ).toBe("C:\\Users\\Operator\\AppData\\Local\\OpenBot\\local-computer"); + }); + + test("rejects a relative data directory, invalid port and an incompatible backend", () => { + const env = { COMPUTER_TOKEN: "test-token" }; + expect(() => + localChromeConfiguration({ + ...env, + OPENBOT_LOCAL_COMPUTER_DIR: "profiles", + }), + ).toThrow("absolute"); + expect(() => localChromeConfiguration({ ...env, PORT: "0" })).toThrow( + "PORT", + ); + expect(() => + localChromeConfiguration({ ...env, COMPUTER_BROWSER_BACKEND: "managed" }), + ).toThrow("COMPUTER_BROWSER_BACKEND"); + expect(() => + localChromeConfiguration({ ...env, COMPUTER_BROWSER_MODE: "headless" }), + ).toThrow("headed"); + }); + + test("missing token exits before starting the computer or creating data", async () => { + const child = Bun.spawn( + [ + process.execPath, + "--no-env-file", + join(import.meta.dir, "../../scripts/start-local-chrome-computer.ts"), + ], + { + env: { COMPUTER_TOKEN: "" }, + stdout: "pipe", + stderr: "pipe", + }, + ); + const [code, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + expect(code).toBe(1); + expect(stdout).toBe(""); + expect(stderr).toContain("COMPUTER_TOKEN is required"); + }); + + test("missing installed Chrome fails clearly without launching a fallback", async () => { + await expect(requireInstalledChrome({}, "win32")).rejects.toThrow( + "Google Chrome was not found", + ); + }); + + test("occupied loopback port fails instead of choosing another port", async () => { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + try { + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Expected TCP listener"); + await expect(requireAvailablePort(address.port)).rejects.toThrow( + `Cannot listen on 127.0.0.1:${address.port}`, + ); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }); +}); diff --git a/agent-computer/tests/navigation-recovery.test.ts b/agent-computer/tests/navigation-recovery.test.ts new file mode 100644 index 000000000..60f3b8afb --- /dev/null +++ b/agent-computer/tests/navigation-recovery.test.ts @@ -0,0 +1,73 @@ +import { expect, test } from "bun:test"; +import type { Page } from "playwright"; +import { assertPageAccess, navigateWebPage } from "../src/navigation"; + +function fixture() { + let url = "https://example.com/"; + let failure = true; + const visited: string[] = []; + const page: Pick = { + url: () => url, + async goto(target) { + visited.push(target); + if (failure) { + failure = false; + url = "chrome-error://chromewebdata/"; + throw new Error("net::ERR_CONNECTION_REFUSED"); + } + url = target; + return null; + }, + }; + return { page, visited }; +} + +test("a failed navigation can recover from Chrome's internal error page without exposing it", async () => { + const { page, visited } = fixture(); + await expect( + navigateWebPage(page, "http://127.0.0.1:64110/", "local-chrome", 1000), + ).rejects.toThrow("ERR_CONNECTION_REFUSED"); + expect(() => assertPageAccess("local-chrome", page.url())).toThrow( + /only web pages/, + ); + assertPageAccess("local-chrome", page.url(), "navigate"); + await navigateWebPage( + page, + "https://example.com/recovered", + "local-chrome", + 1000, + ); + expect(visited).toEqual([ + "http://127.0.0.1:64110/", + "https://example.com/recovered", + ]); + expect(() => assertPageAccess("local-chrome", page.url())).not.toThrow(); +}); + +test("navigation never accepts file/chrome targets or returns a nonweb redirect result", async () => { + const { page, visited } = fixture(); + for (const url of [ + "file:///etc/passwd", + "chrome://settings", + "javascript:alert(1)", + ]) { + expect(() => assertPageAccess("local-chrome", url)).toThrow(); + await expect( + navigateWebPage(page, url, "local-chrome", 1000), + ).rejects.toThrow(); + } + expect(visited).toHaveLength(0); + const redirected: Pick = { + goto: async () => null, + url: () => "file:///private/secret", + }; + await expect( + navigateWebPage( + redirected, + "https://example.com/redirect", + "local-chrome", + 1000, + ), + ).rejects.toThrow(/only web pages/); + expect(() => assertPageAccess("local-chrome", "about:blank")).not.toThrow(); +}); diff --git a/agent-computer/tests/profile-launch-options.test.ts b/agent-computer/tests/profile-launch-options.test.ts new file mode 100644 index 000000000..de0e8f1f1 --- /dev/null +++ b/agent-computer/tests/profile-launch-options.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import { existsSync } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +const original = { ...process.env }; +afterEach(() => { + for (const name of [ + "COMPUTER_BROWSER_BACKEND", + "COMPUTER_BROWSER_MODE", + "COMPUTER_SANDBOX", + ]) { + if (original[name] === undefined) delete process.env[name]; + else process.env[name] = original[name]; + } +}); + +// The root CI install deliberately excludes this separately deployed package's dependencies. +describe.skipIf( + !existsSync(join(import.meta.dir, "../node_modules/playwright/package.json")), +)("persistent launch options", () => { + test.each(["managed", "local-chrome"])( + "%s passes the runtime to the persistent launcher", + async (backend) => { + const { chromium } = await import("playwright"); + process.env.COMPUTER_BROWSER_BACKEND = backend; + delete process.env.COMPUTER_BROWSER_MODE; + delete process.env.COMPUTER_SANDBOX; + const launch = spyOn( + chromium, + "launchPersistentContext", + ).mockImplementation(async () => { + throw new Error("launch inspected without starting a browser"); + }); + const { createProfiles } = await import( + `../src/profiles?launch-options=${backend}` + ); + const root = await mkdtemp(join(tmpdir(), "openbot-launch-options-")); + const profiles = createProfiles(root); + try { + await expect(profiles.page("bot-one")).rejects.toThrow( + "launch inspected", + ); + expect(launch).toHaveBeenCalledTimes(1); + const [directory, options] = launch.mock.calls[0]!; + expect(directory).toBe(join(root, "bot-one")); + expect(options?.channel).toBe( + backend === "managed" ? "chromium" : "chrome", + ); + expect(options?.headless).toBe(backend === "managed"); + expect(options?.handleSIGINT).toBe(false); + expect(options?.handleSIGTERM).toBe(false); + expect(options?.ignoreDefaultArgs).toEqual(["--enable-automation"]); + if (backend === "local-chrome") { + expect(options?.chromiumSandbox).toBe(true); + expect(options?.args).not.toContain("--password-store=basic"); + expect(options?.args).not.toContain("--no-sandbox"); + } + } finally { + await profiles.closeAll(); + launch.mockRestore(); + await rm(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/agent-computer/tests/reset-run.test.ts b/agent-computer/tests/reset-run.test.ts index 29a58d931..39c08b396 100644 --- a/agent-computer/tests/reset-run.test.ts +++ b/agent-computer/tests/reset-run.test.ts @@ -170,12 +170,24 @@ describe.skipIf(!asked)("the run this computer reports", () => { * governed action, including the ones it is about to refuse, so a 409 here would turn every ref * it holds into an unanswerable question for as long as somebody was driving. */ + const requested = await fetch(`${BASE}/control/request`, { + method: "POST", + headers: { + "x-openbot-bot-id": "bot-8", + "x-openbot-computer-token": TOKEN, + "content-type": "application/json", + }, + body: JSON.stringify({ reason: "Read the run while a human drives" }), + }); + const state: { request: { id: string } } = await requested.json(); const taken = await fetch(`${BASE}/control/take`, { method: "POST", headers: { "x-openbot-bot-id": "bot-8", "x-openbot-computer-token": TOKEN, + "content-type": "application/json", }, + body: JSON.stringify({ requestId: state.request.id }), }); expect(taken.status).toBe(200); diff --git a/agent-computer/tests/sessions.test.ts b/agent-computer/tests/sessions.test.ts index 85605dfc6..86fc152f4 100644 --- a/agent-computer/tests/sessions.test.ts +++ b/agent-computer/tests/sessions.test.ts @@ -137,7 +137,7 @@ describe("renewing a run", () => { expect(sessions.renewRun("bot-1")).toBe("run-1"); }); - test("the generation carries on, because the browser is what restarts it", () => { + test("replacement invalidates snapshots without resetting the generation", () => { // Deliberately not reset here. The counter belongs to the browser, and `sessionFor` mints it at // zero for a session that is new; a renew that also zeroed it would make a fresh generation one // arrive under a fresh run, which is the pair the server has no way to order. @@ -148,6 +148,22 @@ describe("renewing a run", () => { const session = sessions.for("bot-1"); session.snapshotId = 7; sessions.renewRun("bot-1"); - expect(sessions.for("bot-1").snapshotId).toBe(7); + expect(sessions.for("bot-1").snapshotId).toBe(8); }); }); + +test("a replacement context interrupts a live handoff; the same context keeps it", async () => { + const sessions = createSessions({ isLive: () => true, mintRun: counting() }); + const context = {}; + const session = sessions.for("bot-1"); + expect(sessions.observeBrowser("bot-1", context)).toBe(false); + const request = session.control.requestHelp("Sign in").request!; + await session.control.take(request.id); + const originalRun = session.run; + expect(sessions.observeBrowser("bot-1", context)).toBe(false); + expect(session.control.get().request?.status).toBe("taken"); + expect(sessions.observeBrowser("bot-1", {})).toBe(true); + expect(session.run).not.toBe(originalRun); + expect(session.control.get(request.id).request?.status).toBe("interrupted"); + expect(session.control.humanMayDrive()).toBe(false); +}); diff --git a/app/src/components/channels/channel-chat.tsx b/app/src/components/channels/channel-chat.tsx index 7e3d05b5f..e7ea26cbe 100644 --- a/app/src/components/channels/channel-chat.tsx +++ b/app/src/components/channels/channel-chat.tsx @@ -8,6 +8,7 @@ import { } from "@copilotkit/react-core/v2"; import { useMutation, useQuery } from "@tanstack/react-query"; import { useCallback, useEffect, useRef, useState } from "react"; +import { HandoffResumeNotice } from "@/components/computer/handoff-resume-notice"; import { attachmentModality } from "@/components/channels/chat-messages"; import { toAgentOptions } from "@/components/channels/composer"; import { ConversationView } from "@/components/channels/conversation-view"; @@ -950,6 +951,19 @@ export function ChannelChat({ * it — and they are independent, so neither is an `else` for the other. */ <> + { + setRunsInFlight((count) => count + 1); + try { + await copilotkit.runAgent({ agent }); + } finally { + setRunsInFlight((count) => count - 1); + } + }} + /> {voiceArchive.error && (

Voice chats couldn’t be loaded. Refresh to try again. diff --git a/app/src/components/computer/computer-view.tsx b/app/src/components/computer/computer-view.tsx index c31179b6a..f3b960f02 100644 --- a/app/src/components/computer/computer-view.tsx +++ b/app/src/components/computer/computer-view.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; import { + cancelControl, type ControlState, readControl, releaseControl, @@ -224,6 +225,10 @@ export function ComputerView({ const [shot, setShot] = useState(null); const [problem, setProblem] = useState(null); const [expanded, setExpanded] = useState(false); + const [streamProblem, setStreamProblem] = useState(null); + const [retryKey, setRetryKey] = useState(0); + const [controlProblem, setControlProblem] = useState(null); + const [changingControl, setChangingControl] = useState(false); const [control, setControl] = useState(null); /** Held only until it is sent. Never lifted into a URL, a log, or anything that outlives this form. */ const [secret, setSecret] = useState(""); @@ -231,16 +236,51 @@ export function ComputerView({ const [sendingSecret, setSendingSecret] = useState(false); const pageVisible = usePageVisible(); const [previewRef, previewIntersecting] = useElementVisible(); - const driving = control?.holder === "human"; + const driving = control?.holder === "human" && !control.transitioning; /** Read by the polling loop without restarting it on control changes. */ const drivingRef = useRef(false); drivingRef.current = driving; - /** Release control; the Bot's waiting tool call resumes from this state change. */ - const handBack = async () => { - const state = await releaseControl(computerId); - if (state) setControl(state); + const changeControl = async (action: "take" | "release" | "cancel") => { + if (changingControl) return; + const requestId = control?.request?.id; + if (action !== "take" && !requestId) { + setControlProblem( + "The handoff request could not be identified. Retry the connection.", + ); + return; + } + setChangingControl(true); + setControlProblem(null); + try { + const state = + action === "take" + ? await takeControl( + computerId, + control?.requested ? requestId : undefined, + ) + : requestId + ? action === "release" + ? await releaseControl(computerId, requestId) + : await cancelControl(computerId, requestId) + : null; + if (!state) + setControlProblem( + "Control could not be changed. Check the connection and retry.", + ); + else { + setControl(state); + if (action === "take") setExpanded(true); + } + } catch { + setControlProblem( + "The computer could not be reached. Retry when it reconnects.", + ); + } finally { + setChangingControl(false); + } }; + const handBack = () => changeControl("release"); /** Secret prompts keep the screen live even though the human does not hold the wheel. */ const secretPending = Boolean(control?.secretWanted); const secretPendingRef = useRef(false); @@ -377,9 +417,17 @@ export function ComputerView({ let live = true; let timer: ReturnType; const tick = async () => { - const state = await readControl(computerId); + const state = await readControl(computerId).catch(() => null); if (!live) return; - if (state) setControl(state); + setControl(state); + if (state) + setControlProblem((previous) => + previous?.startsWith("Reconnecting:") ? null : previous, + ); + if (!state) + setControlProblem( + "Reconnecting: browser ownership could not be checked. Input is paused.", + ); timer = setTimeout(tick, 1000); }; void tick(); @@ -520,11 +568,8 @@ export function ComputerView({ + + ) : null} + {!settled && control?.transitioning ? ( +

+ Finishing the assistant's current action before giving you control… +

+ ) : null} + {!settled && control?.request?.status === "interrupted" ? ( +

+ The browser was interrupted. {control.request.interruption} Open the + screen and take control again to check it. +

+ ) : null} + {!settled && controlProblem ? ( +

+ {controlProblem} +

+ ) : null} {/* Secret values go directly to the page path and are never included in the conversation. Audit records that a secret was supplied, not the value. @@ -649,7 +722,8 @@ export function ComputerView({ {/* A live screen that ends reports why through `onProblem`, and this is the @@ -657,9 +731,19 @@ export function ComputerView({ landed in `problem`, which only the sibling `NothingToSee` reads, so the screen ended with the stale last frame frozen on the canvas and nothing said. */} - {problem ? ( -
- {problem} + {streamProblem ? ( +
+ {streamProblem} +
) : null}
@@ -680,6 +764,19 @@ export function ComputerView({ offering control of whatever the Bot has open now. Those sentences are about the present and this view is a record; a record does not get a steering wheel. */} + {controlProblem ? ( +

+ {controlProblem} +

+ ) : null} + {control?.transitioning ? ( +

+ Finishing the current action before giving you control… +

+ ) : null} {settled ? null : (
@@ -709,6 +806,7 @@ export function ComputerView({ {driving ? ( + ) : ( + + )} +
+ ); +} diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx index 5d5393fe0..27df8698a 100644 --- a/app/src/components/computer/live-screen.tsx +++ b/app/src/components/computer/live-screen.tsx @@ -1,4 +1,9 @@ import { useCallback, useEffect, useRef, useState } from "react"; +import { readControl } from "@/lib/computers/control"; +import { + connectScreen, + type ScreenSocket, +} from "@/lib/computers/screen-connection"; import { keyOf } from "@/lib/hotkeys/hotkeys"; import { socketUrl } from "@/lib/socket-url"; import { currentPageVisible } from "./preview-visibility"; @@ -53,6 +58,7 @@ type Props = { driving: boolean; /** Called with a human-readable reason when the stream cannot be established. */ onProblem?: (problem: string | null) => void; + retryKey?: number; }; type FrameMessage = { @@ -61,7 +67,12 @@ type FrameMessage = { height: number; }; -export function LiveScreen({ computerId, driving, onProblem }: Props) { +export function LiveScreen({ + computerId, + driving, + onProblem, + retryKey = 0, +}: Props) { const canvasRef = useRef(null); const socketRef = useRef(null); /** Keydowns handled locally whose matching keyup must not leak to the remote browser. */ @@ -73,15 +84,17 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { /** Monotonic guard so a slow older decode cannot replace a newer frame. */ const latestFrameId = useRef(0); const [connected, setConnected] = useState(false); + const inputReady = useRef(false); + const previouslyDriving = useRef(driving); + // biome-ignore lint/correctness/useExhaustiveDependencies: explicit Retry restarts this connection. useEffect(() => { - // The server's own address, so no proxy has to carry the upgrade. The scheme still follows - // the page: wss when the app is served over https. - const socket = new WebSocket( - socketUrl(`/api/computers/${encodeURIComponent(computerId)}/stream`), - ); - socketRef.current = socket; let closed = false; + setConnected(false); + inputReady.current = false; + frameSize.current = null; + latestFrame.current = null; + latestFrameId.current++; const drawFrame = async (frame: FrameMessage, frameId: number) => { /** @@ -127,12 +140,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { void drawFrame(frame, latestFrameId.current); }; - socket.onopen = () => { - setConnected(true); - onProblem?.(null); - }; - - socket.onmessage = async (event) => { + const onMessage = (event: { data: unknown }) => { let parsed: unknown; try { parsed = JSON.parse(String(event.data)); @@ -201,23 +209,95 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { void drawFrame(frame, frameId); }; + const connection = connectScreen({ + open: () => { + const socket = new WebSocket( + socketUrl(`/api/computers/${encodeURIComponent(computerId)}/stream`), + ); + socketRef.current = socket; + // Event handlers use only data, so adapt the native event surface without changing the protocol. + return { + set onopen(callback: ScreenSocket["onopen"]) { + socket.onopen = callback + ? () => { + void callback(); + } + : null; + }, + set onclose(callback: ScreenSocket["onclose"]) { + socket.onclose = callback; + }, + set onerror(callback: ScreenSocket["onerror"]) { + socket.onerror = callback; + }, + set onmessage(callback: ScreenSocket["onmessage"]) { + socket.onmessage = callback; + }, + close: () => socket.close(), + }; + }, + verifyOwnership: async () => { + const state = await readControl(computerId); + return state?.holder === "human" && !state.transitioning; + }, + onReady: (ready) => { + inputReady.current = ready; + setConnected(ready); + }, + onProblem: (problem) => onProblem?.(problem), + onDisconnect: () => { + frameSize.current = null; + latestFrame.current = null; + latestFrameId.current++; + const canvas = canvasRef.current; + if (canvas) + canvas.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height); + }, + onMessage, + }); document.addEventListener("visibilitychange", drawLatestFrame); - socket.onerror = () => onProblem?.("The live screen could not be reached."); - socket.onclose = () => setConnected(false); - return () => { closed = true; + connection.stop(); document.removeEventListener("visibilitychange", drawLatestFrame); - socket.close(); socketRef.current = null; }; - // The socket is per Bot; switching Bot must close this stream and open the next one. - }, [computerId, onProblem]); + }, [computerId, onProblem, retryKey]); + + useEffect(() => { + const wasDriving = previouslyDriving.current; + previouslyDriving.current = driving; + if ( + wasDriving || + !driving || + inputReady.current || + socketRef.current?.readyState !== WebSocket.OPEN + ) + return; + let live = true; + void readControl(computerId) + .then((state) => { + if (!live || socketRef.current?.readyState !== WebSocket.OPEN) return; + inputReady.current = state?.holder === "human" && !state.transitioning; + setConnected(inputReady.current); + }) + .catch(() => { + inputReady.current = false; + }); + return () => { + live = false; + }; + }, [computerId, driving]); const send = useCallback( (message: Record) => { const socket = socketRef.current; - if (!driving || socket?.readyState !== WebSocket.OPEN) return; + if ( + !driving || + !inputReady.current || + socket?.readyState !== WebSocket.OPEN + ) + return; socket.send(JSON.stringify(message)); }, [driving], @@ -288,7 +368,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { * page. */ useEffect(() => { - if (!driving) return; + if (!driving || !connected) return; const onKeyDown = (event: KeyboardEvent) => { if (event.key === "Escape") return; // Escape still closes the view. if (isPasteShortcut(event)) { @@ -341,7 +421,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { window.removeEventListener("paste", onPaste); localKeyUps.current.clear(); }; - }, [driving, send]); + }, [driving, connected, send]); /** * The wheel, forwarded while driving, from a listener that is allowed to stop it here. @@ -353,7 +433,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { */ useEffect(() => { const canvas = canvasRef.current; - if (!driving || !canvas) return; + if (!driving || !connected || !canvas) return; const onWheel = (event: WheelEvent) => { const point = at(event); if (!point) return; @@ -368,7 +448,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { }; canvas.addEventListener("wheel", onWheel, { passive: false }); return () => canvas.removeEventListener("wheel", onWheel); - }, [driving, at, send]); + }, [driving, connected, at, send]); return ( null); if (!live) return; setNeeded( - Boolean(state && (state.requested || state.secretWanted !== undefined)), + Boolean( + state && + (state.requested || + state.holder === "human" || + state.request?.status === "interrupted" || + state.secretWanted !== undefined), + ), ); }; diff --git a/app/src/lib/computers/call.ts b/app/src/lib/computers/call.ts new file mode 100644 index 000000000..d28fc77ea --- /dev/null +++ b/app/src/lib/computers/call.ts @@ -0,0 +1,72 @@ +import { tryClient } from "@/lib/client"; +import { reportComputerActivity } from "@/lib/copilot/computer-activity"; + +export type ToolOutcome = Record & { ok: boolean }; + +/** + * Exported for the test that covers what a Bot is told when a call is refused. + * + * The distinctions this draws from a status and a body decide the model's next step, and they are + * drawn nowhere else, so they are worth pinning without standing up the tool registrations and the + * runtime around them. + */ +export async function callComputer( + botId: string, + path: string, + /* + * A body, not a `RequestInit`. The client serialises it, so a caller that stringified first would + * send a JSON string of a JSON string — which is what happened, briefly, when this moved over. + */ + init?: { method?: string; body?: unknown }, + signal?: AbortSignal, +): Promise { + // Announce before the call so the screen can open while the action is running. + reportComputerActivity(botId); + let response: Response; + try { + response = await tryClient(`/api/computers/${botId}${path}`, { + method: init?.method, + body: init?.body, + // Abort cancels the request and prevents later actions, but cannot undo browser work already executing. + signal, + }); + } catch (error) { + // An abort is a stopped run, not a computer failure. + if (error instanceof DOMException && error.name === "AbortError") { + return { ok: false, reason: "Stopped.", stopped: true }; + } + return { + ok: false, + reason: "The assistant's computer could not be reached.", + }; + } + + const body = (await response.json().catch(() => null)) as Record< + string, + unknown + > | null; + + if (!response.ok) { + return { + ok: false, + reason: (body?.error as string) ?? "That did not work.", + // Preserve refusal/stale-ref/control distinctions for the model's next step. + ...(response.status === 403 + ? { refused: true, rule: body?.rule ?? null } + : {}), + ...(response.status === 409 + ? body?.humanHasControl === true + ? { + humanHasControl: true, + requestId: body?.requestId, + handoff: body?.handoff, + } + : body?.snapshotRequired === true + ? { staleRefs: true, snapshotRequired: true } + : { staleRefs: true } + : {}), + }; + } + + return { ok: true, ...(body ?? {}) }; +} diff --git a/app/src/lib/computers/control.ts b/app/src/lib/computers/control.ts index 686c2547d..18f4d4cd3 100644 --- a/app/src/lib/computers/control.ts +++ b/app/src/lib/computers/control.ts @@ -11,44 +11,58 @@ import { tryClient } from "@/lib/client"; * should say nothing, not tear down the screen the person is looking at. */ -export type ControlState = { - holder: "bot" | "human"; - since: string; - reason?: string; - requested: boolean; - /** What the Bot is waiting for, by name only. Present means show the masked prompt. */ - secretWanted?: string; -}; +export type { ComputerControlState as ControlState } from "../../../../shared/computer-control"; +import type { ComputerControlState as ControlState } from "../../../../shared/computer-control"; async function callControl( computerId: string, path: string, - method?: string, + body?: unknown, + signal?: AbortSignal, ): Promise { const response = await tryClient( - `/api/computers/${computerId}${path}`, - method ? { method } : {}, + `/api/computers/${encodeURIComponent(computerId)}${path}`, + { ...(body === undefined ? {} : { method: "POST", body }), signal }, ); if (!response.ok) return null; - // A non-JSON or wrong-shaped body used to throw out of the readers and reject the panel's - // poll. Reads answer null on failure, so malformed succeeds as missing. - const body = (await response.json().catch(() => null)) as unknown; - if (!body || typeof body !== "object" || Array.isArray(body)) return null; - const holder = (body as { holder?: unknown }).holder; - if (holder !== "bot" && holder !== "human") return null; - return body as ControlState; + const value: unknown = await response.json().catch(() => null); + if (!value || typeof value !== "object" || !("holder" in value)) return null; + if (value.holder !== "bot" && value.holder !== "human") return null; + return value as ControlState; } -export function readControl(computerId: string) { - return callControl(computerId, "/control"); +export function readControl( + computerId: string, + requestId?: string, + signal?: AbortSignal, +) { + return callControl( + computerId, + `/control${requestId ? `?requestId=${encodeURIComponent(requestId)}` : ""}`, + undefined, + signal, + ); +} + +export async function takeControl(computerId: string, requestId?: string) { + const id = + requestId ?? + ( + await callControl(computerId, "/control/request", { + reason: "I want to take control of the browser.", + }) + )?.request?.id; + return id + ? callControl(computerId, "/control/take", { requestId: id }) + : null; } -export function takeControl(computerId: string) { - return callControl(computerId, "/control/take", "POST"); +export function releaseControl(computerId: string, requestId: string) { + return callControl(computerId, "/control/release", { requestId }); } -export function releaseControl(computerId: string) { - return callControl(computerId, "/control/release", "POST"); +export function cancelControl(computerId: string, requestId: string) { + return callControl(computerId, "/control/cancel", { requestId }); } /** diff --git a/app/src/lib/computers/handoff.ts b/app/src/lib/computers/handoff.ts new file mode 100644 index 000000000..2de57684f --- /dev/null +++ b/app/src/lib/computers/handoff.ts @@ -0,0 +1,288 @@ +import type { + BrowserChallenge, + HandoffRequest, +} from "../../../../shared/computer-control"; +import { callComputer, type ToolOutcome } from "./call"; +import { cancelControl, readControl } from "./control"; + +export type PendingHandoff = { requestId: string; toolCallId: string }; +const active = new Map(); +const key = (botId: string) => `openbot:pending-handoff:${botId}`; + +/** Bookkeeping only. Never persist page text, credentials, or the human's input. */ +export function pendingHandoff(botId: string): PendingHandoff | null { + if (typeof sessionStorage === "undefined") return null; + const text = sessionStorage.getItem(key(botId)); + if (!text) return null; + try { + const value: unknown = JSON.parse(text); + return value && + typeof value === "object" && + "requestId" in value && + typeof value.requestId === "string" && + "toolCallId" in value && + typeof value.toolCallId === "string" + ? { requestId: value.requestId, toolCallId: value.toolCallId } + : null; + } catch { + return null; + } +} +export function rememberHandoff( + botId: string, + requestId: string, + toolCallId?: string, +) { + if (toolCallId && typeof sessionStorage !== "undefined") + sessionStorage.setItem( + key(botId), + JSON.stringify({ requestId, toolCallId }), + ); +} +export function forgetHandoff(botId: string, requestId: string) { + if (pendingHandoff(botId)?.requestId === requestId) + sessionStorage.removeItem(key(botId)); +} +export function hasActiveHandoff(botId: string, requestId: string) { + return (active.get(`${botId}:${requestId}`) ?? 0) > 0; +} + +function pause(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) { + resolve(); + return; + } + const finish = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", finish); + resolve(); + }; + const timer = setTimeout(finish, ms); + signal?.addEventListener("abort", finish, { once: true }); + }); +} + +/** No human-control deadline: waiting expiry and all terminal outcomes belong to the server. */ +export async function awaitHandoff( + botId: string, + requestId: string, + signal?: AbortSignal, + pollMs = 1000, +): Promise { + const identity = `${botId}:${requestId}`; + active.set(identity, (active.get(identity) ?? 0) + 1); + let lastRequest: HandoffRequest | undefined; + let failures = 0; + try { + while (!signal?.aborted) { + const state = await readControl(botId, requestId, signal).catch( + () => null, + ); + if (signal?.aborted) break; + if (!state) { + if (++failures >= 5) + return { + ok: false, + requestId, + reconnecting: true, + reason: + "The handoff could not be reached. Reopen this conversation and use Resume task to check this request again.", + }; + await pause(pollMs, signal); + continue; + } + failures = 0; + if (state.request?.id !== requestId) + return { + ok: false, + requestId, + reason: + "The computer did not return the requested handoff. Nothing has been marked complete.", + }; + lastRequest = state.request; + if (lastRequest.status === "completed") { + const snapshot = await callComputer( + botId, + "/snapshot", + { method: "POST" }, + signal, + ); + if (!snapshot.ok) + return { + ...snapshot, + requestId, + handoffStatus: "completed", + snapshotRequired: true, + }; + const nextChallenge = challengeFrom(snapshot); + if (nextChallenge) { + if (nextChallenge.requestId === requestId) + return { + ok: false, + requestId, + reason: + "The completed request still reports an active challenge. Check the browser before continuing.", + }; + rememberHandoff( + botId, + nextChallenge.requestId, + pendingHandoff(botId)?.toolCallId, + ); + const next = await awaitHandoff( + botId, + nextChallenge.requestId, + signal, + pollMs, + ); + return { ...next, challenge: nextChallenge }; + } + return { + ...snapshot, + requestId, + handoffStatus: "completed", + result: + "The person handed control back. This fresh snapshot describes the page now.", + }; + } + if (lastRequest.status !== "waiting" && lastRequest.status !== "taken") { + return { + ok: false, + requestId, + handoffStatus: lastRequest.status, + reason: + lastRequest.interruption ?? + `The handoff was ${lastRequest.status}. The person has not completed this request.`, + }; + } + await pause(pollMs, signal); + } + // Stop detaches a driver. Only a still-waiting request may be cancelled. + if (signal?.reason === "detached") + return { + ok: false, + stopped: true, + requestId, + reason: "The waiting view disconnected.", + }; + const current = await readControl( + botId, + requestId, + AbortSignal.timeout(2000), + ).catch(() => null); + if ( + current?.request?.id === requestId && + current.request.status === "waiting" + ) + await cancelControl(botId, requestId).catch(() => null); + return { + ok: false, + stopped: true, + requestId, + reason: + "Stopped waiting. A person who has control keeps it until they hand back.", + }; + } finally { + const remaining = (active.get(identity) ?? 1) - 1; + if (remaining) active.set(identity, remaining); + else active.delete(identity); + } +} + +function requestFrom(result: ToolOutcome): HandoffRequest | undefined { + const request = result.request; + return request && + typeof request === "object" && + "id" in request && + typeof request.id === "string" + ? (request as HandoffRequest) + : undefined; +} + +export async function runHelpRequest( + botId: string, + reason: string, + toolCallId?: string, + signal?: AbortSignal, +): Promise { + const stored = pendingHandoff(botId); + if (toolCallId && stored?.toolCallId === toolCallId) + return awaitHandoff(botId, stored.requestId, signal); + const asked = await callComputer( + botId, + "/control/request", + { method: "POST", body: { reason, ...(toolCallId ? { toolCallId } : {}) } }, + signal, + ); + if (!asked.ok) return asked; + const request = requestFrom(asked); + if (!request) + return { + ok: false, + reason: "The computer did not identify the handoff request.", + }; + rememberHandoff(botId, request.id, toolCallId); + return awaitHandoff(botId, request.id, signal); +} + +export async function runNavigation( + botId: string, + url: string, + toolCallId?: string, + signal?: AbortSignal, +): Promise { + const stored = pendingHandoff(botId); + if (toolCallId && stored?.toolCallId === toolCallId) + return awaitHandoff(botId, stored.requestId, signal); + const result = await callComputer( + botId, + "/navigate", + { method: "POST", body: { url, ...(toolCallId ? { toolCallId } : {}) } }, + signal, + ); + const challenge = challengeFrom(result); + if (!challenge?.requestId) return result; + rememberHandoff(botId, challenge.requestId, toolCallId); + const resumed = await awaitHandoff(botId, challenge.requestId, signal); + // Only the resumed snapshot describes the current page. The initial response belongs to + // the challenge page and may carry text/truncation fields absent from a snapshot. + return { ...resumed, challenge, challengeResolved: resumed.ok }; +} + +function challengeFrom(result: ToolOutcome): BrowserChallenge | undefined { + const value = result.challenge; + if ( + !value || + typeof value !== "object" || + !("requestId" in value) || + typeof value.requestId !== "string" || + !("kind" in value) || + (value.kind !== "cloudflare" && value.kind !== "visible-challenge") || + !("reason" in value) || + typeof value.reason !== "string" + ) + return undefined; + return { requestId: value.requestId, kind: value.kind, reason: value.reason }; +} + +export async function runBrowserRead( + botId: string, + path: "/read" | "/snapshot", + toolCallId?: string, + signal?: AbortSignal, +): Promise { + const stored = pendingHandoff(botId); + if (toolCallId && stored?.toolCallId === toolCallId) + return awaitHandoff(botId, stored.requestId, signal); + const result = await callComputer( + botId, + path, + path === "/snapshot" ? { method: "POST" } : undefined, + signal, + ); + const challenge = challengeFrom(result); + if (!challenge) return result; + rememberHandoff(botId, challenge.requestId, toolCallId); + const resumed = await awaitHandoff(botId, challenge.requestId, signal); + return { ...resumed, challenge, challengeResolved: resumed.ok }; +} diff --git a/app/src/lib/computers/screen-connection.ts b/app/src/lib/computers/screen-connection.ts new file mode 100644 index 000000000..c0edef069 --- /dev/null +++ b/app/src/lib/computers/screen-connection.ts @@ -0,0 +1,103 @@ +/** Small transport lifecycle shared by the live viewer and deterministic reconnect tests. */ +export type ScreenSocket = { + onopen: (() => void | Promise) | null; + onclose: (() => void) | null; + onerror: (() => void) | null; + onmessage: ((event: { data: unknown }) => void) | null; + close: () => void; +}; +type Options = { + open: () => ScreenSocket; + verifyOwnership: () => Promise; + onReady: (ready: boolean) => void; + onProblem: (problem: string | null) => void; + onDisconnect: () => void; + onMessage: (event: { data: unknown }) => void; + schedule?: (run: () => void, delay: number) => () => void; +}; +const BACKOFF = [500, 1000, 2000, 4000, 8000]; +export function connectScreen(options: Options) { + let stopped = false; + let attempts = 0; + let socket: ScreenSocket | undefined; + let cancelTimer: (() => void) | undefined; + const schedule = + options.schedule ?? + ((run, delay) => { + const timer = setTimeout(run, delay); + return () => clearTimeout(timer); + }); + const connect = () => { + if (stopped) return; + const current = options.open(); + socket = current; + let disconnected = false; + const owns = () => !stopped && socket === current && !disconnected; + const disconnect = () => { + if (!owns()) return; + disconnected = true; + options.onDisconnect(); + options.onReady(false); + const delay = BACKOFF[attempts++]; + if (delay === undefined) { + options.onProblem( + "The live screen is disconnected. Retry to reconnect.", + ); + return; + } + options.onProblem("Reconnecting to the live screen…"); + cancelTimer = schedule(connect, delay); + }; + current.onopen = async () => { + // The initial mount receives freshly polled ownership from its parent. Reconnect must refresh it. + const permitted = + attempts === 0 || (await options.verifyOwnership().catch(() => false)); + if (!owns()) return; + options.onProblem(null); + options.onReady(permitted); + }; + current.onmessage = (event) => { + if (!owns()) return; + let message: { type?: string; error?: string } | undefined; + try { + message = JSON.parse(String(event.data)); + } catch { + /* Frame parser handles malformed input. */ + } + if (message?.type === "error") { + // The server deliberately leaves superseded sockets open. Never compete with the other tab. + if ( + /superseded|another (?:viewer|screen)|opened elsewhere|watched somewhere else|another tab/i.test( + message.error ?? "", + ) + ) { + stopped = true; + cancelTimer?.(); + options.onDisconnect(); + options.onReady(false); + options.onProblem( + message.error ?? + "This screen is open in another tab. Retry to take it here.", + ); + current.close(); + return; + } + } + options.onMessage(event); + }; + current.onerror = () => { + disconnect(); + current.close(); + }; + current.onclose = disconnect; + }; + connect(); + return { + stop: () => { + stopped = true; + cancelTimer?.(); + options.onReady(false); + socket?.close(); + }, + }; +} diff --git a/app/src/lib/copilot/computer-tools.tsx b/app/src/lib/copilot/computer-tools.tsx index 20d9f75da..543118d85 100644 --- a/app/src/lib/copilot/computer-tools.tsx +++ b/app/src/lib/copilot/computer-tools.tsx @@ -7,29 +7,35 @@ import { tryClient } from "@/lib/client"; import { noteBrowsed, recordActivity } from "@/lib/computers/activity"; import { type ControlState, readControl } from "@/lib/computers/control"; import { useActiveBotHolder } from "./active-bot"; -import { reportComputerActivity } from "./computer-activity"; +import { callComputer, type ToolOutcome } from "@/lib/computers/call"; +import { + runBrowserRead, + runHelpRequest, + runNavigation, +} from "@/lib/computers/handoff"; /** * Frontend registrations for computer tools, including inline rendering and policy-refusal display. */ /** What every computer call returns to the model: either the result, or a reason it did not happen. */ -export type ToolOutcome = Record & { ok: boolean }; +export { callComputer } from "@/lib/computers/call"; +export type { ToolOutcome } from "@/lib/computers/call"; /** - * Human-assistance wait window. Long enough for a user to return, finite so the run can unblock. + * Secret-entry wait window. Browser takeovers use server-authoritative request state instead. */ -const WAIT_FOR_PERSON_MS = 10 * 60_000; +const WAIT_FOR_SECRET_MS = 10 * 60_000; /** How often the waiting handler asks whether the person has answered yet. */ const WAIT_POLL_MS = 1_000; -/** Hold the tool call open until the human control/secret prompt is answered, cancelled, or expires. */ -async function waitForPerson( +/** Hold the secret-entry call open until its masked prompt is answered, cancelled, or expires. */ +async function waitForSecret( botId: string, done: (state: ControlState) => boolean, signal: AbortSignal | undefined, - giveUpAfterMs = WAIT_FOR_PERSON_MS, + giveUpAfterMs = WAIT_FOR_SECRET_MS, ): Promise<"answered" | "gave up" | "cancelled"> { const deadline = Date.now() + giveUpAfterMs; while (Date.now() < deadline) { @@ -42,68 +48,6 @@ async function waitForPerson( return "gave up"; } -/** - * Exported for the test that covers what a Bot is told when a call is refused. - * - * The distinctions this draws from a status and a body decide the model's next step, and they are - * drawn nowhere else, so they are worth pinning without standing up the tool registrations and the - * runtime around them. - */ -export async function callComputer( - botId: string, - path: string, - /* - * A body, not a `RequestInit`. The client serialises it, so a caller that stringified first would - * send a JSON string of a JSON string — which is what happened, briefly, when this moved over. - */ - init?: { method?: string; body?: unknown }, - signal?: AbortSignal, -): Promise { - // Announce before the call so the screen can open while the action is running. - reportComputerActivity(botId); - let response: Response; - try { - response = await tryClient(`/api/computers/${botId}${path}`, { - method: init?.method, - body: init?.body, - // Abort cancels the request and prevents later actions, but cannot undo browser work already executing. - signal, - }); - } catch (error) { - // An abort is a stopped run, not a computer failure. - if (error instanceof DOMException && error.name === "AbortError") { - return { ok: false, reason: "Stopped.", stopped: true }; - } - return { - ok: false, - reason: "The assistant's computer could not be reached.", - }; - } - - const body = (await response.json().catch(() => null)) as Record< - string, - unknown - > | null; - - if (!response.ok) { - return { - ok: false, - reason: (body?.error as string) ?? "That did not work.", - // Preserve refusal/stale-ref/control distinctions for the model's next step. - ...(response.status === 403 - ? { refused: true, rule: body?.rule ?? null } - : {}), - ...(response.status === 409 - ? body?.humanHasControl === true - ? { humanHasControl: true } - : { staleRefs: true } - : {}), - }; - } - - return { ok: true, ...(body ?? {}) }; -} - /** What a computer tool's render can read back out of its own result. */ type ComputerOutcome = { ok?: boolean; @@ -258,40 +202,9 @@ export function ComputerTools() { }: { signal?: AbortSignal; toolCall?: { id?: string } } = {}, ) => { const computerId = bot.current; - const result = await callComputer( - computerId, - "/navigate", - { - method: "POST", - /* - * Which turn is asking, so the server can file the picture under it. - * - * The handler's context carries the tool call, which is worth saying because assuming it - * did not is how the frame ended up keyed on the page instead: two visits to one address - * then collided, and resolving that by letting the newer win made a past turn's picture - * change under the person reading it. - */ - body: { url, ...(toolCall?.id ? { toolCallId: toolCall.id } : {}) }, - }, - signal, - ); - /* - * This Bot has a page of its own now, so the pane may default to the screen. - * - * Until it does, the screen shows whatever the shared computer had open last, which may be - * another Bot's page from an hour ago. Captioning that as this Bot's screen is confidently - * wrong, and worse than showing nothing. - */ + const result = await runNavigation(computerId, url, toolCall?.id, signal); if (result.ok) noteBrowsed(computerId); - return result.ok - ? { - ok: true, - title: result.title, - url: result.url, - text: result.text, - truncated: result.truncated, - } - : result; + return result; }, render: ({ result, status, toolCallId }) => { /* @@ -345,7 +258,13 @@ export function ComputerTools() { "Read the page currently open on your computer, without opening anything. Use this after you " + "click something that changes the page, such as submitting a form, to find out what it now says.", parameters: z.object({}), - handler: async () => callComputer(bot.current, "/read"), + handler: async ( + _input: Record, + { + signal, + toolCall, + }: { signal?: AbortSignal; toolCall?: { id?: string } } = {}, + ) => runBrowserRead(bot.current, "/read", toolCall?.id, signal), render: () => null, }); @@ -357,8 +276,13 @@ export function ComputerTools() { "use the refs it returns. Always send back the snapshotId it gives you. If an action reports " + "that your refs are stale, the page changed: call this again and use the new refs.", parameters: z.object({}), - handler: async () => - callComputer(bot.current, "/snapshot", { method: "POST" }), + handler: async ( + _input: Record, + { + signal, + toolCall, + }: { signal?: AbortSignal; toolCall?: { id?: string } } = {}, + ) => runBrowserRead(bot.current, "/snapshot", toolCall?.id, signal), // Snapshot renders a count only; navigate owns the screen view. render: ({ result, status }) => { const outcome = outcomeOf(result); @@ -553,7 +477,7 @@ export function ComputerTools() { if (!asked.ok) return asked; // Completion is `secretWanted` clearing; the value never returns to the model. - const outcome = await waitForPerson( + const outcome = await waitForSecret( botId, (state) => state.secretWanted === undefined, signal, @@ -629,36 +553,11 @@ export function ComputerTools() { }), handler: async ( input: { reason: string }, - { signal }: { signal?: AbortSignal } = {}, - ) => { - const botId = bot.current; - const asked = await callComputer( - botId, - "/control/request", - { - method: "POST", - body: input, - }, - signal, - ); - if (!asked.ok) return asked; - - // Resolved when the wheel is back with the Bot and no help request remains outstanding. - const outcome = await waitForPerson( - botId, - (state) => state.holder === "bot" && !state.requested, + { signal, - ); - return { - ok: true, - result: - outcome === "answered" - ? "The person has finished and handed control back. Take a fresh snapshot: the page may have changed while they were driving." - : outcome === "cancelled" - ? "The request was cancelled." - : "Nobody took control. Say what you still need rather than trying to do it yourself.", - }; - }, + toolCall, + }: { signal?: AbortSignal; toolCall?: { id?: string } } = {}, + ) => runHelpRequest(bot.current, input.reason, toolCall?.id, signal), // Rendered by ComputerView as the take-the-wheel prompt. render: () => null, }); diff --git a/app/src/lib/copilot/handoff-resume.ts b/app/src/lib/copilot/handoff-resume.ts new file mode 100644 index 000000000..d146a24c4 --- /dev/null +++ b/app/src/lib/copilot/handoff-resume.ts @@ -0,0 +1,116 @@ +import type { Message, ToolCall } from "@ag-ui/core"; +import { + awaitHandoff, + forgetHandoff, + hasActiveHandoff, + type PendingHandoff, +} from "@/lib/computers/handoff"; +import { newId } from "@/lib/new-id"; +import { repairUnansweredToolCalls } from "./repair-history"; + +export type HandoffAgent = { + messages: Message[]; + isRunning: boolean; + setMessages: (messages: Message[]) => void; +}; + +/** These are transport placeholders / retryable transport failures, never accepted results. */ +function needsAnswer(message: Message): boolean { + if (message.role !== "tool") return false; + if (message.content.trim() === "Forwarded to client") return true; + try { + const result: unknown = JSON.parse(message.content); + if (result === "Forwarded to client") return true; + return Boolean( + result && + typeof result === "object" && + "ok" in result && + result.ok === false && + (("reconnecting" in result && result.reconnecting === true) || + ("snapshotRequired" in result && result.snapshotRequired === true)), + ); + } catch { + return false; + } +} + +export function findPendingTool( + messages: Message[], + pending: PendingHandoff, +): ToolCall | null { + const call = messages + .flatMap((message) => + message.role === "assistant" ? (message.toolCalls ?? []) : [], + ) + .find( + (tool) => + tool.id === pending.toolCallId && + [ + "computer_request_help", + "computer_navigate", + "computer_read", + "computer_snapshot", + ].includes(tool.function.name), + ); + if (!call) return null; + const accepted = messages.some( + (message) => + message.role === "tool" && + message.toolCallId === call.id && + !needsAnswer(message), + ); + return accepted ? null : call; +} + +/** Resume the restored call on its original channel agent; never add a new user task. */ +export async function resumeHandoffTask( + botId: string, + pending: PendingHandoff, + agent: HandoffAgent, + run: () => Promise, + signal?: AbortSignal, +) { + const hasCall = agent.messages.some( + (m) => + m.role === "assistant" && + m.toolCalls?.some((c) => c.id === pending.toolCallId), + ); + if (!hasCall) + throw new Error("The handoff's tool call is not in this conversation."); + if (!findPendingTool(agent.messages, pending)) + throw new Error("This tool call is already answered."); + if (agent.isRunning || hasActiveHandoff(botId, pending.requestId)) + throw new Error("This task is already running."); + const result = await awaitHandoff(botId, pending.requestId, signal); + if (signal?.aborted) return; + if (result.reconnecting || result.snapshotRequired) + throw new Error( + String( + result.reason ?? "The computer could not be reached. Retry to resume.", + ), + ); + // Another handler/tab may have supplied the result while we waited. Keep accepted work accepted. + if (!findPendingTool(agent.messages, pending)) return; + const messages = agent.messages.filter( + (m) => + !( + m.role === "tool" && + m.toolCallId === pending.toolCallId && + needsAnswer(m) + ), + ); + const caller = messages.findIndex( + (m) => + m.role === "assistant" && + m.toolCalls?.some((c) => c.id === pending.toolCallId), + ); + messages.splice(caller + 1, 0, { + id: newId(), + role: "tool", + toolCallId: pending.toolCallId, + content: JSON.stringify(result), + }); + agent.setMessages([...repairUnansweredToolCalls(messages)]); + forgetHandoff(botId, pending.requestId); + await run(); +} diff --git a/app/tests/computer-handoff-rejoin.test.ts b/app/tests/computer-handoff-rejoin.test.ts new file mode 100644 index 000000000..a82ee47d2 --- /dev/null +++ b/app/tests/computer-handoff-rejoin.test.ts @@ -0,0 +1,199 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { + awaitHandoff, + pendingHandoff, + rememberHandoff, + runBrowserRead, + runHelpRequest, + runNavigation, +} from "../src/lib/computers/handoff"; +import { callComputer } from "../src/lib/computers/call"; + +const realFetch = globalThis.fetch; +const paths: string[] = []; +beforeAll(() => GlobalRegistrator.register()); +afterEach(() => { + globalThis.fetch = realFetch; + sessionStorage.clear(); + paths.length = 0; +}); +afterAll(() => GlobalRegistrator.unregister()); +function serve(answer: (path: string) => Response) { + globalThis.fetch = Object.assign( + async (url: Parameters[0]) => { + const path = String(url); + paths.push(path); + return answer(path); + }, + { preconnect: () => undefined }, + ); +} +function state(id: string, status: string) { + return Response.json({ + holder: status === "taken" ? "human" : "bot", + requested: status === "waiting", + request: { id, status }, + }); +} + +test("replayed navigation after reload rejoins stored identity without navigating again", async () => { + rememberHandoff("bot-1", "request-1", "tool-1"); + serve((path) => + path.endsWith("/snapshot") + ? Response.json({ snapshotId: 8 }) + : state("request-1", "completed"), + ); + expect( + await runNavigation("bot-1", "https://example.com", "tool-1"), + ).toMatchObject({ ok: true, snapshotId: 8 }); + expect(paths.some((path) => path.endsWith("/navigate"))).toBe(false); + expect(pendingHandoff("bot-1")).toEqual({ + requestId: "request-1", + toolCallId: "tool-1", + }); +}); + +test("replayed help call does not create another request", async () => { + rememberHandoff("bot-1", "request-1", "tool-1"); + serve(() => state("request-1", "expired")); + expect(await runHelpRequest("bot-1", "Sign in", "tool-1")).toMatchObject({ + ok: false, + handoffStatus: "expired", + }); + expect(paths).toEqual(["/api/computers/bot-1/control?requestId=request-1"]); +}); + +test("a challenge still present in the fresh snapshot joins its new request before resuming", async () => { + rememberHandoff("bot-1", "request-1", "tool-1"); + let snapshots = 0; + serve((path) => { + if (path.endsWith("/snapshot")) + return Response.json( + ++snapshots === 1 + ? { + snapshotId: 8, + challenge: { + kind: "visible-challenge", + reason: "Please finish verification", + requestId: "request-2", + }, + } + : { snapshotId: 9 }, + ); + return state( + path.endsWith("request-2") ? "request-2" : "request-1", + "completed", + ); + }); + expect(await awaitHandoff("bot-1", "request-1")).toMatchObject({ + ok: true, + requestId: "request-2", + snapshotId: 9, + }); + expect(pendingHandoff("bot-1")?.requestId).toBe("request-2"); + expect(paths).toContain("/api/computers/bot-1/control?requestId=request-2"); +}); + +test("Stop cancels a still-waiting request by ID", async () => { + const controller = new AbortController(); + controller.abort(); + serve((path) => + state("request-1", path.endsWith("/cancel") ? "cancelled" : "waiting"), + ); + expect( + await awaitHandoff("bot-1", "request-1", controller.signal), + ).toMatchObject({ stopped: true, ok: false }); + expect(paths).toEqual([ + "/api/computers/bot-1/control?requestId=request-1", + "/api/computers/bot-1/control/cancel", + ]); +}); + +test("an abort wakes a long polling delay without waiting for that delay", async () => { + const controller = new AbortController(); + serve(() => state("request-1", "taken")); + const result = awaitHandoff("bot-1", "request-1", controller.signal, 30_000); + setTimeout(() => controller.abort(), 10); + expect(await result).toMatchObject({ stopped: true, ok: false }); + expect(paths.some((path) => path.endsWith("/cancel"))).toBe(false); +}, 250); + +test("snapshot-required and request identity survive tool refusal shaping", async () => { + serve(() => + Response.json( + { error: "Take a fresh snapshot", snapshotRequired: true }, + { status: 409 }, + ), + ); + expect(await callComputer("bot-1", "/click")).toMatchObject({ + ok: false, + snapshotRequired: true, + }); + serve(() => + Response.json( + { + error: "Help is pending", + humanHasControl: true, + requestId: "request-1", + handoff: { id: "request-1", status: "waiting" }, + }, + { status: 409 }, + ), + ); + expect(await callComputer("bot-1", "/click")).toMatchObject({ + ok: false, + requestId: "request-1", + handoff: { id: "request-1" }, + }); +}); + +test.each(["navigate", "read"] as const)( + "%s resumes with only current page data after the human changes the page", + async (operation) => { + const challenge = { + kind: "cloudflare", + reason: "Verify you are human", + requestId: "request-1", + }; + serve((path) => { + if (path.endsWith(`/${operation}`)) + return Response.json({ + url: "https://site.test/challenge", + title: "Verification required", + text: "Verify you are human BEFORE clearance", + truncated: true, + challenge, + }); + if (path.endsWith("/snapshot")) + return Response.json({ + url: "https://site.test/account", + title: "Account", + snapshotId: 8, + elements: [ + { ref: "e1", role: "heading", name: "Welcome AFTER clearance" }, + ], + }); + return state("request-1", "completed"); + }); + const result = + operation === "navigate" + ? await runNavigation("bot-1", "https://site.test/account", "tool-1") + : await runBrowserRead("bot-1", "/read", "tool-1"); + expect(result).toMatchObject({ + ok: true, + url: "https://site.test/account", + title: "Account", + snapshotId: 8, + elements: [ + { ref: "e1", role: "heading", name: "Welcome AFTER clearance" }, + ], + handoffStatus: "completed", + challenge, + challengeResolved: true, + }); + expect(result.text).toBeUndefined(); + expect(result.truncated).toBeUndefined(); + expect(JSON.stringify(result)).not.toContain("BEFORE clearance"); + }, +); diff --git a/app/tests/computer-handoff.test.ts b/app/tests/computer-handoff.test.ts new file mode 100644 index 000000000..61682b7a0 --- /dev/null +++ b/app/tests/computer-handoff.test.ts @@ -0,0 +1,179 @@ +import { + afterEach, + beforeEach, + expect, + type Mock, + spyOn, + test, +} from "bun:test"; +import type { + ComputerControlState, + HandoffStatus, +} from "../../shared/computer-control"; +import { + awaitHandoff, + runHelpRequest, + runNavigation, +} from "../src/lib/computers/handoff"; +import { releaseControl, takeControl } from "../src/lib/computers/control"; + +let fetchSpy: Mock; +beforeEach(() => { + fetchSpy = spyOn(globalThis, "fetch"); +}); +const requests: { path: string; body: unknown }[] = []; +function state(status: HandoffStatus, id = "request-1"): ComputerControlState { + return { + holder: status === "taken" ? "human" : "bot", + since: "2026-09-26T00:00:00Z", + requested: status === "waiting", + transitioning: false, + resumeSnapshotRequired: status === "completed", + request: { + id, + status, + reason: "Please sign in", + source: "model", + createdAt: "2026-09-26T00:00:00Z", + updatedAt: "2026-09-26T00:00:00Z", + }, + }; +} +function serve(answer: (path: string, body: unknown) => unknown) { + fetchSpy.mockImplementation( + Object.assign( + async ( + url: Parameters[0], + options?: Parameters[1], + ) => { + const path = String(url); + const body: unknown = options?.body + ? JSON.parse(String(options.body)) + : undefined; + requests.push({ path, body }); + return Response.json(answer(path, body)); + }, + { preconnect: () => undefined }, + ), + ); +} +afterEach(() => { + requests.length = 0; + fetchSpy.mockRestore(); +}); + +test("exact request completion takes a fresh snapshot before returning", async () => { + serve((path) => + path.endsWith("/snapshot") + ? { snapshotId: 42, elements: [{ ref: "fresh" }] } + : state("completed"), + ); + const result = await awaitHandoff("bot-1", "request-1"); + expect(result).toMatchObject({ + ok: true, + handoffStatus: "completed", + snapshotId: 42, + }); + expect(requests.map((r) => r.path)).toEqual([ + "/api/computers/bot-1/control?requestId=request-1", + "/api/computers/bot-1/snapshot", + ]); +}); + +test.each(["expired", "cancelled", "interrupted"] as const)( + "%s never means a person finished", + async (status) => { + serve(() => state(status)); + expect(await awaitHandoff("bot-1", "request-1")).toMatchObject({ + ok: false, + handoffStatus: status, + }); + expect(requests).toHaveLength(1); + }, +); + +test("taken remains pending even past a ten-minute wall-clock jump", async () => { + const clock = spyOn(Date, "now"); + let polls = 0; + serve((path) => { + if (path.endsWith("/snapshot")) return { snapshotId: 2 }; + polls++; + clock.mockReturnValue(polls === 1 ? 0 : 25 * 60_000); + return state(polls < 3 ? "taken" : "completed"); + }); + try { + expect( + await awaitHandoff("bot-1", "request-1", undefined, 0), + ).toMatchObject({ ok: true }); + expect(polls).toBe(3); + } finally { + clock.mockRestore(); + } +}); + +test("a mismatched response cannot complete the intended request", async () => { + serve(() => state("completed", "different-request")); + expect(await awaitHandoff("bot-1", "request-1")).toMatchObject({ ok: false }); + expect(requests.some((r) => r.path.endsWith("/snapshot"))).toBe(false); +}); + +test("Stop while taken detaches without cancelling or releasing control", async () => { + const controller = new AbortController(); + serve(() => { + controller.abort(); + return state("taken"); + }); + expect( + await awaitHandoff("bot-1", "request-1", controller.signal, 0), + ).toMatchObject({ ok: false, stopped: true }); + expect(requests.some((r) => /cancel|release/.test(r.path))).toBe(false); +}); + +test("navigation challenge joins existing request and preserves challenge metadata", async () => { + const challenge = { + kind: "cloudflare", + reason: "Verify you are human", + requestId: "request-1", + }; + serve((path) => + path.endsWith("/navigate") + ? { url: "https://example.com", challenge } + : path.endsWith("/snapshot") + ? { snapshotId: 12, url: "https://example.com/welcome" } + : state("completed"), + ); + const result = await runNavigation("bot-1", "https://example.com", "tool-1"); + expect(result).toMatchObject({ ok: true, challenge, snapshotId: 12 }); + expect(requests[0]?.body).toEqual({ + url: "https://example.com", + toolCallId: "tool-1", + }); + expect(requests.some((r) => r.path.endsWith("/control/request"))).toBe(false); +}); + +test("help uses the original tool call as its idempotency key", async () => { + serve((path) => + path.endsWith("/snapshot") ? { snapshotId: 12 } : state("completed"), + ); + expect( + await runHelpRequest("bot-1", "Please sign in", "tool-1"), + ).toMatchObject({ ok: true }); + expect(requests[0]?.body).toEqual({ + reason: "Please sign in", + toolCallId: "tool-1", + }); +}); + +test("manual takeover first creates a request and take/release name that ID", async () => { + serve(() => state("taken")); + await takeControl("bot-1"); + await releaseControl("bot-1", "request-1"); + expect(requests.map((r) => [r.path, r.body])).toEqual([ + [ + "/api/computers/bot-1/control/request", + { reason: "I want to take control of the browser." }, + ], + ["/api/computers/bot-1/control/take", { requestId: "request-1" }], + ["/api/computers/bot-1/control/release", { requestId: "request-1" }], + ]); +}); diff --git a/app/tests/handoff-resume.test.ts b/app/tests/handoff-resume.test.ts new file mode 100644 index 000000000..0652d0c6b --- /dev/null +++ b/app/tests/handoff-resume.test.ts @@ -0,0 +1,146 @@ +import { + afterEach, + beforeEach, + expect, + type Mock, + spyOn, + test, +} from "bun:test"; +import type { Message } from "@ag-ui/core"; +import { + findPendingTool, + resumeHandoffTask, +} from "../src/lib/copilot/handoff-resume"; + +let fetchSpy: Mock; +beforeEach(() => { + fetchSpy = spyOn(globalThis, "fetch"); +}); +afterEach(() => fetchSpy.mockRestore()); +const pending = { requestId: "request-1", toolCallId: "tool-1" }; +const messages: Message[] = [ + { + id: "assistant-1", + role: "assistant", + toolCalls: [ + { + id: "tool-1", + type: "function", + function: { + name: "computer_request_help", + arguments: '{"reason":"Sign in"}', + }, + }, + ], + }, +]; +function resumedAgent() { + const rows = [...messages]; + const order: string[] = []; + const agent = { + messages: rows, + isRunning: false, + setMessages: (next: Message[]) => { + agent.messages = next; + order.push("result"); + }, + }; + const run = async () => { + order.push("run"); + }; + fetchSpy.mockImplementation( + Object.assign( + async (path: Parameters[0]) => { + if (String(path).endsWith("/snapshot")) { + order.push("snapshot"); + return Response.json({ snapshotId: 8, elements: [] }); + } + return Response.json({ + holder: "bot", + requested: false, + request: { id: "request-1", status: "completed" }, + }); + }, + { preconnect: () => undefined }, + ), + ); + return { agent, run, order }; +} + +test("restored missing tool result is completed on the same agent before continuation", async () => { + const r = resumedAgent(); + expect(findPendingTool(r.agent.messages, pending)?.id).toBe("tool-1"); + await resumeHandoffTask("bot-1", pending, r.agent, r.run); + expect(r.order).toEqual(["snapshot", "result", "run"]); + expect(r.agent.messages.filter((m) => m.role === "tool")).toHaveLength(1); + expect(r.agent.messages.at(-1)).toMatchObject({ + role: "tool", + toolCallId: "tool-1", + }); +}); + +test("accepted tool results cannot be resumed or duplicated", async () => { + const r = resumedAgent(); + r.agent.messages.push({ + id: "result", + role: "tool", + toolCallId: "tool-1", + content: '{"ok":true}', + }); + expect(findPendingTool(r.agent.messages, pending)).toBeNull(); + await expect( + resumeHandoffTask("bot-1", pending, r.agent, r.run), + ).rejects.toThrow("already answered"); + expect(r.order).toEqual([]); +}); + +test("a different thread without the restored call cannot start a new task", async () => { + const r = resumedAgent(); + r.agent.messages = []; + await expect( + resumeHandoffTask("bot-1", pending, r.agent, r.run), + ).rejects.toThrow("not in this conversation"); + expect(r.order).toEqual([]); +}); + +test("SDK's forwarded-to-client placeholder is replaced by the actual result", async () => { + const r = resumedAgent(); + r.agent.messages.push({ + id: "placeholder", + role: "tool", + toolCallId: "tool-1", + content: "Forwarded to client", + }); + await resumeHandoffTask("bot-1", pending, r.agent, r.run); + expect(r.agent.messages.filter((m) => m.role === "tool")).toHaveLength(1); + expect( + r.agent.messages.some( + (m) => m.role === "tool" && m.content === "Forwarded to client", + ), + ).toBe(false); + expect(r.order).toEqual(["snapshot", "result", "run"]); +}); + +test("an interrupted restored handoff continues with an honest failure result", async () => { + const r = resumedAgent(); + fetchSpy.mockImplementation( + Object.assign( + async () => + Response.json({ + holder: "bot", + requested: false, + request: { + id: "request-1", + status: "interrupted", + interruption: "The browser restarted.", + }, + }), + { preconnect: () => undefined }, + ), + ); + await resumeHandoffTask("bot-1", pending, r.agent, r.run); + const result = r.agent.messages.find((m) => m.role === "tool"); + expect(result?.content).toContain('"ok":false'); + expect(result?.content).toContain('"handoffStatus":"interrupted"'); + expect(r.order).toEqual(["result", "run"]); +}); diff --git a/app/tests/screen-connection.test.ts b/app/tests/screen-connection.test.ts new file mode 100644 index 000000000..8181f8a78 --- /dev/null +++ b/app/tests/screen-connection.test.ts @@ -0,0 +1,100 @@ +import { expect, test } from "bun:test"; +import { + connectScreen, + type ScreenSocket, +} from "../src/lib/computers/screen-connection"; + +function rig() { + const sockets: ScreenSocket[] = []; + const timers: { run: () => void; delay: number; cancelled: boolean }[] = []; + const statuses: string[] = []; + let verified = false; + let stopped = 0; + const control = connectScreen({ + open: () => { + const socket: ScreenSocket = { + onopen: null, + onclose: null, + onerror: null, + onmessage: null, + close: () => undefined, + }; + sockets.push(socket); + return socket; + }, + schedule: (run, delay) => { + const timer = { run, delay, cancelled: false }; + timers.push(timer); + return () => { + timer.cancelled = true; + }; + }, + verifyOwnership: async () => verified, + onReady: (ready) => statuses.push(ready ? "ready" : "blocked"), + onProblem: (problem) => { + if (problem) statuses.push(problem); + }, + onDisconnect: () => { + stopped++; + }, + onMessage: () => undefined, + }); + return { + sockets, + timers, + statuses, + control, + verify: () => { + verified = true; + }, + stopped: () => stopped, + }; +} + +test("connection loss disables input, retries boundedly, then asks for Retry", async () => { + const r = rig(); + r.sockets[0]?.onopen?.(); + r.sockets[0]?.onclose?.(); + expect(r.statuses.at(-2)).toBe("blocked"); + for (let i = 0; i < 5; i++) { + r.timers[i]?.run(); + r.sockets[i + 1]?.onclose?.(); + } + expect(r.timers.map((t) => t.delay)).toEqual([500, 1000, 2000, 4000, 8000]); + expect(r.statuses.at(-1)).toContain("Retry"); + expect(r.stopped()).toBe(6); + r.control.stop(); +}); + +test("reconnect reacquires ownership before input becomes ready", async () => { + const r = rig(); + r.sockets[0]?.onclose?.(); + r.timers[0]?.run(); + await r.sockets[1]?.onopen?.(); + expect(r.statuses.at(-1)).toBe("blocked"); + r.control.stop(); +}); + +test("a superseded viewer stays stopped until explicit Retry", () => { + const r = rig(); + r.sockets[0]?.onmessage?.({ + data: JSON.stringify({ + type: "error", + error: + "This screen is now being watched somewhere else, so it stopped here.", + }), + }); + r.sockets[0]?.onclose?.(); + expect(r.timers).toHaveLength(0); + expect(r.statuses.at(-1)).toContain("watched somewhere else"); + r.control.stop(); +}); + +test("unmount cancels a pending reconnect and ignores old socket callbacks", () => { + const r = rig(); + r.sockets[0]?.onclose?.(); + r.control.stop(); + r.timers[0]?.run(); + expect(r.timers[0]?.cancelled).toBe(true); + expect(r.sockets).toHaveLength(1); +}); diff --git a/docs/configuration.md b/docs/configuration.md index c32a655f9..3d1df1ac8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -372,7 +372,9 @@ then is a row nothing will read. | `COMPUTER_TOKEN` | Secret every computer request must present. The computer refuses to start without it. | | `COMPUTER_MAX_BROWSERS` | How many Bots may hold a running browser at once. `8` by default; the least recently used is closed past it. | | `COMPUTER_BROWSER_IDLE_MS` | How long an untouched browser is kept. 30 minutes by default; `0` keeps them resident. | -| `COMPUTER_BROWSER_MODE` | `headless` by default; set to `headed` to run full Chromium on a private virtual display for human takeover. | +| `COMPUTER_BROWSER_BACKEND` | `managed` by default (full bundled Chromium); `local-chrome` opts into installed Chrome with dedicated profiles and a loopback API. | +| `COMPUTER_BROWSER_MODE` | Managed defaults to `headless` (full Chromium's new headless mode). `headed` uses Xvfb on Linux and a native window on macOS/Windows. Local Chrome requires `headed`. | +| `OPENBOT_LOCAL_COMPUTER_DIR` | Local startup helper's absolute data root. Defaults to the platform's OpenBot user-data directory; contains `profiles/` and `workspace/`. | | `COMPUTER_SUPERVISOR_URL` | Supervisor URL for per-Bot computers. If absent, Bots share `AGENT_COMPUTER_URL`. | | `SUPERVISOR_TOKEN` | Bearer token required by the supervisor. | | `AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS` | Local-only private-host browsing when `true`. A deployment running with `NODE_ENV=production` refuses to start while it is set. Cloud metadata addresses are refused either way. | @@ -398,6 +400,91 @@ docker ps -aq --filter "label=openbot.namespace=openbot" | xargs -r docker rm -f The supervisor recreates each computer with the same named volumes on its next request. +### Headed managed Chromium in Docker or Helm + +The managed backend uses Playwright's full `chromium` channel in either mode. The Docker image +already installs that browser and Xvfb; it does not require Google Chrome. For Compose, set +`COMPUTER_BROWSER_MODE=headed` in the deployment environment and recreate the shared computer or +supervisor as applicable: + +```sh +docker compose up -d --force-recreate agent-computer supervisor +``` + +Existing supervisor-created computers retain their mode until recreated as described above. The +same per-Bot profile volumes and streamed viewer continue to work. For Helm, add to your values: + +```yaml +computers: + extraEnv: + - name: COMPUTER_BROWSER_MODE + value: headed +``` + +Apply the Helm upgrade. A newly created computer uses the new setting; recreate existing supervised +computers through your deployment's lifecycle controls while keeping their persistent volumes. + +### Installed Chrome for a local API deployment + +This source/deployment-checkout option launches a separate installed Google Chrome window for each +active Bot on macOS or Windows. The existing in-app viewer, browser tools, authentication, and Bot +access policy still apply. Linux uses a private Xvfb display and the viewer. This is not a packaged +desktop toggle or a bridge from a hosted API: the API must run on the same machine and reach the +helper through loopback. + +Install [Bun](https://bun.com/docs/installation) and [Google Chrome](https://www.google.com/chrome/) +in its standard location, then install dependencies from the checkout root: + +```sh +bun install --frozen-lockfile +bun install --cwd agent-computer --frozen-lockfile +``` + +Use the same existing `COMPUTER_TOKEN` for the API and helper. It can be set in the checkout's `.env` +(Bun loads it) or supplied securely in each process environment; the helper never prints it. Set the +following API configuration and clear both supervisor selectors, including any inherited process +environment values, because either selector takes precedence over the shared URL: + +```dotenv +AGENT_COMPUTER_URL=http://127.0.0.1:4101 +COMPUTER_SUPERVISOR_URL= +COMPUTER_SANDBOX_NAMESPACE= +``` + +In the helper's environment, leave `COMPUTER_BROWSER_BACKEND` and `COMPUTER_BROWSER_MODE` unset, or +set them to `local-chrome` and `headed`. An inherited `managed` or `headless` setting is an error. +Leave `PORT` unset for 4101, or explicitly choose another port and update the API URL to match. Start: + +```sh +bun scripts/start-local-chrome-computer.ts +``` + +Restart the API with the configuration above. Open a Bot's computer and navigate to a website; its +dedicated Chrome window starts on first use. Use **Take control** in the app before interacting and +**Hand back** when finished. Closing a viewer does not close the Bot's browser or erase its logins. +The helper exits with an error for a missing token, missing Chrome, or occupied port instead of +silently selecting another browser or port. Ctrl-C shuts down its computer process and browsers. + +The helper ignores inherited `PROFILES_DIR` and `WORKSPACE_DIR`. Its defaults are: + +- macOS: `~/Library/Application Support/OpenBot/local-computer` +- Windows: `%LOCALAPPDATA%\OpenBot\local-computer` +- Linux: `${XDG_DATA_HOME:-~/.local/share}/openbot/local-computer` + +Set `OPENBOT_LOCAL_COMPUTER_DIR` to an absolute, dedicated app-owned directory to change that root. +Do not point it at your personal Chrome data. Each Bot uses a separate persistent subdirectory under +`profiles/`; no existing Chrome session is attached, and no TCP debugging endpoint is exposed. +Native Chrome retains its process sandbox and OS credential store. File tools remain confined to +the helper's `workspace/`, and shell execution is refused: use OpenBot's separately approved host +access tools for host commands. Native Chrome runs with your OS account's network access; this mode +is not a container or an OS network sandbox. Keep the API's private-host browsing opt-in off unless +you intentionally need it for your local deployment. + +To switch back, stop the helper, restore your prior `AGENT_COMPUTER_URL` and supervisor/sandbox +selectors, and restart the API. Unset `COMPUTER_BROWSER_BACKEND` (or set `managed`) on the managed +computer process; choose `headless` or `headed` as before. Local Chrome profiles remain in the local +data root, separate from managed computer volumes. + `agent-computer` also reads: - `ACTION_TIMEOUT_MS` diff --git a/scripts/start-local-chrome-computer.ts b/scripts/start-local-chrome-computer.ts new file mode 100644 index 000000000..52f6f18f7 --- /dev/null +++ b/scripts/start-local-chrome-computer.ts @@ -0,0 +1,174 @@ +import { access, mkdir } from "node:fs/promises"; +import { constants } from "node:fs"; +import { createServer } from "node:net"; +import { homedir } from "node:os"; +import { join, posix, win32 } from "node:path"; +import { browserRuntimeFromEnv } from "../agent-computer/src/browser-runtime"; + +type Environment = Record; + +/** Resolve only app-owned paths; inherited container or personal-profile paths are never reused. */ +export function localChromeConfiguration( + env: Environment, + platform: string = process.platform, + userHome = homedir(), +) { + if (!env.COMPUTER_TOKEN?.trim()) { + throw new Error( + "COMPUTER_TOKEN is required. Use the same secret as the local OpenBot API.", + ); + } + if ( + env.COMPUTER_BROWSER_BACKEND?.trim() && + env.COMPUTER_BROWSER_BACKEND.trim() !== "local-chrome" + ) { + throw new Error( + "This helper requires COMPUTER_BROWSER_BACKEND=local-chrome; unset the managed backend setting.", + ); + } + const path = platform === "win32" ? win32 : posix; + const dataHome = + platform === "darwin" + ? path.join(userHome, "Library", "Application Support", "OpenBot") + : platform === "win32" + ? path.join( + env.LOCALAPPDATA || path.join(userHome, "AppData", "Local"), + "OpenBot", + ) + : path.join( + env.XDG_DATA_HOME || path.join(userHome, ".local", "share"), + "openbot", + ); + const root = + env.OPENBOT_LOCAL_COMPUTER_DIR?.trim() || + path.join(dataHome, "local-computer"); + if (!path.isAbsolute(root)) + throw new Error("OPENBOT_LOCAL_COMPUTER_DIR must be an absolute path."); + const port = env.PORT?.trim() || "4101"; + if (!/^\d+$/.test(port) || Number(port) < 1 || Number(port) > 65535) { + throw new Error("PORT must be a whole number between 1 and 65535."); + } + const childEnv: Environment = { + ...env, + COMPUTER_BROWSER_BACKEND: "local-chrome", + COMPUTER_BROWSER_MODE: env.COMPUTER_BROWSER_MODE?.trim() || "headed", + COMPUTER_SANDBOX: "on", + PORT: port, + PROFILES_DIR: path.join(root, "profiles"), + WORKSPACE_DIR: path.join(root, "workspace"), + }; + browserRuntimeFromEnv(childEnv, platform); + return { root: path.normalize(root), port: Number(port), env: childEnv }; +} + +/** Match Playwright's stable Chrome channel locations; never accept an executable from a tool input. */ +export function chromeExecutableCandidates( + env: Environment, + platform: string = process.platform, +): string[] { + if (platform === "darwin") + return ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"]; + if (platform === "linux") return ["/opt/google/chrome/chrome"]; + if (platform === "win32") { + return [ + env.LOCALAPPDATA, + env.PROGRAMFILES, + env["PROGRAMFILES(X86)"], + ...(env.HOMEDRIVE + ? [ + win32.join(env.HOMEDRIVE, "Program Files"), + win32.join(env.HOMEDRIVE, "Program Files (x86)"), + ] + : []), + ] + .filter((prefix): prefix is string => Boolean(prefix)) + .map((prefix) => + win32.join(prefix, "Google", "Chrome", "Application", "chrome.exe"), + ); + } + throw new Error(`Local Chrome is not supported on ${platform}.`); +} + +export async function requireInstalledChrome( + env: Environment, + platform: string = process.platform, +): Promise { + for (const candidate of chromeExecutableCandidates(env, platform)) { + try { + await access( + candidate, + platform === "win32" ? constants.F_OK : constants.X_OK, + ); + return; + } catch (error) { + if ( + !error || + typeof error !== "object" || + !("code" in error) || + !["ENOENT", "EACCES"].includes(String(error.code)) + ) + throw error; + } + } + throw new Error( + "Google Chrome was not found in its standard installation location. Install Google Chrome before starting the local computer.", + ); +} + +export async function requireAvailablePort(port: number): Promise { + await new Promise((resolve, reject) => { + const probe = createServer(); + probe.once("error", (error) => + reject( + new Error( + `Cannot listen on 127.0.0.1:${port}; free that port or set PORT explicitly.`, + { cause: error }, + ), + ), + ); + probe.listen(port, "127.0.0.1", () => + probe.close((error) => (error ? reject(error) : resolve())), + ); + }); +} + +async function main(): Promise { + const config = localChromeConfiguration(process.env); + await requireInstalledChrome(config.env); + await requireAvailablePort(config.port); + await mkdir(config.root, { recursive: true, mode: 0o700 }); + for (const directory of [config.env.PROFILES_DIR, config.env.WORKSPACE_DIR]) { + if (directory) await mkdir(directory, { recursive: true, mode: 0o700 }); + } + const computerDirectory = join(import.meta.dir, "..", "agent-computer"); + console.info( + `Starting local Chrome computer at http://127.0.0.1:${config.port}; data: ${config.root}`, + ); + const child = Bun.spawn([process.execPath, "--no-env-file", "src/index.ts"], { + cwd: computerDirectory, + env: config.env, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + }); + const stop = () => child.kill("SIGTERM"); + process.on("SIGINT", stop); + process.on("SIGTERM", stop); + try { + process.exitCode = await child.exited; + } finally { + process.off("SIGINT", stop); + process.off("SIGTERM", stop); + } +} + +if (import.meta.main) { + try { + await main(); + } catch (error) { + console.error( + error instanceof Error ? error.message : "Local Chrome startup failed.", + ); + process.exitCode = 1; + } +} diff --git a/server/src/audit.ts b/server/src/audit.ts index 9df95edb5..dcaba60ad 100644 --- a/server/src/audit.ts +++ b/server/src/audit.ts @@ -245,6 +245,7 @@ export const auditEventTypes = [ // useful fact for an investigator is that a human drove this browser between these two times, and // logging every click a person made would bury it while telling nobody anything. "computer.help_requested", + "computer.help_cancelled", "computer.control_taken", "computer.control_released", // A credential a person entered by hand. The row records that it happened, what it was called and diff --git a/server/src/computer/client.ts b/server/src/computer/client.ts index bac3890a8..058f0d4b9 100644 --- a/server/src/computer/client.ts +++ b/server/src/computer/client.ts @@ -1,3 +1,4 @@ +import type { HandoffRequest } from "../../../shared/computer-control"; import type { NavigateResult } from "./schema"; import { checkNavigationTarget } from "./target"; @@ -59,7 +60,10 @@ export class WorkspaceRequestError extends Error { /** The page changed after the caller received its element references. */ export class StaleSnapshotError extends Error { - constructor(reason: string) { + constructor( + reason: string, + readonly snapshotRequired = false, + ) { super(reason); this.name = "StaleSnapshotError"; } @@ -74,12 +78,27 @@ export class StaleSnapshotError extends Error { * puts on the body, which is the only thing in the response that distinguishes them. */ export class HumanHasControlError extends Error { - constructor(reason: string) { + constructor( + reason: string, + readonly requestId?: string, + readonly handoff?: HandoffRequest, + ) { super(reason); this.name = "HumanHasControlError"; } } +/** Invalid, unknown, or superseded handoff identity is distinct from page freshness. */ +export class HandoffRequestError extends Error { + constructor( + reason: string, + readonly status: 400 | 404 | 409, + ) { + super(reason); + this.name = "HandoffRequestError"; + } +} + /** * Transport options used inside the computer gateway. * @@ -117,6 +136,7 @@ export interface ComputerTransport { baseUrl: string, botId: string, url: string, + toolCallId?: string, ): Promise; } @@ -226,6 +246,7 @@ export function createComputerTransport( baseUrl: string, botId: string, url: string, + toolCallId?: string, ): Promise { const verdict = checkNavigationTarget(url, { allowPrivateHosts: options.allowPrivateHosts, @@ -235,6 +256,7 @@ export function createComputerTransport( } return post(baseUrl, botId, "/navigate", { url: verdict.url, + ...(toolCallId ? { toolCallId } : {}), }); } @@ -248,12 +270,23 @@ function throwMappedError( ): never { const detail = typeof body?.error === "string" ? body.error : `HTTP ${status}`; + if ( + body?.controlRequestError === true && + (status === 400 || status === 404 || status === 409) + ) + throw new HandoffRequestError(detail, status); if (status === 409) { // The computer says which kind of 409 this is. Absent, it is the ordinary one. if (body?.humanHasControl === true) { - throw new HumanHasControlError(detail); + throw new HumanHasControlError( + detail, + typeof body.requestId === "string" ? body.requestId : undefined, + body.handoff && typeof body.handoff === "object" + ? (body.handoff as HandoffRequest) + : undefined, + ); } - throw new StaleSnapshotError(detail); + throw new StaleSnapshotError(detail, body?.snapshotRequired === true); } if (status === 403) { throw new WorkspaceRefusedError(detail); diff --git a/server/src/computer/gateway.ts b/server/src/computer/gateway.ts index 5aa514607..c1be93812 100644 --- a/server/src/computer/gateway.ts +++ b/server/src/computer/gateway.ts @@ -30,6 +30,7 @@ export { ComputerUnavailableError, ElementNotFoundError, HumanHasControlError, + HandoffRequestError, NavigationRefusedError, StaleSnapshotError, WorkspaceRefusedError, @@ -135,6 +136,7 @@ export interface ComputerGateway { botId: string, actor: ActionActor, url: string, + toolCallId?: string, ): Promise; click( botId: string, @@ -180,14 +182,28 @@ export interface ComputerGateway { actor: ActionActor, input: WriteFileInput, ): Promise; - control(botId: string): Promise; + control(botId: string, requestId?: string): Promise; requestHelp( botId: string, actor: ActionActor, reason: string, + toolCallId?: string, + ): Promise; + takeControl( + botId: string, + actor: ActionActor, + requestId: string, + ): Promise; + releaseControl( + botId: string, + actor: ActionActor, + requestId: string, + ): Promise; + cancelControl( + botId: string, + actor: ActionActor, + requestId: string, ): Promise; - takeControl(botId: string, actor: ActionActor): Promise; - releaseControl(botId: string, actor: ActionActor): Promise; requestSecret( botId: string, actor: ActionActor, @@ -663,9 +679,15 @@ export function createComputerGateway( * row and do not ask. What IS recorded is the period: who, when, and why the Bot asked, the fact * an investigator wants is that a human drove this browser between two times. */ - async requestHelp(botId: string, actor: ActionActor, reason: string) { + async requestHelp( + botId: string, + actor: ActionActor, + reason: string, + toolCallId?: string, + ) { const state = await post(botId, "/control/request", { reason, + ...(toolCallId ? { toolCallId } : {}), }); await writeControlEvent(auditStore, "computer.help_requested", { botId, @@ -675,8 +697,10 @@ export function createComputerGateway( return state; }, - async takeControl(botId: string, actor: ActionActor) { - const state = await post(botId, "/control/take", {}); + async takeControl(botId: string, actor: ActionActor, requestId: string) { + const state = await post(botId, "/control/take", { + requestId, + }); await writeControlEvent(auditStore, "computer.control_taken", { botId, actor, @@ -687,8 +711,10 @@ export function createComputerGateway( return state; }, - async releaseControl(botId: string, actor: ActionActor) { - const state = await post(botId, "/control/release", {}); + async releaseControl(botId: string, actor: ActionActor, requestId: string) { + const state = await post(botId, "/control/release", { + requestId, + }); await writeControlEvent(auditStore, "computer.control_released", { botId, actor, @@ -696,8 +722,22 @@ export function createComputerGateway( return state; }, - control(botId: string): Promise { - return get(botId, "/control"); + async cancelControl(botId: string, actor: ActionActor, requestId: string) { + const state = await post(botId, "/control/cancel", { + requestId, + }); + await writeControlEvent(auditStore, "computer.help_cancelled", { + botId, + actor, + }); + return state; + }, + + control(botId: string, requestId?: string): Promise { + return get( + botId, + `/control${requestId ? `?requestId=${encodeURIComponent(requestId)}` : ""}`, + ); }, /** Return every computer that the configured provider owns. */ @@ -835,13 +875,19 @@ export function createComputerGateway( * The transport applies its target guard before it sends a request. This is * the minimum rule that applies even when the action policy permits the URL. */ - navigate(botId: string, actor: ActionActor, url: string) { + navigate( + botId: string, + actor: ActionActor, + url: string, + toolCallId?: string, + ) { return govern( "computer_navigate", botId, actor, { targetUrl: url }, - async () => transport.navigate(await locate(botId), botId, url), + async () => + transport.navigate(await locate(botId), botId, url, toolCallId), ); }, @@ -1203,6 +1249,7 @@ async function writeControlEvent( auditStore: AuditStore, eventType: | "computer.help_requested" + | "computer.help_cancelled" | "computer.control_taken" | "computer.control_released" | "computer.secret_requested" diff --git a/server/src/computer/routes.ts b/server/src/computer/routes.ts index bad163f9f..5c49df936 100644 --- a/server/src/computer/routes.ts +++ b/server/src/computer/routes.ts @@ -12,6 +12,7 @@ import { ComputerUnavailableError, ElementNotFoundError, HumanHasControlError, + HandoffRequestError, NavigationRefusedError, StaleSnapshotError, WorkspaceRefusedError, @@ -253,6 +254,7 @@ export function createComputerRoutes( : { userId: context.var.actor.id }), }, body.url.trim(), + toolCallId || undefined, ); await keepFrameOf(botId, toolCallId, result.url, result.title); return context.json(result); @@ -345,7 +347,12 @@ export function createComputerRoutes( */ routes.get("/:botId/control", async (context) => { try { - return context.json(await gateway.control(context.req.param("botId"))); + return context.json( + await gateway.control( + context.req.param("botId"), + context.req.query("requestId"), + ), + ); } catch (error) { return context.json(errorBody(error), statusFor(error)); } @@ -359,6 +366,7 @@ export function createComputerRoutes( typeof body?.reason === "string" && body.reason.trim() ? body.reason.trim() : "The assistant needs a person to continue.", + typeof body?.toolCallId === "string" ? body.toolCallId : undefined, ), ), ); @@ -415,11 +423,21 @@ export function createComputerRoutes( ); routes.post("/:botId/control/take", (context) => - act(context, (botId, actor) => gateway.takeControl(botId, actor)), + act(context, (botId, actor, body) => + gateway.takeControl(botId, actor, handoffId(body)), + ), ); routes.post("/:botId/control/release", (context) => - act(context, (botId, actor) => gateway.releaseControl(botId, actor)), + act(context, (botId, actor, body) => + gateway.releaseControl(botId, actor, handoffId(body)), + ), + ); + + routes.post("/:botId/control/cancel", (context) => + act(context, (botId, actor, body) => + gateway.cancelControl(botId, actor, handoffId(body)), + ), ); /** The Bot asking for a value it must not be told. */ @@ -918,11 +936,37 @@ function errorBody(error: unknown): Record { return { error: describe(error), // Not "the refs are stale, take another snapshot", which is what the surface says without it. - ...(error instanceof HumanHasControlError ? { humanHasControl: true } : {}), + ...(error instanceof HumanHasControlError + ? { + humanHasControl: true, + requestId: error.requestId, + handoff: error.handoff, + } + : {}), + ...(error instanceof StaleSnapshotError + ? { + stale: true, + ...(error.snapshotRequired ? { snapshotRequired: true } : {}), + } + : {}), + ...(error instanceof HandoffRequestError + ? { controlRequestError: true } + : {}), }; } -function statusFor(error: unknown): 409 | 500 | 503 { +function handoffId(body: Record | null): string { + if ( + typeof body?.requestId !== "string" || + !body.requestId.trim() || + body.requestId.length > 200 + ) + throw new HandoffRequestError("A requestId is required.", 400); + return body.requestId; +} + +function statusFor(error: unknown): 400 | 404 | 409 | 500 | 503 { + if (error instanceof HandoffRequestError) return error.status; if (error instanceof StaleSnapshotError) return 409; // Same status as a stale snapshot and for the same reason: nothing is broken, the caller has to do // something else first. What differs is what that something is, which the body carries. diff --git a/server/src/computer/schema.ts b/server/src/computer/schema.ts index 70e88ac94..57055252f 100644 --- a/server/src/computer/schema.ts +++ b/server/src/computer/schema.ts @@ -1,3 +1,7 @@ +import type { + BrowserChallenge, + ComputerControlState, +} from "../../../shared/computer-control"; /** * The computer-use contract. * @@ -65,8 +69,9 @@ export function isActingTool(name: string): name is ComputerActingToolName { export type ComputerToolName = (typeof COMPUTER_TOOLS)[number]; -export type NavigateInput = { url: string }; +export type NavigateInput = { url: string; toolCallId?: string }; export type NavigateResult = { + challenge?: BrowserChallenge; url: string; title: string; /** @@ -127,6 +132,7 @@ export type SnapshotElement = { }; export type SnapshotResult = { + challenge?: BrowserChallenge; /** * Which snapshot these refs belong to. Must be sent back with every action. * @@ -266,15 +272,7 @@ export type WriteFileResult = { */ export type ControlHolder = "bot" | "human"; -export type ControlState = { - holder: ControlHolder; - /** ISO timestamp of the last handover, so the surface can say how long this has been going on. */ - since: string; - /** Why the Bot asked for help, in its own words. Shown to the person being handed the wheel. */ - reason?: string; - /** The Bot has asked and nobody has taken over yet. */ - requested: boolean; -}; +export type ControlState = ComputerControlState; /** * A value the Bot needs and must not be told: a password, a one-time code. diff --git a/server/tests/computer-gateway.test.ts b/server/tests/computer-gateway.test.ts index 87df7a5c2..9515a6615 100644 --- a/server/tests/computer-gateway.test.ts +++ b/server/tests/computer-gateway.test.ts @@ -851,8 +851,8 @@ describe("the computer gateway", () => { await gateway.listFiles("bot-1", ACTOR, { path: "notes" }); await gateway.control("bot-1"); await gateway.requestHelp("bot-1", ACTOR, "Sign in"); - await gateway.takeControl("bot-1", ACTOR); - await gateway.releaseControl("bot-1", ACTOR); + await gateway.takeControl("bot-1", ACTOR, "request-1"); + await gateway.releaseControl("bot-1", ACTOR, "request-1"); await gateway.requestSecret("bot-1", ACTOR, { label: "Password", ref: "e1", diff --git a/server/tests/computer-handoff-contract.test.ts b/server/tests/computer-handoff-contract.test.ts new file mode 100644 index 000000000..42d28cedc --- /dev/null +++ b/server/tests/computer-handoff-contract.test.ts @@ -0,0 +1,197 @@ +import { expect, test } from "bun:test"; +import type { MiddlewareHandler } from "hono"; +import type { + BrowserChallenge, + ComputerControlState, +} from "../../shared/computer-control"; +import type { AppVariables } from "../src/auth/guards"; +import { createComputerGateway } from "../src/computer/gateway"; +import type { PolicyStore } from "../src/computer/policy-store"; +import type { ComputerProvider } from "../src/computer/provider"; +import { createComputerRoutes } from "../src/computer/routes"; + +const state: ComputerControlState = { + holder: "bot", + since: "2026-09-26T00:00:00Z", + requested: true, + transitioning: false, + resumeSnapshotRequired: false, + request: { + id: "request-1", + toolCallId: "call-1", + source: "model", + reason: "Sign in", + status: "waiting", + createdAt: "2026-09-26T00:00:00Z", + updatedAt: "2026-09-26T00:00:00Z", + expiresAt: "2026-09-26T00:10:00Z", + }, +}; +function fixture( + reply: (path: string, init?: RequestInit) => Response = () => + Response.json(state), +) { + const requests: { path: string; init?: RequestInit }[] = []; + const provider: ComputerProvider = { + name: "fixture", + isolation: "per-bot", + locate: async () => "http://agent-computer:4100", + status: async (botId) => ({ botId, state: "ready" }), + stop: async () => ({ wasRunning: true }), + reset: async () => ({ cleared: true }), + list: async () => [], + }; + const fetchImpl: typeof fetch = Object.assign( + async (input: Parameters[0], init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + const path = `${url.pathname}${url.search}`; + requests.push({ path, init }); + return reply(path, init); + }, + { preconnect: fetch.preconnect }, + ); + const policy = { mode: "enforce" as const, deny: [], allow: ["true"] }; + const gateway = createComputerGateway({ + provider, + fetchImpl, + policy: () => policy, + auditStore: { insert: async () => {} }, + }); + const policyStore: PolicyStore = { + get: () => policy, + set: async () => {}, + reset: async () => {}, + load: async () => "configuration", + refresh: async () => {}, + }; + const user: MiddlewareHandler<{ Variables: AppVariables }> = async ( + context, + next, + ) => { + context.set("actor", { + id: "user-1", + email: "test@example.com", + role: "user", + }); + await next(); + }; + const routes = createComputerRoutes( + gateway, + policyStore, + user, + async () => true, + ); + const post = (path: string, body: unknown) => + routes.request(`http://app.test/bot-1${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + return { requests, routes, post }; +} + +test("request/rejoin and exact polling/take/release/cancel identities reach the computer", async () => { + const { post, routes, requests } = fixture(); + expect( + ( + await post("/control/request", { + reason: "Sign in", + toolCallId: "call-1", + }) + ).status, + ).toBe(200); + expect( + (await routes.request("http://app.test/bot-1/control?requestId=old%2Bid")) + .status, + ).toBe(200); + for (const path of ["take", "release", "cancel"]) + expect( + (await post(`/control/${path}`, { requestId: "request-1" })).status, + ).toBe(200); + expect(requests.map((r) => r.path)).toEqual([ + "/control/request", + "/control?requestId=old%2Bid", + "/control/take", + "/control/release", + "/control/cancel", + ]); + expect(JSON.parse(String(requests[0]?.init?.body))).toEqual({ + reason: "Sign in", + toolCallId: "call-1", + }); + for (const request of requests.slice(2)) + expect(JSON.parse(String(request.init?.body))).toEqual({ + requestId: "request-1", + }); + const count = requests.length; + expect((await post("/control/take", {})).status).toBe(400); + expect(requests).toHaveLength(count); +}); + +test("navigation carries tool identity and challenge; read and snapshot preserve challenge metadata", async () => { + const challenge: BrowserChallenge = { + kind: "cloudflare", + reason: "Complete the challenge", + requestId: "request-1", + }; + const { post, routes, requests } = fixture((path) => + Response.json( + path === "/snapshot" + ? { + snapshotId: 1, + url: "https://example.com", + title: "Challenge", + elements: [], + truncated: false, + challenge, + } + : { + url: "https://example.com", + title: "Challenge", + text: "Verify", + truncated: false, + elapsedMs: 1, + challenge, + }, + ), + ); + const navigation = await post("/navigate", { + url: "https://example.com", + toolCallId: "call-1", + }); + expect((await navigation.json()).challenge).toEqual(challenge); + const sent = requests.find((r) => r.path === "/navigate"); + expect(JSON.parse(String(sent?.init?.body))).toEqual({ + url: "https://example.com/", + toolCallId: "call-1", + }); + expect( + (await (await routes.request("http://app.test/bot-1/read")).json()) + .challenge, + ).toEqual(challenge); + expect((await (await post("/snapshot", {})).json()).challenge).toEqual( + challenge, + ); +}); + +test("ownership, freshness, unknown ID and superseded ID survive transport and route shaping", async () => { + for (const [status, body] of [ + [ + 409, + { + error: "Wait", + humanHasControl: true, + requestId: "request-1", + handoff: state.request, + }, + ], + [409, { error: "Snapshot required", stale: true, snapshotRequired: true }], + [404, { error: "Unknown request", controlRequestError: true }], + [409, { error: "Old request", controlRequestError: true }], + ] as const) { + const { post } = fixture(() => Response.json(body, { status })); + const response = await post("/control/take", { requestId: "request-1" }); + expect(response.status).toBe(status); + expect(await response.json()).toEqual(body); + } +}); diff --git a/shared/computer-control.ts b/shared/computer-control.ts new file mode 100644 index 000000000..07ad41915 --- /dev/null +++ b/shared/computer-control.ts @@ -0,0 +1,44 @@ +/** Durable, request-scoped browser handoff shared by the computer, API, and app. */ +export type HandoffStatus = + | "waiting" + | "taken" + | "completed" + | "cancelled" + | "expired" + | "interrupted"; +export type HandoffSource = + | "model" + | "cloudflare" + | "visible-challenge" + | "manual"; +export type HandoffRequest = { + id: string; + toolCallId?: string; + reason: string; + source: HandoffSource; + status: HandoffStatus; + createdAt: string; + updatedAt: string; + expiresAt?: string; + finishedAt?: string; + interruption?: string; +}; +export type ComputerControlState = { + holder: "bot" | "human"; + since: string; + requested: boolean; + reason?: string; + request?: HandoffRequest; + transitioning: boolean; + resumeSnapshotRequired: boolean; + secretWanted?: string; + secretRef?: string; + secretSnapshotId?: number; +}; +export type RequestHelpInput = { reason: string; toolCallId?: string }; +export type ControlRequestInput = { requestId: string }; +export type BrowserChallenge = { + kind: "cloudflare" | "visible-challenge"; + reason: string; + requestId: string; +};