From 0e53012f374e2f5e6600623e2be134b4babbc50b Mon Sep 17 00:00:00 2001 From: Janghoon Lee <44862514+its-janghoon@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:43:23 +0000 Subject: [PATCH 1/3] feat(browser): drive the person's own Chrome through the Redrob extension The browser tool launched its own headless Chromium, so logged-in sites and the Redrob extension were out of reach. Now, when the person's Chrome with the Redrob extension is connected to the same account, the tool runs there. Path: the desk relay, with no change to the extension, Redrob Desktop or the chat server. The extension already takes the desk role in room desk: and answers paired devices with its forward.ts route set. The daemon now also joins that room as a device (kind "web", "Redrob Code on "), the same handshake Redrob chat's web page uses: desk.hello, the six-digit Allow in Chrome once, then sealed req/res frames. The room is the account, so the browser is signed in to the same Redrob account. - desk/extension.ts: ExtensionLink, the device side. It runs beside the desk relay and shares its token refresh. It only listens until the first request; desk.hello and the pairing notification go out when the tool needs the browser. The device key and the paired browser are kept in the desk secret scope (browserKey, browserDesk). A request waits up to two minutes for Allow, once per pairing request. - crypto.ts: Channel.device and devicePairingCode, the device side of the existing handshake. - POST /desk/browser on the daemon: one request of the extension's route set, for engines that are not the daemon (redrob run, the terminal app). GET /desk/status gains `browser` (off, none, found, pairing with code, ready, denied); `redrob desk status` prints it. - browser/extension.ts: each action (open, read, click, type, scroll, wait) becomes one short task in the same conversation and tab (followUp, chatId code-), and the extension's agent answers with a reading in the tool's shape (URL, title, text, numbered elements). Approvals, payments and sign-ins stay the person's step in Chrome. A question from the browser's agent ends the task and is the result. Screenshot is not available there (the route set has no picture); the tool says to use read. - Every result starts with a line naming the browser used. Without a daemon, relay, browser or Allow, the tool falls back to its own Chromium and says why; a session stays in one browser until the next open. - The tool is offered where a Chromium is installed or an engine service record exists. --- packages/client/src/contract.ts | 1 + .../client/src/generated-effect/client.ts | 13 + packages/client/src/generated/client.ts | 14 + packages/client/src/generated/types.ts | 79 +++ packages/protocol/src/groups/channels.ts | 45 ++ packages/redrob/src/browser/extension.ts | 274 +++++++++ packages/redrob/src/browser/tool.ts | 14 +- packages/redrob/src/cli/cmd/desk.ts | 4 + packages/redrob/src/cli/cmd/serve.ts | 1 + packages/redrob/src/desk/crypto.ts | 32 + packages/redrob/src/desk/extension.ts | 567 ++++++++++++++++++ packages/redrob/src/desk/relay.ts | 3 +- packages/redrob/src/desk/service.ts | 41 ++ packages/redrob/src/tool/browser.ts | 20 +- packages/redrob/src/tool/browser.txt | 8 +- packages/redrob/src/tool/registry.ts | 8 +- .../redrob/test/browser/extension.test.ts | 140 +++++ packages/redrob/test/desk/extension.test.ts | 282 +++++++++ packages/sdk/js/src/v2/gen/sdk.gen.ts | 27 + packages/sdk/js/src/v2/gen/types.gen.ts | 51 ++ packages/server/src/channels.ts | 27 + packages/server/src/handlers/channels.ts | 9 + 22 files changed, 1646 insertions(+), 14 deletions(-) create mode 100644 packages/redrob/src/browser/extension.ts create mode 100644 packages/redrob/src/desk/extension.ts create mode 100644 packages/redrob/test/browser/extension.test.ts create mode 100644 packages/redrob/test/desk/extension.test.ts diff --git a/packages/client/src/contract.ts b/packages/client/src/contract.ts index 691e6f5d1a..2acb59ee83 100644 --- a/packages/client/src/contract.ts +++ b/packages/client/src/contract.ts @@ -65,6 +65,7 @@ export const endpointNames = { "desk.revoke": "deskRevoke", "desk.machine.get": "deskMachine", "desk.machine.update": "deskMachineUpdate", + "desk.browser": "deskBrowser", } as const export const omitEndpoints = new Set(["fs.read", "pty.connect", "pty.connectToken"]) diff --git a/packages/client/src/generated-effect/client.ts b/packages/client/src/generated-effect/client.ts index 27c07a1c49..c70b9e6ba4 100644 --- a/packages/client/src/generated-effect/client.ts +++ b/packages/client/src/generated-effect/client.ts @@ -109,6 +109,18 @@ const Endpoint2_12 = (raw: RawClient["server.channels"]) => (input?: Endpoint2_1 payload: { name: input?.["name"], kind: input?.["kind"], access: input?.["access"] }, }).pipe(Effect.mapError(mapClientError)) +type Endpoint2_13Request = Parameters[0] +type Endpoint2_13Input = { + readonly method: Endpoint2_13Request["payload"]["method"] + readonly path: Endpoint2_13Request["payload"]["path"] + readonly body?: Endpoint2_13Request["payload"]["body"] + readonly pairMs?: Endpoint2_13Request["payload"]["pairMs"] +} +const Endpoint2_13 = (raw: RawClient["server.channels"]) => (input: Endpoint2_13Input) => + raw["desk.browser"]({ + payload: { method: input["method"], path: input["path"], body: input["body"], pairMs: input["pairMs"] }, + }).pipe(Effect.mapError(mapClientError)) + const adaptGroup2 = (raw: RawClient["server.channels"]) => ({ status: Endpoint2_0(raw), reload: Endpoint2_1(raw), @@ -123,6 +135,7 @@ const adaptGroup2 = (raw: RawClient["server.channels"]) => ({ deskRevoke: Endpoint2_10(raw), deskMachine: Endpoint2_11(raw), deskMachineUpdate: Endpoint2_12(raw), + deskBrowser: Endpoint2_13(raw), }) const Endpoint3_0 = (raw: RawClient["server.apps"]) => () => raw["apps.list"]({}).pipe(Effect.mapError(mapClientError)) diff --git a/packages/client/src/generated/client.ts b/packages/client/src/generated/client.ts index 4b43fb03d4..572ceec1d3 100644 --- a/packages/client/src/generated/client.ts +++ b/packages/client/src/generated/client.ts @@ -26,6 +26,8 @@ import type { ServerChannelsDeskMachineOutput, ServerChannelsDeskMachineUpdateInput, ServerChannelsDeskMachineUpdateOutput, + ServerChannelsDeskBrowserInput, + ServerChannelsDeskBrowserOutput, ServerAppsListOutput, ServerAppsRegisterInput, ServerAppsRegisterOutput, @@ -465,6 +467,18 @@ export function make(options: ClientOptions) { }, requestOptions, ), + deskBrowser: (input: ServerChannelsDeskBrowserInput, requestOptions?: RequestOptions) => + request( + { + method: "POST", + path: `/desk/browser`, + body: { method: input["method"], path: input["path"], body: input["body"], pairMs: input["pairMs"] }, + successStatus: 200, + declaredStatuses: [503, 401, 400], + empty: false, + }, + requestOptions, + ), }, "server.apps": { list: (requestOptions?: RequestOptions) => diff --git a/packages/client/src/generated/types.ts b/packages/client/src/generated/types.ts index b812df1325..75a7d468a0 100644 --- a/packages/client/src/generated/types.ts +++ b/packages/client/src/generated/types.ts @@ -203,6 +203,14 @@ export type ServerChannelsDeskStatusOutput = { readonly addedAt: number readonly online: boolean }> + readonly browser?: + | { + readonly state: "off" | "none" | "found" | "pairing" | "ready" | "denied" + readonly name?: string | undefined + readonly code?: string | undefined + readonly reason?: string | undefined + } + | undefined } export type ServerChannelsDeskEnableInput = { readonly enabled: { readonly enabled: boolean }["enabled"] } @@ -229,6 +237,14 @@ export type ServerChannelsDeskEnableOutput = { readonly addedAt: number readonly online: boolean }> + readonly browser?: + | { + readonly state: "off" | "none" | "found" | "pairing" | "ready" | "denied" + readonly name?: string | undefined + readonly code?: string | undefined + readonly reason?: string | undefined + } + | undefined } export type ServerChannelsDeskLoginOutput = { @@ -259,6 +275,14 @@ export type ServerChannelsDeskLogoutOutput = { readonly addedAt: number readonly online: boolean }> + readonly browser?: + | { + readonly state: "off" | "none" | "found" | "pairing" | "ready" | "denied" + readonly name?: string | undefined + readonly code?: string | undefined + readonly reason?: string | undefined + } + | undefined } export type ServerChannelsDeskTokenSetInput = { @@ -301,6 +325,14 @@ export type ServerChannelsDeskTokenSetOutput = { readonly addedAt: number readonly online: boolean }> + readonly browser?: + | { + readonly state: "off" | "none" | "found" | "pairing" | "ready" | "denied" + readonly name?: string | undefined + readonly code?: string | undefined + readonly reason?: string | undefined + } + | undefined } export type ServerChannelsDeskTokenClearOutput = { @@ -325,6 +357,14 @@ export type ServerChannelsDeskTokenClearOutput = { readonly addedAt: number readonly online: boolean }> + readonly browser?: + | { + readonly state: "off" | "none" | "found" | "pairing" | "ready" | "denied" + readonly name?: string | undefined + readonly code?: string | undefined + readonly reason?: string | undefined + } + | undefined } export type ServerChannelsDeskApproveInput = { readonly code: { readonly code: string }["code"] } @@ -385,6 +425,45 @@ export type ServerChannelsDeskMachineUpdateOutput = { readonly access: "read_only" | "ask" | "trusted" } +export type ServerChannelsDeskBrowserInput = { + readonly method: { + readonly method: "GET" | "POST" + readonly path: string + readonly body?: unknown | undefined + readonly pairMs?: number | undefined + }["method"] + readonly path: { + readonly method: "GET" | "POST" + readonly path: string + readonly body?: unknown | undefined + readonly pairMs?: number | undefined + }["path"] + readonly body?: { + readonly method: "GET" | "POST" + readonly path: string + readonly body?: unknown | undefined + readonly pairMs?: number | undefined + }["body"] + readonly pairMs?: { + readonly method: "GET" | "POST" + readonly path: string + readonly body?: unknown | undefined + readonly pairMs?: number | undefined + }["pairMs"] +} + +export type ServerChannelsDeskBrowserOutput = { + readonly status: number + readonly body?: unknown | undefined + readonly browser: { + readonly state: "off" | "none" | "found" | "pairing" | "ready" | "denied" + readonly name?: string | undefined + readonly code?: string | undefined + readonly reason?: string | undefined + } + readonly error?: string | undefined +} + export type ServerAppsListOutput = { readonly apps: ReadonlyArray<{ readonly id: string diff --git a/packages/protocol/src/groups/channels.ts b/packages/protocol/src/groups/channels.ts index e57bbdb2c6..419dffea17 100644 --- a/packages/protocol/src/groups/channels.ts +++ b/packages/protocol/src/groups/channels.ts @@ -42,6 +42,18 @@ export const DeskLogin = Schema.Struct({ expiresAt: Schema.Number, }).annotate({ identifier: "DeskLogin" }) +/** + * Where this engine stands with the person's browser extension (redrob-extension, docs/remote-control.md): + * `off` no relay connection, `none` no browser with the extension in this account, `found` one is + * online, `pairing` waiting for Allow in that browser (`code`), `ready`, `denied` (`reason`). + */ +export const DeskBrowserState = Schema.Struct({ + state: Schema.Literals(["off", "none", "found", "pairing", "ready", "denied"]), + name: Schema.optional(Schema.String), + code: Schema.optional(Schema.String), + reason: Schema.optional(Schema.String), +}).annotate({ identifier: "DeskBrowserState" }) + export const DeskStatus = Schema.Struct({ /** The switch (`channels.json` entry `desk`); on unless turned off. */ enabled: Schema.Boolean, @@ -66,8 +78,27 @@ export const DeskStatus = Schema.Struct({ online: Schema.Boolean, }), ), + /** The person's Chrome with the Redrob extension, as the `browser` tool reaches it over the relay. */ + browser: Schema.optional(DeskBrowserState), }).annotate({ identifier: "DeskStatus" }) +/** One request of the extension's route set (its `src/remote/forward.ts`); anything else it answers 403. */ +export const DeskBrowserRequest = Schema.Struct({ + method: Schema.Literals(["GET", "POST"]), + path: Schema.String, + body: Schema.optional(Schema.Unknown), + /** How long to wait for Allow in the browser the first time, in ms (default two minutes). */ + pairMs: Schema.optional(Schema.Number), +}).annotate({ identifier: "DeskBrowserRequest" }) + +export const DeskBrowserResponse = Schema.Struct({ + /** The extension's status; 0 when the request did not reach it (`error` says why). */ + status: Schema.Number, + body: Schema.optional(Schema.Unknown), + browser: DeskBrowserState, + error: Schema.optional(Schema.String), +}).annotate({ identifier: "DeskBrowserResponse" }) + /** Redrob Desktop's own redrob-auth access token (client `redrob-desktop`), for `PUT /desk/token`. */ export const DeskToken = Schema.Struct({ accessToken: Schema.String, @@ -267,3 +298,17 @@ export const ChannelsGroup = HttpApiGroup.make("server.channels") }), ), ) + .add( + HttpApiEndpoint.post("desk.browser", "/desk/browser", { + payload: DeskBrowserRequest, + success: DeskBrowserResponse, + error: ServiceUnavailableError, + }).annotateMerge( + OpenApi.annotations({ + identifier: "v2.desk.browser", + summary: "Ask the person's browser extension", + description: + "Sends one request of the Redrob extension's route set to the person's Chrome over the desk relay, as a paired device (the first time the person allows this engine in Chrome). Used by the browser tool of every engine on this machine. status 0 when it did not reach the extension. 503 on a server that is not the registered daemon.", + }), + ), + ) diff --git a/packages/redrob/src/browser/extension.ts b/packages/redrob/src/browser/extension.ts new file mode 100644 index 0000000000..293d1abf38 --- /dev/null +++ b/packages/redrob/src/browser/extension.ts @@ -0,0 +1,274 @@ +/** + * The `browser` tool in the person's own Chrome, through the Redrob extension. + * + * The extension takes tasks, not clicks: its own agent drives the page (redrob-extension AGENTS.md, + * "Nothing outside the extension calls the layer directly"), and every caller uses the same route + * set (`POST /browser/task`, `GET /browser/task/:id`, docs/remote-control.md). So each action of + * the tool becomes one short, exact task in the same conversation and tab (`followUp`), and the + * extension answers with a reading in the tool's own shape: URL, title, text and numbered elements. + * Approvals, payments and sign-ins stay the person's step in Chrome, at the level they chose there. + * + * The daemon holds the relay link (desk/extension.ts). An engine that is the daemon asks it + * directly; any other engine on this machine (`redrob run`, the terminal app) asks it with + * `POST /desk/browser`. With no daemon, no relay, no browser, or no Allow, `act` returns a fallback + * reason and the tool uses its own Chromium as before. + */ +import * as Channels from "@redrob-code/server/channels" +import * as Registry from "@/daemon/registry" + +export type Action = "open" | "read" | "click" | "type" | "scroll" | "wait" | "screenshot" + +export type Input = { + readonly action: Action + readonly url?: string + readonly ref?: string + readonly text?: string + readonly submit?: boolean + readonly direction?: "down" | "up" + readonly amount?: number + readonly seconds?: number +} + +export type Ask = (request: Channels.DeskBrowserRequest, signal?: AbortSignal) => Promise + +export type Result = + | { readonly kind: "extension"; readonly browser: string; readonly url: string; readonly text: string } + | { readonly kind: "fallback"; readonly reason: string } + +const POLL_MS = 1_000 +/** A task waits at most this long, as long as a hand-over to the person may last. */ +const TASK_MS = 10 * 60_000 +const TERMINAL = new Set(["done", "failed", "stopped", "capped"]) + +/** How the reply is asked for, so the model reads the same shape as from its own Chromium. */ +export const READING = [ + "Then reply with only a reading of the page as it is now, in exactly this form and nothing else:", + "URL:
", + "Title: ", + "", + "<the page's visible text, up to about 3000 characters>", + "", + "Elements:", + "[e1] <role> «<text>»", + "One line per link, button and field on the page you could act on, numbered e1, e2, … in page order (at most 60).", + "Do nothing else on the page: no other clicks, typing or submitting.", +].join("\n") + +type Session = { + /** The last task, so the next action goes on in its conversation and tab. */ + task?: string + /** The element lines of the last reading, by ref, so a click names what it clicks. */ + elements: Map<string, string> + /** The browser this session last used; a session stays in it until the next `open`. */ + used?: "extension" | "chromium" +} + +const sessions = new Map<string, Session>() + +function session(id: string): Session { + let state = sessions.get(id) + if (!state) { + state = { elements: new Map() } + sessions.set(id, state) + if (sessions.size > 200) sessions.delete(sessions.keys().next().value!) + } + return state +} + +/** The chat id the extension keys follow-ups on: one per Code session, in its id shape. */ +export function chatId(sessionID: string): string { + return `code-${sessionID}`.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128) +} + +/** `[e5] button «Send»` from a reading's Elements list, or just the ref when it is not there. */ +function label(state: Session, ref: string): string { + const line = state.elements.get(ref.replace(/^\[|\]$/g, "")) + return line ? `[${ref}] ${line}` : `[${ref}]` +} + +/** The task one action becomes. Undefined for an action the extension cannot do (screenshot). */ +export function task(input: Input, state: Session): string | undefined { + const quote = (text: string) => `«${text.replace(/»/g, "”")}»` + switch (input.action) { + case "open": + return `Open ${input.url} in this tab.\n${READING}` + case "read": + return `Read the page in this tab again.\n${READING}` + case "click": + return `Click ${label(state, input.ref ?? "")} from your last reading, once.\n${READING}` + case "type": + return [ + `Type exactly this text into the field ${label(state, input.ref ?? "")} from your last reading, replacing what is in it: ${quote(input.text ?? "")}.`, + input.submit ? "Then press Enter once." : "Do not press Enter.", + READING, + ].join("\n") + case "scroll": + return input.ref + ? `Scroll ${label(state, input.ref)} from your last reading into view.\n${READING}` + : `Scroll the page ${input.direction ?? "down"} by about ${input.amount ?? 600} pixels.\n${READING}` + case "wait": + return input.text + ? `Wait until the text ${quote(input.text)} is visible in this tab, at most ${Math.min(input.seconds ?? 30, 60)} seconds.\n${READING}` + : `Read the page in this tab again.\n${READING}` + case "screenshot": + return undefined + } +} + +/** The element lines of a reading, by ref. */ +export function elements(text: string): Map<string, string> { + const out = new Map<string, string>() + for (const line of text.split("\n")) { + const found = /^\s*\[(e\d+)\]\s*(.+?)\s*$/.exec(line) + if (found) out.set(found[1]!, found[2]!) + } + return out +} + +type TaskEvent = { n: number; kind: string; text?: string; tool?: string; url?: string; question?: string; site?: string } +type TaskView = { id: string; state: string; events: TaskEvent[]; answer?: string; error?: string } + +const sleep = (ms: number, signal?: AbortSignal) => + new Promise<void>((resolve) => { + const timer = setTimeout(resolve, ms) + signal?.addEventListener("abort", () => (clearTimeout(timer), resolve()), { once: true }) + }) + +/** + * Runs one action in the person's Chrome. A `fallback` result means it never reached the extension + * (the tool uses its own Chromium and says why); anything that went wrong after that throws. + */ +export async function act( + input: Input, + context: { readonly sessionID: string; readonly signal?: AbortSignal; readonly ask?: Ask }, +): Promise<Result> { + const result = await attempt(input, context) + if (result.kind === "fallback") session(context.sessionID).used = "chromium" + return result +} + +async function attempt( + input: Input, + context: { readonly sessionID: string; readonly signal?: AbortSignal; readonly ask?: Ask }, +): Promise<Result> { + const state = session(context.sessionID) + if (input.action !== "open" && state.used === "chromium") return { kind: "fallback", reason: "" } + const ask = context.ask ?? (await asker()) + if (!ask) return { kind: "fallback", reason: "no Redrob engine service is running on this computer" } + const text = task(input, state) + if (text === undefined) { + if (state.used !== "extension") return { kind: "fallback", reason: "" } + throw new Error( + "A screenshot of the person's own Chrome is not available through the Redrob extension. Use `read` for the page as text.", + ) + } + if (input.action === "wait" && !input.text && input.seconds) await sleep(Math.min(input.seconds, 60) * 1000, context.signal) + + // The same Code session goes on in the same conversation and tab, also after an engine restart. + const chat = chatId(context.sessionID) + if (!state.task) { + const found = await ask({ method: "GET", path: `/browser/task?chatId=${chat}` }, context.signal).catch(() => undefined) + const id = (found?.body as { id?: unknown } | undefined)?.id + if (found?.status === 200 && typeof id === "string") state.task = id + } + const created = await ask( + { + method: "POST", + path: "/browser/task", + body: { task: text, origin: "code", chatId: chat, ...(state.task ? { followUp: state.task } : {}) }, + }, + context.signal, + ) + const id = (created.body as { id?: unknown } | undefined)?.id + if (created.status !== 200 || typeof id !== "string") { + if (created.status === 0 || created.status === 503) { + return { kind: "fallback", reason: created.error ?? `the extension is not ready (${created.browser.state})` } + } + throw new Error(`The Redrob extension refused the task (HTTP ${created.status}).`) + } + state.used = "extension" + state.task = id + const browser = created.browser.name ?? "Chrome" + const view = await follow(ask, id, context.signal) + const answer = view.answer?.trim() ?? "" + if (view.state === "failed") throw new Error(`${browser}: ${view.error ?? "the task failed"}`) + if (view.state === "stopped" && !answer) throw new Error(`${browser}: the task was stopped`) + const asked = view.events.findLast((event) => event.kind === "approval_needed" && event.tool === "ask-input") + const body = answer || (asked?.question ? `The browser's agent asks: ${asked.question}` : "(no reading came back)") + const found = elements(body) + if (found.size > 0) state.elements = found + const url = + /^URL:\s*(\S+)/m.exec(body)?.[1] ?? view.events.findLast((event) => typeof event.url === "string")?.url ?? input.url ?? "" + return { kind: "extension", browser, url, text: body } +} + +/** Polls the task until it ends. A question for the person ends it too: the model reads it in the result. */ +async function follow(ask: Ask, id: string, signal?: AbortSignal): Promise<TaskView> { + const events: TaskEvent[] = [] + let after = -1 + const end = Date.now() + TASK_MS + const stop = () => ask({ method: "POST", path: `/browser/task/${id}/stop` }).catch(() => undefined) + while (true) { + if (signal?.aborted) { + await stop() + throw new Error("aborted") + } + const response = await ask({ method: "GET", path: `/browser/task/${id}?after=${after}` }, signal) + if (response.status !== 200) throw new Error(`The Redrob extension lost the task (HTTP ${response.status}).`) + const view = response.body as TaskView + for (const event of view.events ?? []) { + events.push(event) + after = Math.max(after, event.n) + } + const question = view.state === "waiting_approval" && events.at(-1)?.kind === "approval_needed" && events.at(-1)?.tool === "ask-input" + if (question) await stop() + if (TERMINAL.has(view.state) || question) return { ...view, events } + if (Date.now() > end) { + await stop() + throw new Error("The task in the person's Chrome took longer than 10 minutes and was stopped.") + } + await sleep(POLL_MS, signal) + } +} + +/** The daemon's relay link: in this process when it is the daemon, else over its local API. */ +export async function asker(env: NodeJS.ProcessEnv = process.env): Promise<Ask | undefined> { + const local = Channels.current()?.desk + if (local?.browser) return (request) => local.browser!(request) + const record = await Registry.read(env).catch(() => undefined) + if (!record || !Registry.alive(record.pid)) return undefined + return async (request, signal) => { + const response = await fetch(`${record.url}/desk/browser`, { + method: "POST", + headers: { ...Registry.headers(record.secret), "Content-Type": "application/json" }, + body: JSON.stringify(request), + // The first request may wait two minutes for Allow in Chrome. + signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(180_000)]) : AbortSignal.timeout(180_000), + }).catch((error: unknown) => ({ ok: false, status: 0, error }) as const) + // An older daemon has no such route (404), a server that is not the daemon has no desk (503). + if (!("json" in response) || !response.ok) { + return { status: 0, browser: { state: "off" }, error: `the engine service has no browser link (HTTP ${response.status})` } + } + return (await response.json()) as Channels.DeskBrowserResponse + } +} + +/** Whether the person's browser may be reachable at all, so the tool is offered without a Chromium. */ +export async function reachable(env: NodeJS.ProcessEnv = process.env): Promise<boolean> { + if (Channels.current()?.desk?.browser) return true + const record = await Registry.read(env).catch(() => undefined) + return !!record && Registry.alive(record.pid) +} + +/** The first line of every result: which browser the action ran in, and why not the person's own. */ +export function line(result: Result): string { + if (result.kind === "extension") return `Browser: the person's own Chrome (${result.browser}), through the Redrob extension.` + return `Browser: Redrob Code's own Chromium on this machine (headless).${result.reason ? ` The person's Chrome is not used: ${result.reason}.` : ""}` +} + +/** Forget a session's state (tests). */ +export function reset(): void { + sessions.clear() +} + +export * as UserBrowser from "./extension" diff --git a/packages/redrob/src/browser/tool.ts b/packages/redrob/src/browser/tool.ts index c8d287aa70..c17c0f68e7 100644 --- a/packages/redrob/src/browser/tool.ts +++ b/packages/redrob/src/browser/tool.ts @@ -18,6 +18,7 @@ import { Tools } from "@redrob-code/core/tool/tools" import DESCRIPTION from "../tool/browser.txt" import { Parameters } from "../tool/browser" import { RunnerBrowser } from "." +import { UserBrowser } from "./extension" export const name = "browser" @@ -46,9 +47,13 @@ const need = (value: string | undefined, message: string): string => { return value } -async function act(input: Input): Promise<Schema.Schema.Type<typeof Output>> { +async function act(input: Input, sessionID: string): Promise<Schema.Schema.Type<typeof Output>> { + if (input.action === "open" && !/^https?:\/\//i.test(input.url ?? "")) throw new Error("open takes an http:// or https:// URL.") + // The person's own Chrome through the Redrob extension when it is linked; else our Chromium. + const user = await UserBrowser.act(input, { sessionID }) + if (user.kind === "extension") return { url: user.url, text: `${UserBrowser.line(user)}\n${user.text}` } const handedBack = await RunnerBrowser.turn() - const lead = handedBack ? `${handedBack}\n\n` : "" + const lead = `${UserBrowser.line(user)}\n${handedBack ? `${handedBack}\n\n` : ""}` if (input.action === "screenshot") { const shot = await RunnerBrowser.screenshot() return { url: shot.url, text: `${lead}Screenshot of ${shot.url} attached.`, jpegBase64: shot.jpegBase64 } @@ -77,7 +82,8 @@ async function act(input: Input): Promise<Schema.Schema.Type<typeof Output>> { const layer = Layer.effectDiscard( Effect.gen(function* () { - if (!(yield* Effect.promise(() => RunnerBrowser.available()))) return + if (!(yield* Effect.promise(async () => (await RunnerBrowser.available()) || (await UserBrowser.reachable())))) + return const tools = yield* Tools.Service const permission = yield* PermissionV2.Service yield* tools @@ -103,7 +109,7 @@ const layer = Layer.effectDiscard( source: { type: "tool", messageID: context.assistantMessageID, callID: context.toolCallID }, }) return yield* Effect.tryPromise({ - try: () => act(input), + try: () => act(input, context.sessionID), catch: (error) => new ToolFailure({ message: error instanceof Error ? error.message : String(error) }), }) }).pipe( diff --git a/packages/redrob/src/cli/cmd/desk.ts b/packages/redrob/src/cli/cmd/desk.ts index 31031cdb29..0d29f71736 100644 --- a/packages/redrob/src/cli/cmd/desk.ts +++ b/packages/redrob/src/cli/cmd/desk.ts @@ -12,6 +12,7 @@ import { Daemon } from "@/daemon/connect" import * as Autostart from "@/daemon/autostart" import { endpoints } from "@/desk/auth" import { chatConnectApi, connect, isToken, thisMachine, type DeskLocal } from "@/desk/connect" +import { describe, type BrowserState } from "@/desk/extension" import { updateMachine } from "@/desk/settings" type Kind = "phone" | "web" @@ -27,6 +28,8 @@ type Status = { login?: Login pending: { code: string; name: string; kind: Kind; expiresAt: number }[] devices: { id: string; name: string; kind: Kind; addedAt: number; online: boolean }[] + /** Absent from an older engine. */ + browser?: BrowserState } type Device = { id: string; name: string; kind: Kind } @@ -82,6 +85,7 @@ function printStatus(status: Status): void { if (status.autoAll !== undefined) UI.println(`auto all from phone: ${status.autoAll ? "on" : "off"}`) if (status.pending.length > 0) UI.println(`${status.pending.length} device(s) waiting; run: redrob desk pending`) UI.println(`${status.devices.length} approved device(s)`) + if (status.browser) UI.println(`your Chrome (Redrob extension): ${describe(status.browser)}`) } /** A local call that must succeed; `call` already printed why it did not. */ diff --git a/packages/redrob/src/cli/cmd/serve.ts b/packages/redrob/src/cli/cmd/serve.ts index a395389eff..ded352e2a9 100644 --- a/packages/redrob/src/cli/cmd/serve.ts +++ b/packages/redrob/src/cli/cmd/serve.ts @@ -163,6 +163,7 @@ export const ServeCommand = effectCmd({ revoke: (id) => found(() => desk.revoke(id)), machine: () => desk.machine(), updateMachine: (update) => desk.updateMachine(update), + browser: (request) => desk.browser(request), }, }) const started = yield* Effect.promise(() => channels.startEnabledChannels(channelFactories).catch(() => [])) diff --git a/packages/redrob/src/desk/crypto.ts b/packages/redrob/src/desk/crypto.ts index c4e9fdc581..2077f2c93a 100644 --- a/packages/redrob/src/desk/crypto.ts +++ b/packages/redrob/src/desk/crypto.ts @@ -89,6 +89,16 @@ export async function pairingCode(desk: KeyPair, phonePub: Uint8Array): Promise< return String(value).padStart(6, "0") } +/** + * The same six digits, device side: ECDH(P, D) equals ECDH(D, P). The engine is the device when it + * calls the Redrob browser extension, which takes the desk role (extension.ts). + */ +export async function devicePairingCode(device: KeyPair, deskPub: Uint8Array): Promise<string> { + const bytes = await hkdf(await ecdh(device.privateKey, deskPub), concat(device.pub, deskPub), "redrob-desk-pair", 4) + const value = new DataView(ab(bytes)).getUint32(0) % 1_000_000 + return String(value).padStart(6, "0") +} + function nonce(counter: bigint): Uint8Array { const out = new Uint8Array(12) new DataView(out.buffer).setBigUint64(4, counter) @@ -132,6 +142,28 @@ export class Channel { return new Channel(deskToPhone, phoneToDesk, input.ownPeerId, input.phonePeerId) } + /** + * Device side of the same handshake, for the engine calling a desk peer (the browser extension): + * `Pe` is this connection's ephemeral key, `De_pub` came in `desk.accept`, `D_pub` in `desk.announce`. + * The device sends with bytes 0..31 and receives with 32..63. + */ + static async device(input: { + device: KeyPair + ephemeral: KeyPair + deskPub: Uint8Array + deskEph: Uint8Array + ownPeerId: string + deskPeerId: string + }): Promise<Channel> { + const ikm = concat(await ecdh(input.ephemeral.privateKey, input.deskEph), await ecdh(input.device.privateKey, input.deskPub)) + const salt = await sha256(concat(input.ephemeral.pub, input.deskEph)) + const okm = await hkdf(ikm, salt, "redrob-desk-v1", 64) + const key = (bytes: Uint8Array, usage: KeyUsage) => crypto.subtle.importKey("raw", ab(bytes), "AES-GCM", false, [usage]) + const deviceToDesk = await key(okm.subarray(0, 32), "encrypt") + const deskToDevice = await key(okm.subarray(32, 64), "decrypt") + return new Channel(deviceToDesk, deskToDevice, input.ownPeerId, input.deskPeerId) + } + async seal(plaintext: Uint8Array): Promise<Uint8Array> { const iv = nonce(this.sendCounter++) const sealed = await crypto.subtle.encrypt( diff --git a/packages/redrob/src/desk/extension.ts b/packages/redrob/src/desk/extension.ts new file mode 100644 index 0000000000..4181d3dc9f --- /dev/null +++ b/packages/redrob/src/desk/extension.ts @@ -0,0 +1,567 @@ +/** + * The engine as a caller of the Redrob browser extension (redrob-extension `docs/remote-control.md`). + * + * The extension already takes the desk role in room `desk:<user id>` and answers paired devices with + * the route set of its `src/remote/forward.ts`. This is the device side of that same handshake, as + * Redrob chat's web page does it (`apps/web/lib/desk/relay-client.ts`): one more socket in the room, + * `desk.hello` (kind "web") to the browser peer, the six-digit pairing the person allows once in + * Chrome, then `{t:"req"}` / `{t:"res"}` in sealed `desk.msg` frames. No new server, route or + * entry point on either side. The room is the account, so a browser found here is signed in to the + * same Redrob account as this engine. + * + * The daemon runs one of these beside its desk relay (service.ts). It only listens until the first + * request: `desk.hello`, and with it the pairing notification in Chrome, goes out when a `browser` + * tool call needs the browser. The device key and the browser it paired with are kept in the desk + * secret scope (`browserKey`, `browserDesk`). + */ +import os from "node:os" +import type { RelayTicket } from "./auth" +import * as DeskCrypto from "./crypto" +import { decodeAction, decodeControl, encodeAction, encodeControl } from "./frames" + +const encoder = new TextEncoder() +const decoder = new TextDecoder() + +/** After connecting, how long to wait for a browser to announce itself before saying there is none. */ +export const ANNOUNCE_WAIT_MS = 4_000 +/** How long a request waits for the person to press Allow in Chrome the first time. */ +export const PAIR_WAIT_MS = 2 * 60_000 +const HANDSHAKE_WAIT_MS = 15_000 +const REQUEST_TIMEOUT_MS = 30_000 + +export type BrowserState = + /** No relay connection (signed out, desk off, or not connected yet). */ + | { readonly state: "off" } + /** Connected; no browser with the Redrob extension is in the room. */ + | { readonly state: "none" } + /** A browser is in the room; this engine has not asked it yet. */ + | { readonly state: "found"; readonly name: string } + /** Asked; the person has to press Allow in that browser (the notification shows this code). */ + | { readonly state: "pairing"; readonly name: string; readonly code: string } + | { readonly state: "ready"; readonly name: string } + | { readonly state: "denied"; readonly name: string; readonly reason: string } + +export type BrowserResponse = { readonly status: number; readonly body?: unknown } + +/** No browser to talk to; `state` says why. The tool falls back to its own Chromium. */ +export class BrowserUnavailable extends Error { + constructor(readonly state: BrowserState) { + super(describe(state)) + } +} + +export function describe(state: BrowserState): string { + switch (state.state) { + case "off": + return "not connected to the Redrob relay (sign in with `redrob desk login`, or open Redrob Desktop)" + case "none": + return "no Chrome with the Redrob extension is connected to this account (extension Settings → Redrob chat → Connect)" + case "found": + return `${state.name} is online` + case "pairing": + return `waiting for Allow in ${state.name} (code ${state.code})` + case "ready": + return `connected to ${state.name}` + case "denied": + return `${state.name} refused this engine (${state.reason}); allow it again from the extension's Settings` + } +} + +/** The relay socket as this module uses it; a test hands in a fake. */ +export type Socket = { + readonly readyState: number + binaryType: string + send(data: string | Uint8Array): void + close(code?: number, reason?: string): void + addEventListener(type: string, listener: (event: any) => void): void +} + +export type Secrets = { + get(key: string): Promise<string | undefined> + set(key: string, value: string): Promise<void> +} + +type Browser = { + readonly id: string + readonly pub: Uint8Array + name: string + peerId?: string + ephemeral?: DeskCrypto.KeyPair + channel?: DeskCrypto.Channel + link: "idle" | "handshake" | "pending" | "ready" | "denied" + code?: string + reason?: string + /** A request already waited out the Allow for the current pairing request. */ + waited?: boolean + sending: Promise<void> +} + +type Waiter = { resolve: (response: BrowserResponse) => void; reject: (error: Error) => void; browser: string } + +const OPEN = 1 + +export class ExtensionLink { + private socket?: Socket + private ticket?: RelayTicket + private stopped = true + private delay = 1_000 + private reconnect?: ReturnType<typeof setTimeout> + private connectedAt = 0 + private readonly browsers = new Map<string, Browser>() + private readonly peers = new Map<string, string>() + private readonly waiting = new Map<number, Waiter>() + private nextId = 1 + private inbound: Promise<void> = Promise.resolve() + private changed: Array<() => void> = [] + private device?: Promise<DeskCrypto.KeyPair> + private chosen?: string + + constructor( + private readonly chat: { relayTicket(): Promise<RelayTicket> }, + private readonly secrets: Secrets, + private readonly options: { + /** How Chrome lists this engine in the extension's Settings. */ + readonly name?: string + readonly socket?: (url: string) => Socket + readonly announceWaitMs?: number + readonly log?: (message: string) => void + } = {}, + ) {} + + start(): void { + if (!this.stopped) return + this.stopped = false + void this.secrets + .get("browserDesk") + .then((id) => (this.chosen ??= id)) + .catch(() => undefined) + void this.connect() + } + + stop(): void { + this.stopped = true + if (this.reconnect) clearTimeout(this.reconnect) + const socket = this.socket + this.socket = undefined + socket?.close(1000, "stop") + this.dropAll(new Error("the relay link stopped")) + } + + /** Where this engine stands with the person's browser, for `GET /desk/status`. */ + status(): BrowserState { + if (!this.socket || this.socket.readyState !== OPEN) return { state: "off" } + const browser = this.pick() + if (!browser) return { state: "none" } + switch (browser.link) { + case "ready": + return { state: "ready", name: browser.name } + case "pending": + return { state: "pairing", name: browser.name, code: browser.code ?? "" } + case "denied": + return { state: "denied", name: browser.name, reason: browser.reason ?? "denied" } + default: + return { state: "found", name: browser.name } + } + } + + /** + * One request of the extension's route set. Says hello first when needed and waits (up to + * `pairMs`) for the person to allow this engine. Throws `BrowserUnavailable` when there is no + * browser to ask; the response's status is the extension's own (403 outside its allowlist). + */ + async request( + method: "GET" | "POST", + path: string, + body?: unknown, + options: { readonly pairMs?: number; readonly timeoutMs?: number; readonly signal?: AbortSignal } = {}, + ): Promise<BrowserResponse> { + const browser = await this.ready(options.pairMs ?? PAIR_WAIT_MS, options.signal) + const id = this.nextId++ + return new Promise<BrowserResponse>((resolve, reject) => { + const timer = setTimeout(() => { + this.waiting.delete(id) + reject(new Error(`${browser.name} did not answer ${method} ${path}`)) + }, options.timeoutMs ?? REQUEST_TIMEOUT_MS) + const done = () => clearTimeout(timer) + this.waiting.set(id, { + browser: browser.id, + resolve: (response) => (done(), resolve(response)), + reject: (error) => (done(), reject(error)), + }) + this.seal(browser, { t: "req", id, method, path, ...(body === undefined ? {} : { body }) }) + }) + } + + // -- linking ----------------------------------------------------------------------------------- + + private async ready(pairMs: number, signal?: AbortSignal): Promise<Browser> { + if (this.stopped) throw new BrowserUnavailable({ state: "off" }) + // Connected, and either a browser announced itself or it had time to. + await this.until( + () => this.pick()?.peerId !== undefined || this.announceWaited(), + HANDSHAKE_WAIT_MS + (this.options.announceWaitMs ?? ANNOUNCE_WAIT_MS), + signal, + ) + let browser = this.pick() + if (!browser?.peerId) throw new BrowserUnavailable(this.status()) + if (browser.link === "denied") throw new BrowserUnavailable(this.status()) + // The person is asked to Allow once per request in Chrome: a request after one that already + // waited out the Allow does not wait again (the Allow still links this engine when it comes). + const wait = browser.waited ? 0 : pairMs + const deadline = Date.now() + wait + if (browser.link === "idle") await this.hello(browser) + await this.until( + () => { + browser = this.pick() + if (!browser?.peerId) return true + if (browser.link === "ready" || browser.link === "denied") return true + // Still in the handshake: give it a moment. Pending: wait for the person up to the deadline. + return browser.link === "pending" && Date.now() >= deadline + }, + Math.max(wait, HANDSHAKE_WAIT_MS), + signal, + ) + browser = this.pick() + if (browser?.link === "pending") browser.waited = true + if (!browser?.peerId || browser.link !== "ready") throw new BrowserUnavailable(this.status()) + return browser + } + + private announceWaited(): boolean { + return this.connectedAt > 0 && Date.now() - this.connectedAt >= (this.options.announceWaitMs ?? ANNOUNCE_WAIT_MS) + } + + /** Resolves when `check` is true, re-checked on every change and every 250 ms, or after `ms`. */ + private until(check: () => boolean, ms: number, signal?: AbortSignal): Promise<void> { + if (check()) return Promise.resolve() + return new Promise<void>((resolve, reject) => { + const end = Date.now() + ms + const finish = () => { + clearInterval(tick) + this.changed = this.changed.filter((listener) => listener !== test) + signal?.removeEventListener("abort", aborted) + } + const test = () => { + if (check() || Date.now() >= end) { + finish() + resolve() + } + } + const aborted = () => { + finish() + reject(new Error("aborted")) + } + const tick = setInterval(test, 250) + tick.unref?.() + this.changed.push(test) + if (signal?.aborted) return aborted() + signal?.addEventListener("abort", aborted, { once: true }) + }) + } + + private notify(): void { + for (const listener of [...this.changed]) listener() + } + + /** The browser to talk to: the one paired before when it is online, else the first online one. */ + private pick(): Browser | undefined { + const online = [...this.browsers.values()].filter((browser) => browser.peerId) + return ( + online.find((browser) => browser.id === this.chosen) ?? + online.find((browser) => browser.link === "ready") ?? + online[0] ?? + (this.chosen ? this.browsers.get(this.chosen) : undefined) + ) + } + + private key(): Promise<DeskCrypto.KeyPair> { + if (!this.device) { + this.device = (async () => { + const stored = await this.secrets.get("browserKey") + if (stored) return DeskCrypto.importPrivate(stored) + const fresh = await DeskCrypto.generate() + await this.secrets.set("browserKey", await DeskCrypto.exportPrivate(fresh)) + return fresh + })() + this.device.catch(() => { + this.device = undefined + }) + } + return this.device + } + + private async hello(browser: Browser): Promise<void> { + if (!browser.peerId) return + const [device, ephemeral] = await Promise.all([this.key(), DeskCrypto.generate()]) + browser.ephemeral = ephemeral + browser.channel = undefined + browser.link = "handshake" + this.notify() + this.sendJson( + "desk.hello", + { + v: 1, + phone: DeskCrypto.b64url(device.pub), + eph: DeskCrypto.b64url(ephemeral.pub), + name: this.options.name ?? `Redrob Code on ${os.hostname()}`, + kind: "web", + }, + browser.peerId, + ) + } + + // -- socket ------------------------------------------------------------------------------------ + + private schedule(): void { + if (this.stopped) return + const delay = this.delay + this.delay = Math.min(this.delay * 2, 5 * 60_000) + this.reconnect = setTimeout(() => void this.connect(), delay) + this.reconnect.unref?.() + } + + private async connect(): Promise<void> { + if (this.stopped || this.socket) return + let ticket: RelayTicket + try { + ticket = await this.chat.relayTicket() + } catch (error) { + this.options.log?.(`browser link: no relay ticket: ${error instanceof Error ? error.message : String(error)}`) + return this.schedule() + } + if (this.stopped || this.socket) return + const url = new URL(ticket.url) + url.searchParams.set("ticket", ticket.ticket) + const socket = this.options.socket + ? this.options.socket(url.toString()) + : (new WebSocket(url.toString()) as unknown as Socket) + socket.binaryType = "arraybuffer" + this.socket = socket + this.ticket = ticket + let openedAt = 0 + socket.addEventListener("open", () => { + openedAt = Date.now() + this.connectedAt = openedAt + this.delay = 1_000 + // Every desk peer in the room answers `hello` with `desk.announce`. + socket.send(encodeControl("hello", ticket.peerId)) + this.notify() + }) + socket.addEventListener("message", (event: { data: string | ArrayBuffer }) => { + const data = event.data + this.inbound = this.inbound.then(() => this.onFrame(socket, data)).catch(() => undefined) + }) + socket.addEventListener("close", () => { + if (this.socket !== socket) return + this.socket = undefined + this.connectedAt = 0 + this.dropAll(new Error("the relay connection closed")) + if (this.stopped) return + if (openedAt && Date.now() - openedAt > 60_000) this.delay = 1_000 + this.schedule() + }) + socket.addEventListener("error", () => { + // `close` follows and reconnects. + }) + } + + private sendJson(namespace: string, value: unknown, targetId?: string): void { + this.sendRaw(namespace, encoder.encode(JSON.stringify(value)), targetId) + } + + private sendRaw(namespace: string, payload: Uint8Array, targetId?: string): void { + const socket = this.socket + const ticket = this.ticket + if (!socket || !ticket || socket.readyState !== OPEN) return + socket.send(encodeAction({ senderId: ticket.peerId, ...(targetId ? { targetId } : {}), namespace }, payload)) + } + + private seal(browser: Browser, message: Record<string, unknown>): void { + const channel = browser.channel + const target = browser.peerId + if (!channel || !target) return + const bytes = encoder.encode(JSON.stringify(message)) + browser.sending = browser.sending + .then(async () => { + if (browser.channel !== channel) return + this.sendRaw("desk.msg", await channel.seal(bytes), target) + }) + .catch(() => undefined) + } + + // -- frames ------------------------------------------------------------------------------------ + + private async onFrame(socket: Socket, data: string | ArrayBuffer): Promise<void> { + if (socket !== this.socket) return + const own = this.ticket?.peerId + if (typeof data === "string") { + const control = decodeControl(data) + if (control?.type === "leave" && control.senderId !== own) this.leave(control.senderId) + return + } + const frame = decodeAction(new Uint8Array(data)) + if (!frame || frame.senderId === own) return + if (frame.targetId && frame.targetId !== own) return + const text = () => { + try { + return JSON.parse(decoder.decode(frame.payload)) as Record<string, unknown> + } catch { + return undefined + } + } + switch (frame.namespace) { + case "desk.announce": + return this.onAnnounce(frame.senderId, text()) + case "desk.pending": + return this.onPending(frame.senderId) + case "desk.accept": + return this.onAccept(frame.senderId, text()) + case "desk.denied": + return this.onDenied(frame.senderId, text()) + case "desk.msg": + return this.onMessage(frame.senderId, frame.payload) + } + } + + private browserOf(peerId: string): Browser | undefined { + const id = this.peers.get(peerId) + return id ? this.browsers.get(id) : undefined + } + + private async onAnnounce(peerId: string, announce: Record<string, unknown> | undefined): Promise<void> { + // Only the extension: a PC running Redrob Code announces without `kind`. + if (!announce || announce.v !== 1 || announce.kind !== "browser") return + if (typeof announce.desk !== "string" || typeof announce.deskId !== "string") return + const pub = DeskCrypto.fromB64url(announce.desk) + if (pub.byteLength !== 65 || (await DeskCrypto.keyId(pub)) !== announce.deskId) return + const name = typeof announce.name === "string" && announce.name.trim() ? announce.name.slice(0, 100) : "Chrome" + let browser = this.browsers.get(announce.deskId) + if (browser && DeskCrypto.b64url(browser.pub) !== announce.desk) return + if (!browser) { + browser = { id: announce.deskId, pub, name, link: "idle", sending: Promise.resolve() } + this.browsers.set(browser.id, browser) + } + browser.name = name + const samePeer = browser.peerId === peerId + if (browser.peerId && !samePeer) this.peers.delete(browser.peerId) + browser.peerId = peerId + this.peers.set(peerId, browser.id) + // A new connection of that browser (its worker restarted): the old session is gone. Say hello + // again only when this engine had already been talking to it. + if (!samePeer && browser.link !== "idle" && browser.link !== "denied") { + this.fail(browser.id, new Error(`${browser.name} reconnected`)) + browser.channel = undefined + browser.link = "idle" + await this.hello(browser) + } + this.notify() + } + + private async onPending(peerId: string): Promise<void> { + const browser = this.browserOf(peerId) + if (!browser || browser.link !== "handshake") return + browser.code = await DeskCrypto.devicePairingCode(await this.key(), browser.pub) + browser.link = "pending" + this.options.log?.(`browser link: allow "Redrob Code" in ${browser.name} (code ${browser.code})`) + this.notify() + } + + private async onAccept(peerId: string, accept: Record<string, unknown> | undefined): Promise<void> { + const browser = this.browserOf(peerId) + const ephemeral = browser?.ephemeral + if (!browser || !ephemeral || !this.ticket || typeof accept?.eph !== "string") return + const deskEph = DeskCrypto.fromB64url(accept.eph) + try { + browser.channel = await DeskCrypto.Channel.device({ + device: await this.key(), + ephemeral, + deskPub: browser.pub, + deskEph, + ownPeerId: this.ticket.peerId, + deskPeerId: peerId, + }) + } catch { + return + } + browser.link = "ready" + browser.code = undefined + browser.waited = false + if (this.chosen !== browser.id) { + this.chosen = browser.id + await this.secrets.set("browserDesk", browser.id).catch(() => undefined) + } + this.notify() + } + + private onDenied(peerId: string, denied: Record<string, unknown> | undefined): void { + const browser = this.browserOf(peerId) + if (!browser) return + browser.channel = undefined + const reason = typeof denied?.reason === "string" ? denied.reason : "denied" + // An Allow nobody pressed in time: the next request asks again. A Deny or a removal sticks + // until this link restarts, so Chrome is not asked over and over. + browser.link = reason === "expired" ? "idle" : "denied" + browser.reason = reason + browser.waited = false + this.fail(browser.id, new BrowserUnavailable(this.status())) + this.notify() + } + + private async onMessage(peerId: string, payload: Uint8Array): Promise<void> { + const browser = this.browserOf(peerId) + const channel = browser?.channel + if (!browser || !channel) return + let plain: Uint8Array | undefined + try { + plain = await channel.open(payload) + } catch { + // A bad tag ends the session: start a fresh handshake. + this.fail(browser.id, new Error(`the session with ${browser.name} broke`)) + await this.hello(browser) + return + } + if (!plain) return + let message: { t?: unknown; id?: unknown; status?: unknown; body?: unknown } + try { + message = JSON.parse(decoder.decode(plain)) + } catch { + return + } + if (message.t !== "res" || typeof message.id !== "number" || typeof message.status !== "number") return + const waiter = this.waiting.get(message.id) + if (!waiter || waiter.browser !== browser.id) return + this.waiting.delete(message.id) + waiter.resolve({ status: message.status, ...(message.body === undefined ? {} : { body: message.body }) }) + } + + private leave(peerId: string): void { + const browser = this.browserOf(peerId) + this.peers.delete(peerId) + if (!browser) return + browser.peerId = undefined + browser.channel = undefined + if (browser.link !== "denied") browser.link = "idle" + this.fail(browser.id, new Error(`${browser.name} went offline`)) + this.notify() + } + + private fail(browserId: string, error: Error): void { + for (const [id, waiter] of this.waiting) { + if (waiter.browser !== browserId) continue + this.waiting.delete(id) + waiter.reject(error) + } + } + + private dropAll(error: Error): void { + for (const browser of this.browsers.values()) { + browser.peerId = undefined + browser.channel = undefined + if (browser.link !== "denied") browser.link = "idle" + } + this.peers.clear() + for (const waiter of this.waiting.values()) waiter.reject(error) + this.waiting.clear() + this.notify() + } +} diff --git a/packages/redrob/src/desk/relay.ts b/packages/redrob/src/desk/relay.ts index d8822cd15f..0ddaedb07b 100644 --- a/packages/redrob/src/desk/relay.ts +++ b/packages/redrob/src/desk/relay.ts @@ -65,7 +65,8 @@ export type RelayStatus = { export class DeskRelay { private readonly api: DaemonApi - private readonly chat: ChatApi + /** Also the browser link's (extension.ts), so both share one token refresh. */ + readonly chat: ChatApi private readonly forwarder: Forwarder private readonly tools: ToolCatalog /** PC schedules: pulls this machine's due firings and runs them (hive.ts). */ diff --git a/packages/redrob/src/desk/service.ts b/packages/redrob/src/desk/service.ts index 59e42a9e68..82bcdbdcf5 100644 --- a/packages/redrob/src/desk/service.ts +++ b/packages/redrob/src/desk/service.ts @@ -9,6 +9,7 @@ */ import { Config, type DaemonConnection } from "@redrob-code/channels-core" import { AppToken, endpoints, finishDeviceLogin, startDeviceLogin, type DeviceAuthorization } from "./auth" +import { BrowserUnavailable, ExtensionLink, type BrowserResponse, type BrowserState } from "./extension" import { NotFound, Pairing, type DeviceKind } from "./pairing" import { DeskRelay, consoleLogger, type Logger } from "./relay" import { allowAutoAll, machine, updateMachine, type Machine, type MachineUpdate } from "./settings" @@ -33,14 +34,30 @@ export type DeskStatus = { readonly login?: DeskLogin readonly pending: readonly { code: string; name: string; kind: DeviceKind; expiresAt: number }[] readonly devices: readonly { id: string; name: string; kind: DeviceKind; addedAt: number; online: boolean }[] + /** The person's Chrome with the Redrob extension, as the `browser` tool reaches it (extension.ts). */ + readonly browser: BrowserState } +/** `POST /desk/browser`: one request of the extension's route set, and what came back. */ +export type DeskBrowserRequest = { + readonly method: "GET" | "POST" + readonly path: string + readonly body?: unknown + /** How long to wait for Allow in Chrome the first time (default two minutes). */ + readonly pairMs?: number +} + +/** `status` 0: the request did not reach the extension; `error` says why and `browser` where it stands. */ +export type DeskBrowserResponse = BrowserResponse & { readonly browser: BrowserState; readonly error?: string } + /** The device grant is already running (or this machine is already signed in). */ export class AlreadySignedIn extends Error {} export class DeskService { readonly pairing: Pairing private relay?: DeskRelay + /** Runs beside the relay: this engine calling the person's browser extension. */ + private extension?: ExtensionLink private login?: { readonly device: DeviceAuthorization; readonly abort: AbortController } private reconciling: Promise<void> = Promise.resolve() /** Redrob Desktop's access token (`PUT /desk/token`). Memory only, never logged. */ @@ -87,11 +104,15 @@ export class DeskService { if (wanted && !this.relay) { this.relay = new DeskRelay(this.daemon, this.pairing, this.log, undefined, this.app) this.relay.start() + this.extension = new ExtensionLink(this.relay.chat, this.tokens(), { log: (m) => this.log.info(m) }) + this.extension.start() this.log.info("relay on") this.onChange() } else if (!wanted && this.relay) { this.relay.stop() this.relay = undefined + this.extension?.stop() + this.extension = undefined this.log.info("relay off") this.onChange() } @@ -107,6 +128,25 @@ export class DeskService { this.login = undefined this.relay?.stop() this.relay = undefined + this.extension?.stop() + this.extension = undefined + } + + /** + * `POST /desk/browser`: a request to the person's browser extension, for the `browser` tool of + * any engine on this machine (browser/extension.ts). A request that did not reach the extension + * comes back as status 0 with the reason, so the tool can fall back to its own Chromium. + */ + async browser(request: DeskBrowserRequest): Promise<DeskBrowserResponse> { + const link = this.extension + if (!link) return { status: 0, browser: { state: "off" } } + try { + const response = await link.request(request.method, request.path, request.body, { pairMs: request.pairMs }) + return { ...response, browser: link.status() } + } catch (error) { + const state = error instanceof BrowserUnavailable ? error.state : link.status() + return { status: 0, browser: state, error: error instanceof Error ? error.message : String(error) } + } } async status(): Promise<DeskStatus> { @@ -137,6 +177,7 @@ export class DeskService { addedAt, online: online.has(id), })), + browser: this.extension?.status() ?? { state: "off" }, } } diff --git a/packages/redrob/src/tool/browser.ts b/packages/redrob/src/tool/browser.ts index fb5772da44..8656b58d92 100644 --- a/packages/redrob/src/tool/browser.ts +++ b/packages/redrob/src/tool/browser.ts @@ -2,6 +2,7 @@ import { Effect, Schema } from "effect" import * as Tool from "./tool" import DESCRIPTION from "./browser.txt" import { RunnerBrowser } from "@/browser" +import { UserBrowser } from "@/browser/extension" export const Parameters = Schema.Struct({ action: Schema.Literals(["open", "read", "click", "type", "scroll", "wait", "screenshot"]).annotate({ @@ -20,7 +21,7 @@ export const Parameters = Schema.Struct({ type Params = Schema.Schema.Type<typeof Parameters> -type Metadata = { action: Params["action"]; url?: string; handedBack?: string } +type Metadata = { action: Params["action"]; url?: string; handedBack?: string; browser?: "extension" | "chromium" } const need = <A>(value: A | undefined, message: string): A => { if (value === undefined || value === "") throw new Error(message) @@ -64,9 +65,22 @@ export const BrowserTool = Tool.define<typeof Parameters, Metadata, never>( always: ["*"], metadata: { url: params.url }, }) + if (params.action === "open" && !/^https?:\/\//i.test(params.url ?? "")) + throw new Error("open takes an http:// or https:// URL.") + // The person's own Chrome through the Redrob extension when it is linked; else our Chromium. + const user = yield* Effect.promise(() => + UserBrowser.act(params, { sessionID: ctx.sessionID, signal: ctx.abort }), + ) + if (user.kind === "extension") + return { + title: `${params.action} ${user.url} (your Chrome)`, + output: `${UserBrowser.line(user)}\n${user.text}`, + metadata: { action: params.action, url: user.url, browser: "extension" as const }, + } + const browserLine = UserBrowser.line(user) const handedBack = yield* Effect.promise(() => RunnerBrowser.turn(ctx.abort)) - const lead = handedBack ? `${handedBack}\n\n` : "" - const meta = { action: params.action, ...(handedBack ? { handedBack } : {}) } + const lead = `${browserLine}\n${handedBack ? `${handedBack}\n\n` : ""}` + const meta = { action: params.action, browser: "chromium" as const, ...(handedBack ? { handedBack } : {}) } if (params.action === "screenshot") { const shot = yield* Effect.promise(() => RunnerBrowser.screenshot()) diff --git a/packages/redrob/src/tool/browser.txt b/packages/redrob/src/tool/browser.txt index 92c79edda3..ae407d7889 100644 --- a/packages/redrob/src/tool/browser.txt +++ b/packages/redrob/src/tool/browser.txt @@ -1,14 +1,14 @@ -Drive a real Chromium browser on this machine (headless, 1280x800). Use it to check a local dev server, click through a web app, or read a page that needs JavaScript. `localhost` here is this machine. +Drive a web browser. When the person's own Chrome has the Redrob extension and is linked to this engine, actions run there, with their sign-ins; otherwise in a headless Chromium on this machine (1280x800), where `localhost` is this machine. The first line of every result says which browser was used. Actions: - open: go to `url` (http or https). -- read: the page as a numbered accessibility tree. Each element has a number like [ref=e5]. +- read: the page as text with numbered elements like [ref=e5] or [e5]. - click: click element `ref`. - type: replace the text in field `ref` with `text`; `submit: true` presses Enter after. - scroll: scroll `direction` (down/up) by `amount` pixels (default 600), or bring element `ref` into view. - wait: wait until `text` is visible, or for `seconds` (at most 60). -- screenshot: a picture of the visible page. +- screenshot: a picture of the visible page (headless Chromium only; in the person's Chrome use read). Every action except screenshot returns a fresh reading. Element numbers belong to the reading they came from: after the page changes (navigation, a new tab, the user taking over), read again before using a number. -The user may watch the browser and take it over. While they hold it, your next browser action waits until they hand it back (up to 10 minutes), and the result starts with a line saying what they did. Password fields are hidden in pictures. +The user may watch the browser and take it over. While they hold it, your next browser action waits until they hand it back (up to 10 minutes), and the result starts with a line saying what they did. Password fields are hidden in pictures. In the person's Chrome, sign-ins, payments and anything their approval setting asks about are their step there: the action waits for them. diff --git a/packages/redrob/src/tool/registry.ts b/packages/redrob/src/tool/registry.ts index a35ce9f8a3..6dc09fbc45 100644 --- a/packages/redrob/src/tool/registry.ts +++ b/packages/redrob/src/tool/registry.ts @@ -15,6 +15,7 @@ import { TodoWriteTool } from "./todo" import { RoutineTool } from "./routine" import { BrowserTool } from "./browser" import { RunnerBrowser } from "@/browser" +import { UserBrowser } from "@/browser/extension" import { MemoryTool } from "./memory" import { WebFetchTool } from "./webfetch" import { WriteTool } from "./write" @@ -218,7 +219,10 @@ const layer = Layer.effect( } yield* config.get() - const hasBrowser = yield* Effect.promise(() => RunnerBrowser.available()) + // Our own Chromium, or the person's Chrome through the Redrob extension (browser/extension.ts). + const hasBrowser = yield* Effect.promise( + async () => (await RunnerBrowser.available()) || (await UserBrowser.reachable()), + ) const questionEnabled = ["app", "cli", "desktop"].includes(flags.client) || flags.enableQuestionTool const tool = yield* Effect.all({ @@ -261,7 +265,7 @@ const layer = Layer.effect( tool.fetch, tool.todo, tool.routine, - // Only where a Chromium is installed (RunnerBrowser.executable). + // Where a Chromium is installed (RunnerBrowser.executable) or the engine service may reach the person's Chrome. ...(hasBrowser ? [tool.browser] : []), tool.memory, tool.search, diff --git a/packages/redrob/test/browser/extension.test.ts b/packages/redrob/test/browser/extension.test.ts new file mode 100644 index 0000000000..b9ba9f2510 --- /dev/null +++ b/packages/redrob/test/browser/extension.test.ts @@ -0,0 +1,140 @@ +import { afterEach, describe, expect, test } from "bun:test" +import { UserBrowser } from "../../src/browser/extension" +import type { DeskBrowserRequest, DeskBrowserResponse } from "@redrob-code/server/channels" + +const READ = [ + "URL: https://example.com/inbox", + "Title: Inbox", + "", + "Three new messages.", + "", + "Elements:", + "[e1] link «Home»", + "[e2] button «Send»", + "[e3] textbox «Search»", +].join("\n") + +const ready = { state: "ready" as const, name: "Chrome on Windows" } + +/** A fake extension behind the daemon's link: tasks end after one poll unless `script` says otherwise. */ +function fake(script: { states?: string[]; answer?: string; events?: unknown[]; create?: DeskBrowserResponse } = {}) { + const calls: DeskBrowserRequest[] = [] + let n = 0 + let polls = 0 + const ask = async (request: DeskBrowserRequest): Promise<DeskBrowserResponse> => { + calls.push(request) + if (request.method === "GET" && request.path.startsWith("/browser/task?chatId=")) return { status: 404, browser: ready } + if (request.method === "POST" && request.path === "/browser/task") return script.create ?? { status: 200, body: { id: `t${++n}` }, browser: ready } + if (request.method === "POST" && request.path.endsWith("/stop")) return { status: 200, body: { ok: true }, browser: ready } + const states = script.states ?? ["done"] + const state = states[Math.min(polls++, states.length - 1)]! + return { + status: 200, + body: { id: `t${n}`, state, events: script.events ?? [], ...(state === "done" ? { answer: script.answer ?? READ } : {}) }, + browser: ready, + } + } + return { ask, calls } +} + +afterEach(() => UserBrowser.reset()) + +describe("UserBrowser.act", () => { + test("open becomes one task for the extension, from Redrob Code, keyed to the session", async () => { + const { ask, calls } = fake() + const result = await UserBrowser.act({ action: "open", url: "https://example.com" }, { sessionID: "ses_1", ask }) + expect(result).toEqual({ kind: "extension", browser: "Chrome on Windows", url: "https://example.com/inbox", text: READ }) + const create = calls.find((call) => call.method === "POST" && call.path === "/browser/task")! + const body = create.body as { task: string; origin: string; chatId: string; followUp?: string } + expect(body.origin).toBe("code") + expect(body.chatId).toBe("code-ses_1") + expect(body.followUp).toBeUndefined() + expect(body.task.startsWith("Open https://example.com in this tab.")).toBe(true) + expect(body.task).toContain("Elements:") + expect(UserBrowser.line(result)).toBe( + "Browser: the person's own Chrome (Chrome on Windows), through the Redrob extension.", + ) + }) + + test("the next action follows up in the same task and names the element from the last reading", async () => { + const { ask, calls } = fake() + await UserBrowser.act({ action: "open", url: "https://example.com" }, { sessionID: "ses_1", ask }) + await UserBrowser.act({ action: "click", ref: "e2" }, { sessionID: "ses_1", ask }) + const creates = calls.filter((call) => call.method === "POST" && call.path === "/browser/task") + const second = creates[1]!.body as { task: string; followUp?: string } + expect(second.followUp).toBe("t1") + expect(second.task.startsWith("Click [e2] button «Send» from your last reading, once.")).toBe(true) + + await UserBrowser.act({ action: "type", ref: "e3", text: "hello", submit: true }, { sessionID: "ses_1", ask }) + const third = calls.filter((call) => call.method === "POST" && call.path === "/browser/task")[2]!.body as { + task: string + followUp?: string + } + expect(third.followUp).toBe("t2") + expect(third.task).toContain("into the field [e3] textbox «Search»") + expect(third.task).toContain("«hello»") + expect(third.task).toContain("Then press Enter once.") + }) + + test("a task found for this chat after a restart is followed up", async () => { + const { ask: base, calls } = fake() + const ask = async (request: DeskBrowserRequest) => + request.path.startsWith("/browser/task?chatId=") + ? { status: 200, body: { id: "old", state: "done", task: "x", origin: "code", last: 3 }, browser: ready } + : base(request) + await UserBrowser.act({ action: "read" }, { sessionID: "ses_9", ask }) + const create = calls.find((call) => call.path === "/browser/task")!.body as { followUp?: string } + expect(create.followUp).toBe("old") + }) + + test("not reaching the extension falls back with the reason, and the session stays on Chromium", async () => { + const { ask, calls } = fake({ + create: { status: 0, browser: { state: "none" }, error: "no Chrome with the Redrob extension is connected" }, + }) + const result = await UserBrowser.act({ action: "open", url: "https://example.com" }, { sessionID: "ses_2", ask }) + expect(result).toEqual({ kind: "fallback", reason: "no Chrome with the Redrob extension is connected" }) + expect(UserBrowser.line(result)).toBe( + "Browser: Redrob Code's own Chromium on this machine (headless). The person's Chrome is not used: no Chrome with the Redrob extension is connected.", + ) + const before = calls.length + expect(await UserBrowser.act({ action: "click", ref: "e1" }, { sessionID: "ses_2", ask })).toEqual({ + kind: "fallback", + reason: "", + }) + expect(calls.length).toBe(before) + }) + + test("waits through the person's approval in Chrome", async () => { + const { ask, calls } = fake({ states: ["running", "waiting_approval", "running", "done"] }) + const result = await UserBrowser.act({ action: "click", ref: "e2" }, { sessionID: "ses_4", ask }) + expect(result.kind).toBe("extension") + expect(calls.filter((call) => call.method === "GET" && call.path.startsWith("/browser/task/")).length).toBe(4) + }) + + test("a question from the browser's agent stops the task and comes back as the result", async () => { + const { ask, calls } = fake({ + states: ["waiting_approval"], + events: [{ n: 0, at: 0, kind: "approval_needed", tool: "ask-input", question: "Which account?" }], + }) + const result = await UserBrowser.act({ action: "read" }, { sessionID: "ses_5", ask }) + expect(result).toMatchObject({ kind: "extension", text: "The browser's agent asks: Which account?" }) + expect(calls.some((call) => call.path === "/browser/task/t1/stop")).toBe(true) + }) + + test("a failed task is an error, not a fallback", async () => { + const { ask } = fake({ states: ["failed"] }) + await expect(UserBrowser.act({ action: "read" }, { sessionID: "ses_6", ask })).rejects.toThrow("Chrome on Windows") + }) + + test("screenshot in the person's Chrome says to read instead; before any use it falls back", async () => { + const { ask } = fake() + expect(await UserBrowser.act({ action: "screenshot" }, { sessionID: "ses_7", ask })).toEqual({ kind: "fallback", reason: "" }) + await UserBrowser.act({ action: "open", url: "https://example.com" }, { sessionID: "ses_8", ask }) + await expect(UserBrowser.act({ action: "screenshot" }, { sessionID: "ses_8", ask })).rejects.toThrow("Use `read`") + }) +}) + +test("chat ids keep the extension's id shape", () => { + expect(UserBrowser.chatId("ses_abc.1")).toBe("code-ses_abc_1") + expect(/^[A-Za-z0-9_-]{1,128}$/.test(UserBrowser.chatId("x".repeat(300)))).toBe(true) +}) diff --git a/packages/redrob/test/desk/extension.test.ts b/packages/redrob/test/desk/extension.test.ts new file mode 100644 index 0000000000..7ac11c965f --- /dev/null +++ b/packages/redrob/test/desk/extension.test.ts @@ -0,0 +1,282 @@ +import { afterEach, describe, expect, test } from "bun:test" +import * as DeskCrypto from "../../src/desk/crypto" +import { decodeAction, decodeControl, encodeAction } from "../../src/desk/frames" +import { BrowserUnavailable, ExtensionLink, type Socket } from "../../src/desk/extension" + +const encoder = new TextEncoder() +const decoder = new TextDecoder() + +/** The relay room: every frame goes to its target, or to everyone else. */ +class Room { + readonly sockets = new Set<FakeSocket>() + + deliver(from: FakeSocket, data: string | Uint8Array): void { + let target: string | undefined + if (typeof data === "string") target = decodeControl(data)?.targetId + else target = decodeAction(data)?.targetId + for (const socket of this.sockets) { + if (socket === from) continue + if (target && socket.peerId !== target) continue + const payload = typeof data === "string" ? data : data.slice().buffer + queueMicrotask(() => socket.emit("message", { data: payload })) + } + } +} + +class FakeSocket implements Socket { + readyState = 0 + binaryType = "blob" + private readonly listeners = new Map<string, Array<(event: any) => void>>() + + constructor( + readonly room: Room, + readonly peerId: string, + ) { + room.sockets.add(this) + queueMicrotask(() => { + this.readyState = 1 + this.emit("open", {}) + }) + } + + send(data: string | Uint8Array): void { + this.room.deliver(this, data) + } + + close(): void { + this.readyState = 3 + this.room.sockets.delete(this) + this.emit("close", { code: 1000 }) + } + + addEventListener(type: string, listener: (event: any) => void): void { + this.listeners.set(type, [...(this.listeners.get(type) ?? []), listener]) + } + + emit(type: string, event: unknown): void { + for (const listener of this.listeners.get(type) ?? []) listener(event) + } +} + +/** + * The Redrob extension's side as redrob-extension `src/remote/relay.ts` does it: announce with + * `kind: "browser"`, `desk.pending` for an unknown device, `desk.accept` for an allowed one, and a + * `res` for every sealed `req`. + */ +class FakeExtension { + readonly socket: FakeSocket + readonly allowed = new Set<string>() + readonly requests: Array<{ method: unknown; path: unknown; body?: unknown }> = [] + lastHello?: { peerId: string; pub: Uint8Array; eph: Uint8Array; name: string } + channel?: DeskCrypto.Channel + answer: (request: { method: unknown; path: unknown; body?: unknown }) => { status: number; body?: unknown } = () => ({ + status: 200, + body: { ok: true }, + }) + + private constructor( + room: Room, + readonly desk: DeskCrypto.KeyPair, + ) { + this.socket = new FakeSocket(room, "ext-peer") + this.socket.addEventListener("open", () => void this.announce()) + this.socket.addEventListener("message", (event: { data: string | ArrayBuffer }) => void this.onData(event.data)) + } + + static async create(room: Room): Promise<FakeExtension> { + return new FakeExtension(room, await DeskCrypto.generate()) + } + + private send(namespace: string, value: unknown, targetId?: string): void { + this.socket.send( + encodeAction({ senderId: "ext-peer", ...(targetId ? { targetId } : {}), namespace }, encoder.encode(JSON.stringify(value))), + ) + } + + async announce(): Promise<void> { + this.send("desk.announce", { + v: 1, + desk: DeskCrypto.b64url(this.desk.pub), + deskId: await DeskCrypto.keyId(this.desk.pub), + name: "Chrome on Windows", + kind: "browser", + }) + } + + async approve(): Promise<void> { + const hello = this.lastHello! + this.allowed.add(await DeskCrypto.keyId(hello.pub)) + await this.accept() + } + + deny(): void { + this.send("desk.denied", { v: 1, reason: "denied" }, this.lastHello!.peerId) + } + + private async accept(): Promise<void> { + const hello = this.lastHello! + const ephemeral = await DeskCrypto.generate() + this.channel = await DeskCrypto.Channel.create({ + desk: this.desk, + ephemeral, + phonePub: hello.pub, + phoneEph: hello.eph, + ownPeerId: "ext-peer", + phonePeerId: hello.peerId, + }) + this.send("desk.accept", { v: 1, eph: DeskCrypto.b64url(ephemeral.pub) }, hello.peerId) + } + + private async onData(data: string | ArrayBuffer): Promise<void> { + if (typeof data === "string") { + if (decodeControl(data)?.type === "hello") await this.announce() + return + } + const frame = decodeAction(new Uint8Array(data)) + if (!frame) return + if (frame.namespace === "desk.hello") { + const hello = JSON.parse(decoder.decode(frame.payload)) as { phone: string; eph: string; name: string; kind: string } + expect(hello.kind).toBe("web") + this.lastHello = { + peerId: frame.senderId, + pub: DeskCrypto.fromB64url(hello.phone), + eph: DeskCrypto.fromB64url(hello.eph), + name: hello.name, + } + if (this.allowed.has(await DeskCrypto.keyId(this.lastHello.pub))) await this.accept() + else this.send("desk.pending", { v: 1 }, frame.senderId) + return + } + if (frame.namespace === "desk.msg" && this.channel) { + const plain = await this.channel.open(frame.payload) + if (!plain) return + const message = JSON.parse(decoder.decode(plain)) as { t: string; id: number; method: unknown; path: unknown; body?: unknown } + if (message.t !== "req") return + this.requests.push({ method: message.method, path: message.path, body: message.body }) + const result = this.answer(message) + const sealed = await this.channel.seal(encoder.encode(JSON.stringify({ t: "res", id: message.id, ...result }))) + this.socket.send(encodeAction({ senderId: "ext-peer", targetId: frame.senderId, namespace: "desk.msg" }, sealed)) + } + } +} + +function secrets() { + const values = new Map<string, string>() + return { values, get: async (key: string) => values.get(key), set: async (key: string, value: string) => void values.set(key, value) } +} + +const links: ExtensionLink[] = [] +afterEach(() => { + for (const link of links.splice(0)) link.stop() +}) + +function link(room: Room, store = secrets()) { + const created = new ExtensionLink( + { relayTicket: async () => ({ url: "wss://relay.test/r", ticket: "t", expiresAt: "", role: "owner", peerId: "code-peer", room: "desk:u" }) }, + store, + { name: "Redrob Code on test", socket: () => new FakeSocket(room, "code-peer"), announceWaitMs: 200 }, + ) + links.push(created) + created.start() + return created +} + +describe("desk crypto, device side", () => { + test("a device channel and a desk channel read each other, and both sides show the same code", async () => { + const [desk, device, deskEph, deviceEph] = await Promise.all([ + DeskCrypto.generate(), + DeskCrypto.generate(), + DeskCrypto.generate(), + DeskCrypto.generate(), + ]) + const deskSide = await DeskCrypto.Channel.create({ + desk, + ephemeral: deskEph, + phonePub: device.pub, + phoneEph: deviceEph.pub, + ownPeerId: "D", + phonePeerId: "P", + }) + const deviceSide = await DeskCrypto.Channel.device({ + device, + ephemeral: deviceEph, + deskPub: desk.pub, + deskEph: deskEph.pub, + ownPeerId: "P", + deskPeerId: "D", + }) + const there = await deskSide.open(await deviceSide.seal(encoder.encode("req"))) + expect(decoder.decode(there!)).toBe("req") + const back = await deviceSide.open(await deskSide.seal(encoder.encode("res"))) + expect(decoder.decode(back!)).toBe("res") + expect(await DeskCrypto.devicePairingCode(device, desk.pub)).toBe(await DeskCrypto.pairingCode(desk, device.pub)) + }) +}) + +describe("ExtensionLink", () => { + test("no browser in the room: unavailable with state none", async () => { + const room = new Room() + const subject = link(room) + const error = await subject.request("GET", "/desk/capabilities").catch((e: unknown) => e) + expect(error).toBeInstanceOf(BrowserUnavailable) + expect((error as BrowserUnavailable).state).toEqual({ state: "none" }) + }) + + test("listens without asking: Chrome gets no pairing request until the first request", async () => { + const room = new Room() + const extension = await FakeExtension.create(room) + const subject = link(room) + await Bun.sleep(100) + expect(subject.status()).toEqual({ state: "found", name: "Chrome on Windows" }) + expect(extension.lastHello).toBeUndefined() + }) + + test("first request waits for Allow, then goes through; the pairing is kept", async () => { + const room = new Room() + const extension = await FakeExtension.create(room) + extension.answer = (request) => ({ status: 200, body: { echo: request.path } }) + const store = secrets() + const subject = link(room, store) + const pending = subject.request("GET", "/desk/capabilities", undefined, { pairMs: 5_000 }) + await Bun.sleep(150) + const status = subject.status() + expect(status.state).toBe("pairing") + const code = status.state === "pairing" ? status.code : "" + expect(code).toBe(await DeskCrypto.pairingCode(extension.desk, extension.lastHello!.pub)) + expect(extension.lastHello!.name).toBe("Redrob Code on test") + await extension.approve() + expect(await pending).toEqual({ status: 200, body: { echo: "/desk/capabilities" } }) + expect(subject.status()).toEqual({ state: "ready", name: "Chrome on Windows" }) + expect(store.values.get("browserDesk")).toBe(await DeskCrypto.keyId(extension.desk.pub)) + expect(store.values.has("browserKey")).toBe(true) + + // Same key on the next start: the extension knows it and accepts at once. + subject.stop() + const again = link(room, store) + expect((await again.request("POST", "/browser/task", { task: "x" })).status).toBe(200) + expect(extension.requests.at(-1)).toEqual({ method: "POST", path: "/browser/task", body: { task: "x" } }) + }) + + test("no Allow in time: unavailable as pairing, and the next request does not wait again", async () => { + const room = new Room() + await FakeExtension.create(room) + const subject = link(room) + const first = await subject.request("GET", "/desk/capabilities", undefined, { pairMs: 300 }).catch((e: unknown) => e) + expect((first as BrowserUnavailable).state.state).toBe("pairing") + const started = Date.now() + const second = await subject.request("GET", "/desk/capabilities", undefined, { pairMs: 5_000 }).catch((e: unknown) => e) + expect((second as BrowserUnavailable).state.state).toBe("pairing") + expect(Date.now() - started).toBeLessThan(2_000) + }) + + test("Deny in Chrome: unavailable as denied", async () => { + const room = new Room() + const extension = await FakeExtension.create(room) + const subject = link(room) + const pending = subject.request("GET", "/desk/capabilities", undefined, { pairMs: 5_000 }).catch((e: unknown) => e) + await Bun.sleep(150) + extension.deny() + const error = await pending + expect((error as BrowserUnavailable).state).toEqual({ state: "denied", name: "Chrome on Windows", reason: "denied" }) + }) +}) diff --git a/packages/sdk/js/src/v2/gen/sdk.gen.ts b/packages/sdk/js/src/v2/gen/sdk.gen.ts index 51e105eefe..fa2516e2d4 100644 --- a/packages/sdk/js/src/v2/gen/sdk.gen.ts +++ b/packages/sdk/js/src/v2/gen/sdk.gen.ts @@ -28,6 +28,7 @@ import type { ConfigProvidersResponses, ConfigUpdateErrors, ConfigUpdateResponses, + DeskBrowserRequest, DeskMachineUpdate, DeskToken, EventSubscribeResponses, @@ -346,6 +347,8 @@ import type { V2CredentialUpdateResponses, V2DeskApproveErrors, V2DeskApproveResponses, + V2DeskBrowserErrors, + V2DeskBrowserResponses, V2DeskDenyErrors, V2DeskDenyResponses, V2DeskEnableErrors, @@ -6116,6 +6119,30 @@ export class Desk extends HeyApiClient { }) } + /** + * Ask the person's browser extension + * + * Sends one request of the Redrob extension's route set to the person's Chrome over the desk relay, as a paired device (the first time the person allows this engine in Chrome). Used by the browser tool of every engine on this machine. status 0 when it did not reach the extension. 503 on a server that is not the registered daemon. + */ + public browser<ThrowOnError extends boolean = false>( + parameters?: { + deskBrowserRequest?: DeskBrowserRequest + }, + options?: Options<never, ThrowOnError>, + ) { + const params = buildClientParams([parameters], [{ args: [{ key: "deskBrowserRequest", map: "body" }] }]) + return (options?.client ?? this.client).post<V2DeskBrowserResponses, V2DeskBrowserErrors, ThrowOnError>({ + url: "/desk/browser", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + private _token?: Token get token(): Token { return (this._token ??= new Token({ client: this.client })) diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index 8ddf3e865d..6d39e9cf7a 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -2975,6 +2975,13 @@ export type DeskLogin = { expiresAt: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" } +export type DeskBrowserState = { + state: "off" | "none" | "found" | "pairing" | "ready" | "denied" + name?: string + code?: string + reason?: string +} + export type DeskStatus = { enabled: boolean signedIn: boolean @@ -2997,6 +3004,7 @@ export type DeskStatus = { addedAt: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" online: boolean }> + browser?: DeskBrowserState } export type ConflictError = { @@ -3036,6 +3044,20 @@ export type DeskMachineUpdate = { access?: "read_only" | "ask" | "trusted" } +export type DeskBrowserRequest = { + method: "GET" | "POST" + path: string + body?: unknown + pairMs?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" +} + +export type DeskBrowserResponse = { + status: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + body?: unknown + browser: DeskBrowserState + error?: string +} + export type BridgedApp = { id: string name: string @@ -13228,6 +13250,35 @@ export type V2DeskMachineUpdateResponses = { export type V2DeskMachineUpdateResponse = V2DeskMachineUpdateResponses[keyof V2DeskMachineUpdateResponses] +export type V2DeskBrowserData = { + body?: DeskBrowserRequest + path?: never + query?: never + url: "/desk/browser" +} + +export type V2DeskBrowserErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError +} + +export type V2DeskBrowserError = V2DeskBrowserErrors[keyof V2DeskBrowserErrors] + +export type V2DeskBrowserResponses = { + /** + * DeskBrowserResponse + */ + 200: DeskBrowserResponse +} + +export type V2DeskBrowserResponse = V2DeskBrowserResponses[keyof V2DeskBrowserResponses] + export type V1AppsListData = { body?: never path?: never diff --git a/packages/server/src/channels.ts b/packages/server/src/channels.ts index 2d300f3bcf..cbda245975 100644 --- a/packages/server/src/channels.ts +++ b/packages/server/src/channels.ts @@ -45,6 +45,31 @@ export type DeskStatus = { readonly login?: DeskLogin readonly pending: readonly { code: string; name: string; kind: DeskKind; expiresAt: number }[] readonly devices: readonly { id: string; name: string; kind: DeskKind; addedAt: number; online: boolean }[] + /** The person's Chrome with the Redrob extension, as the `browser` tool reaches it. Absent from an older controller. */ + readonly browser?: DeskBrowserState +} + +export type DeskBrowserState = { + readonly state: "off" | "none" | "found" | "pairing" | "ready" | "denied" + readonly name?: string + readonly code?: string + readonly reason?: string +} + +/** `POST /desk/browser` body: one request of the extension's route set (redrob-extension `src/remote/forward.ts`). */ +export type DeskBrowserRequest = { + readonly method: "GET" | "POST" + readonly path: string + readonly body?: unknown + readonly pairMs?: number +} + +/** `status` 0: the request did not reach the extension; `error` says why. */ +export type DeskBrowserResponse = { + readonly status: number + readonly body?: unknown + readonly browser: DeskBrowserState + readonly error?: string } /** `PUT /desk/token` body: the app's redrob-auth access token and its expiry (epoch ms). */ @@ -77,6 +102,8 @@ export type DeskController = { /** This machine; optional so an older controller still type-checks (the route answers 503 without it). */ machine?(): Promise<DeskMachine> updateMachine?(update: DeskMachineUpdate): Promise<DeskMachine> + /** The person's browser extension over the relay; optional so an older controller still type-checks (503 without it). */ + browser?(request: DeskBrowserRequest): Promise<DeskBrowserResponse> } export type DeskMachineKind = "desktop" | "server" | "cloud" diff --git a/packages/server/src/handlers/channels.ts b/packages/server/src/handlers/channels.ts index 0f40246a18..28ab236db1 100644 --- a/packages/server/src/handlers/channels.ts +++ b/packages/server/src/handlers/channels.ts @@ -146,5 +146,14 @@ export const ChannelsHandler = HttpApiBuilder.group(Api, "server.channels", (han if (!update) return yield* Effect.fail(noDesk()) return yield* Effect.promise(() => update.call(controller, ctx.payload)) }), + ) + .handle( + "desk.browser", + Effect.fn(function* (ctx) { + const controller = yield* desk() + const ask = controller.browser + if (!ask) return yield* Effect.fail(noDesk()) + return yield* Effect.promise(() => ask.call(controller, ctx.payload)) + }), ), ) From cab46c3c5757a2244eb070cfb7a56a2d5cc44529 Mon Sep 17 00:00:00 2001 From: Janghoon Lee <44862514+its-janghoon@users.noreply.github.com> Date: Fri, 13 Nov 2026 08:04:45 +0000 Subject: [PATCH 2/3] fix(desk): reconnect the browser link when Redrob Desktop hands over a token On the Windows check the stored refresh token was stale, so the browser link's first tickets failed and it backed off. PUT /desk/token kicked only the desk relay. It now kicks the browser link too, and the link keeps one ticket request in flight at a time, as the relay does. --- packages/redrob/src/desk/extension.ts | 16 +++++++++++++++- packages/redrob/src/desk/service.ts | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/redrob/src/desk/extension.ts b/packages/redrob/src/desk/extension.ts index 4181d3dc9f..71096b7b56 100644 --- a/packages/redrob/src/desk/extension.ts +++ b/packages/redrob/src/desk/extension.ts @@ -106,6 +106,8 @@ export class ExtensionLink { private stopped = true private delay = 1_000 private reconnect?: ReturnType<typeof setTimeout> + /** A ticket request is in flight. */ + private connecting = false private connectedAt = 0 private readonly browsers = new Map<string, Browser>() private readonly peers = new Map<string, string>() @@ -147,6 +149,15 @@ export class ExtensionLink { this.dropAll(new Error("the relay link stopped")) } + /** A new token arrived (Redrob Desktop's `PUT /desk/token`): when not connected, try now instead of after the backoff. */ + kick(): void { + if (this.stopped || this.socket) return + this.delay = 1_000 + if (this.reconnect) clearTimeout(this.reconnect) + this.reconnect = undefined + void this.connect() + } + /** Where this engine stands with the person's browser, for `GET /desk/status`. */ status(): BrowserState { if (!this.socket || this.socket.readyState !== OPEN) return { state: "off" } @@ -321,13 +332,16 @@ export class ExtensionLink { } private async connect(): Promise<void> { - if (this.stopped || this.socket) return + if (this.stopped || this.socket || this.connecting) return let ticket: RelayTicket + this.connecting = true try { ticket = await this.chat.relayTicket() } catch (error) { this.options.log?.(`browser link: no relay ticket: ${error instanceof Error ? error.message : String(error)}`) return this.schedule() + } finally { + this.connecting = false } if (this.stopped || this.socket) return const url = new URL(ticket.url) diff --git a/packages/redrob/src/desk/service.ts b/packages/redrob/src/desk/service.ts index 82bcdbdcf5..a710120422 100644 --- a/packages/redrob/src/desk/service.ts +++ b/packages/redrob/src/desk/service.ts @@ -243,6 +243,7 @@ export class DeskService { this.app.set(token.accessToken, token.expiresAt, email || undefined) await this.reconcile() this.relay?.kick() + this.extension?.kick() return this.status() } From 41491cf5430930fb0fc10ab564bf9ca8ba7e07b2 Mon Sep 17 00:00:00 2001 From: Janghoon Lee <44862514+its-janghoon@users.noreply.github.com> Date: Fri, 13 Nov 2026 08:55:31 +0000 Subject: [PATCH 3/3] fix(desk): a fresh key and code per pairing request, and log the handshake The browser link stored its device key the first time it said hello, so every pairing request from this engine showed the same six digits (286874 on the Windows check) and the extension's notification kept the same id (`pair:<code>`). An Allow given to an earlier request's prompt could not match the request going on, and nothing in the log said what the extension answered. - The key is kept only once the extension sent desk.accept. Until then each pairing request uses its own key, kept while that request waits, so its code is new; an expired or denied request drops it. - The code is known when the hello goes out, so status and the log show it before the extension answers. - An extension that no longer knows an allowed key (removed in its Settings) gets a new request with a fresh key. - A browser that leaves in the middle of the handshake gets the hello again when it is back, with the same key. - An Allow that arrives while no tool call waits still links. - Every step is logged with a timestamp: browser online, hello with name and code, waiting for Allow, allowed, denied/expired, offline. --- packages/redrob/src/desk/extension.ts | 104 +++++++++++++++----- packages/redrob/test/desk/extension.test.ts | 33 +++++++ 2 files changed, 113 insertions(+), 24 deletions(-) diff --git a/packages/redrob/src/desk/extension.ts b/packages/redrob/src/desk/extension.ts index 71096b7b56..4b8f6c59a8 100644 --- a/packages/redrob/src/desk/extension.ts +++ b/packages/redrob/src/desk/extension.ts @@ -93,6 +93,10 @@ type Browser = { reason?: string /** A request already waited out the Allow for the current pairing request. */ waited?: boolean + /** The key this engine said hello with. */ + device?: DeskCrypto.KeyPair + /** It left in the middle of a handshake: say hello again when it is back. */ + rehello?: boolean sending: Promise<void> } @@ -115,7 +119,9 @@ export class ExtensionLink { private nextId = 1 private inbound: Promise<void> = Promise.resolve() private changed: Array<() => void> = [] - private device?: Promise<DeskCrypto.KeyPair> + private allowed?: Promise<DeskCrypto.KeyPair | undefined> + /** The key of the pairing request going on, until it is allowed, denied or expires. */ + private candidate?: DeskCrypto.KeyPair private chosen?: string constructor( @@ -285,28 +291,48 @@ export class ExtensionLink { ) } - private key(): Promise<DeskCrypto.KeyPair> { - if (!this.device) { - this.device = (async () => { - const stored = await this.secrets.get("browserKey") - if (stored) return DeskCrypto.importPrivate(stored) - const fresh = await DeskCrypto.generate() - await this.secrets.set("browserKey", await DeskCrypto.exportPrivate(fresh)) - return fresh - })() - this.device.catch(() => { - this.device = undefined - }) + /** + * The key the extension allowed before (`browserKey`, kept only once a `desk.accept` came), else + * undefined. A key that was never allowed is not kept: each pairing request uses a fresh key, + * so its six digits are new too, and an Allow left over from an earlier request cannot match. + */ + private allowedKey(): Promise<DeskCrypto.KeyPair | undefined> { + if (!this.allowed) { + this.allowed = (async () => { + const [stored, desk] = await Promise.all([this.secrets.get("browserKey"), this.secrets.get("browserDesk")]) + return stored && desk ? DeskCrypto.importPrivate(stored) : undefined + })().catch(() => undefined) } - return this.device + return this.allowed + } + + /** The name the extension shows in its Allow prompt and its list of allowed devices. */ + private get ownName(): string { + return this.options.name ?? `Redrob Code on ${os.hostname()}` + } + + private log(message: string): void { + this.options.log?.(`${new Date().toISOString()} browser link: ${message}`) } private async hello(browser: Browser): Promise<void> { if (!browser.peerId) return - const [device, ephemeral] = await Promise.all([this.key(), DeskCrypto.generate()]) + const allowed = await this.allowedKey() + // Not allowed yet: one key per pairing request, kept for as long as that request waits. + const device = allowed ?? (this.candidate ??= await DeskCrypto.generate()) + const ephemeral = await DeskCrypto.generate() + browser.device = device browser.ephemeral = ephemeral browser.channel = undefined browser.link = "handshake" + browser.rehello = false + // The extension shows the same six digits (ECDH both ways); known before it answers. + browser.code = await DeskCrypto.devicePairingCode(device, browser.pub) + this.log( + allowed + ? `hello to ${browser.name} (${browser.id}) as "${this.ownName}", allowed before` + : `asking ${browser.name} (${browser.id}) to allow "${this.ownName}" (code ${browser.code})`, + ) this.notify() this.sendJson( "desk.hello", @@ -314,7 +340,7 @@ export class ExtensionLink { v: 1, phone: DeskCrypto.b64url(device.pub), eph: DeskCrypto.b64url(ephemeral.pub), - name: this.options.name ?? `Redrob Code on ${os.hostname()}`, + name: this.ownName, kind: "web", }, browser.peerId, @@ -338,7 +364,7 @@ export class ExtensionLink { try { ticket = await this.chat.relayTicket() } catch (error) { - this.options.log?.(`browser link: no relay ticket: ${error instanceof Error ? error.message : String(error)}`) + this.log(`no relay ticket: ${error instanceof Error ? error.message : String(error)}`) return this.schedule() } finally { this.connecting = false @@ -457,12 +483,14 @@ export class ExtensionLink { } browser.name = name const samePeer = browser.peerId === peerId + if (!samePeer) this.log(`${name} (${browser.id}) is online`) if (browser.peerId && !samePeer) this.peers.delete(browser.peerId) browser.peerId = peerId this.peers.set(peerId, browser.id) - // A new connection of that browser (its worker restarted): the old session is gone. Say hello - // again only when this engine had already been talking to it. - if (!samePeer && browser.link !== "idle" && browser.link !== "denied") { + // A new connection of that browser (its worker restarted, or it left in the middle of the + // handshake): its old session and its pairing request are gone. Say hello again when this + // engine had been talking to it, with the same key, so the code on screen stays the same. + if (!samePeer && (browser.rehello || (browser.link !== "idle" && browser.link !== "denied"))) { this.fail(browser.id, new Error(`${browser.name} reconnected`)) browser.channel = undefined browser.link = "idle" @@ -474,20 +502,33 @@ export class ExtensionLink { private async onPending(peerId: string): Promise<void> { const browser = this.browserOf(peerId) if (!browser || browser.link !== "handshake") return - browser.code = await DeskCrypto.devicePairingCode(await this.key(), browser.pub) + if (browser.device && browser.device === (await this.allowedKey())) { + // The extension forgot this engine (removed in its Settings, or reinstalled): start a new + // pairing request with a fresh key instead of re-showing the old one's code. + this.log(`${browser.name} no longer knows this engine; asking again`) + await this.forgetAllowed() + return this.hello(browser) + } browser.link = "pending" - this.options.log?.(`browser link: allow "Redrob Code" in ${browser.name} (code ${browser.code})`) + this.log(`waiting for Allow in ${browser.name}: "${this.ownName}" asks, code ${browser.code}`) this.notify() } + private async forgetAllowed(): Promise<void> { + this.allowed = Promise.resolve(undefined) + this.candidate = undefined + await this.secrets.set("browserDesk", "").catch(() => undefined) + } + private async onAccept(peerId: string, accept: Record<string, unknown> | undefined): Promise<void> { const browser = this.browserOf(peerId) const ephemeral = browser?.ephemeral - if (!browser || !ephemeral || !this.ticket || typeof accept?.eph !== "string") return + const device = browser?.device + if (!browser || !ephemeral || !device || !this.ticket || typeof accept?.eph !== "string") return const deskEph = DeskCrypto.fromB64url(accept.eph) try { browser.channel = await DeskCrypto.Channel.device({ - device: await this.key(), + device, ephemeral, deskPub: browser.pub, deskEph, @@ -500,6 +541,14 @@ export class ExtensionLink { browser.link = "ready" browser.code = undefined browser.waited = false + this.log(`${browser.name} allowed "${this.ownName}"; linked`) + // Keep the key only now that the extension knows it: the next start says hello without a code. + if (this.candidate === device) { + this.candidate = undefined + this.allowed = Promise.resolve(device) + await this.secrets.set("browserKey", await DeskCrypto.exportPrivate(device)).catch(() => undefined) + await this.secrets.set("browserDesk", browser.id).catch(() => undefined) + } if (this.chosen !== browser.id) { this.chosen = browser.id await this.secrets.set("browserDesk", browser.id).catch(() => undefined) @@ -517,6 +566,11 @@ export class ExtensionLink { browser.link = reason === "expired" ? "idle" : "denied" browser.reason = reason browser.waited = false + browser.code = undefined + this.log(`${browser.name} answered ${reason}`) + // That pairing request is over: the next one gets a fresh key and code. + if (browser.device === this.candidate) this.candidate = undefined + else if (reason === "revoked") void this.forgetAllowed() this.fail(browser.id, new BrowserUnavailable(this.status())) this.notify() } @@ -554,7 +608,9 @@ export class ExtensionLink { if (!browser) return browser.peerId = undefined browser.channel = undefined + if (browser.link === "handshake" || browser.link === "pending" || browser.link === "ready") browser.rehello = true if (browser.link !== "denied") browser.link = "idle" + this.log(`${browser.name} (${browser.id}) went offline`) this.fail(browser.id, new Error(`${browser.name} went offline`)) this.notify() } diff --git a/packages/redrob/test/desk/extension.test.ts b/packages/redrob/test/desk/extension.test.ts index 7ac11c965f..de6d38c62e 100644 --- a/packages/redrob/test/desk/extension.test.ts +++ b/packages/redrob/test/desk/extension.test.ts @@ -269,6 +269,39 @@ describe("ExtensionLink", () => { expect(Date.now() - started).toBeLessThan(2_000) }) + test("each pairing request has its own code; a request nobody allowed leaves no key behind", async () => { + const room = new Room() + const extension = await FakeExtension.create(room) + const store = secrets() + const first = link(room, store) + await first.request("GET", "/desk/capabilities", undefined, { pairMs: 300 }).catch(() => undefined) + const one = first.status() + const firstPub = DeskCrypto.b64url(extension.lastHello!.pub) + first.stop() + expect(store.values.has("browserKey")).toBe(false) + const second = link(room, store) + await second.request("GET", "/desk/capabilities", undefined, { pairMs: 300 }).catch(() => undefined) + expect(DeskCrypto.b64url(extension.lastHello!.pub)).not.toBe(firstPub) + expect(one.state === "pairing" && second.status().state === "pairing").toBe(true) + // The code the engine shows is the one the extension computes for that request. + const shown = second.status() + expect(shown.state === "pairing" ? shown.code : "").toBe( + await DeskCrypto.pairingCode(extension.desk, extension.lastHello!.pub), + ) + }) + + test("an Allow while no tool call waits still links, and the next request goes straight through", async () => { + const room = new Room() + const extension = await FakeExtension.create(room) + const subject = link(room) + await subject.request("GET", "/desk/capabilities", undefined, { pairMs: 200 }).catch(() => undefined) + expect(subject.status().state).toBe("pairing") + await extension.approve() + await Bun.sleep(50) + expect(subject.status()).toEqual({ state: "ready", name: "Chrome on Windows" }) + expect((await subject.request("GET", "/desk/capabilities")).status).toBe(200) + }) + test("Deny in Chrome: unavailable as denied", async () => { const room = new Room() const extension = await FakeExtension.create(room)