From a84c0afd984b4964f4d0cf22fb7c4d06c6b4b4ac Mon Sep 17 00:00:00 2001 From: Janghoon Lee <44862514+its-janghoon@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:38:23 +0900 Subject: [PATCH 1/2] feat(browser): Korean browser rows, a live Allow notice, and a fresh pairing request after expiry (#260) Browser tool rows in the TUI follow its language (ko/en), from metadata.browser chrome|own. While a call waits for Allow in the person's Chrome the row shows the code (metadata.pairing), cleared on Allow, deny, expiry or fallback. With no Chrome with the extension, the first fallback of a session says how to connect (metadata.connect and a line for the model). The link expires a pairing request after ten minutes on its own and the next call sends a fresh one with a new code; a repeat call waits 20 s. --- packages/redrob/src/browser/extension.ts | 129 ++++++++++++++---- packages/redrob/src/desk/extension.ts | 29 +++- packages/redrob/src/tool/browser.ts | 30 +++- .../redrob/test/browser/extension.test.ts | 59 +++++++- packages/redrob/test/desk/extension.test.ts | 31 ++++- packages/tui/src/i18n/en.ts | 15 ++ packages/tui/src/i18n/ko.ts | 14 ++ packages/tui/src/routes/session/index.tsx | 29 +++- packages/tui/src/util/tool-display.ts | 56 ++++++++ packages/tui/test/util/tool-display.test.ts | 62 ++++++++- 10 files changed, 417 insertions(+), 37 deletions(-) diff --git a/packages/redrob/src/browser/extension.ts b/packages/redrob/src/browser/extension.ts index 293d1abf38..9806f93709 100644 --- a/packages/redrob/src/browser/extension.ts +++ b/packages/redrob/src/browser/extension.ts @@ -33,7 +33,25 @@ export type Ask = (request: Channels.DeskBrowserRequest, signal?: AbortSignal) = export type Result = | { readonly kind: "extension"; readonly browser: string; readonly url: string; readonly text: string } - | { readonly kind: "fallback"; readonly reason: string } + | { + readonly kind: "fallback" + readonly reason: string + /** No Chrome with the Redrob extension in this account, said once per session: tell the person how to connect. */ + readonly connect?: boolean + } + +/** A pairing request waiting for Allow in the person's Chrome while a tool call waits on it; undefined when none. */ +export type Pairing = { readonly code: string; readonly name: string } | undefined + +export type Context = { + readonly sessionID: string + readonly signal?: AbortSignal + readonly ask?: Ask + /** Where the daemon's link stands; polled while a call waits, for `onPairing`. */ + readonly observe?: () => Promise + /** Called with the request while one waits for Allow, and with undefined once it is over. */ + readonly onPairing?: (pairing: Pairing) => void +} const POLL_MS = 1_000 /** A task waits at most this long, as long as a hand-over to the person may last. */ @@ -61,6 +79,8 @@ type Session = { elements: Map /** The browser this session last used; a session stays in it until the next `open`. */ used?: "extension" | "chromium" + /** The how-to-connect line was given in this session. */ + hinted?: boolean } const sessions = new Map() @@ -138,19 +158,43 @@ const sleep = (ms: number, signal?: AbortSignal) => * Runs one action in the person's Chrome. A `fallback` result means it never reached the extension * (the tool uses its own Chromium and says why); anything that went wrong after that throws. */ -export async function act( - input: Input, - context: { readonly sessionID: string; readonly signal?: AbortSignal; readonly ask?: Ask }, -): Promise { +export async function act(input: Input, context: Context): Promise { const result = await attempt(input, context) if (result.kind === "fallback") session(context.sessionID).used = "chromium" return result } -async function attempt( - input: Input, - context: { readonly sessionID: string; readonly signal?: AbortSignal; readonly ask?: Ask }, -): Promise { +/** + * Polls the link while `run` waits and reports a pairing request waiting for Allow through + * `onPairing`; it is cleared when `run` settles (allowed, denied, expired or fallen back). + */ +async function watching(context: Context, run: () => Promise): Promise { + const onPairing = context.onPairing + const observe = onPairing ? (context.observe ?? (await observer())) : undefined + if (!observe || !onPairing) return run() + let shown: string | undefined + let done = false + const tick = async () => { + const state = await observe().catch(() => undefined) + if (done) return + const code = state?.state === "pairing" && state.code ? state.code : undefined + if (code === shown) return + shown = code + onPairing(code ? { code, name: state?.name ?? "Chrome" } : undefined) + } + const timer = setInterval(() => void tick(), 1_000) + timer.unref?.() + void tick() + try { + return await run() + } finally { + done = true + clearInterval(timer) + if (shown) onPairing(undefined) + } +} + +async function attempt(input: Input, context: Context): Promise { const state = session(context.sessionID) if (input.action !== "open" && state.used === "chromium") return { kind: "fallback", reason: "" } const ask = context.ask ?? (await asker()) @@ -166,24 +210,36 @@ async function attempt( // The same Code session goes on in the same conversation and tab, also after an engine restart. const chat = chatId(context.sessionID) - if (!state.task) { - const found = await ask({ method: "GET", path: `/browser/task?chatId=${chat}` }, context.signal).catch(() => undefined) - const id = (found?.body as { id?: unknown } | undefined)?.id - if (found?.status === 200 && typeof id === "string") state.task = id + const fallback = (response: Channels.DeskBrowserResponse): Result => { + const connect = response.browser.state === "none" && !state.hinted + if (connect) state.hinted = true + return { + kind: "fallback", + reason: response.error ?? `the extension is not ready (${response.browser.state})`, + ...(connect ? { connect } : {}), + } } - const created = await ask( - { - method: "POST", - path: "/browser/task", - body: { task: text, origin: "code", chatId: chat, ...(state.task ? { followUp: state.task } : {}) }, - }, - context.signal, - ) + // The first request may wait for Allow in Chrome; the person sees the code while it does. + const created = await watching(context, async () => { + if (!state.task) { + const found = await ask({ method: "GET", path: `/browser/task?chatId=${chat}` }, context.signal).catch(() => undefined) + // Not reaching the extension: no second wait for the same answer. + if (found?.status === 0) return found + const id = (found?.body as { id?: unknown } | undefined)?.id + if (found?.status === 200 && typeof id === "string") state.task = id + } + return ask( + { + method: "POST", + path: "/browser/task", + body: { task: text, origin: "code", chatId: chat, ...(state.task ? { followUp: state.task } : {}) }, + }, + context.signal, + ) + }) const id = (created.body as { id?: unknown } | undefined)?.id if (created.status !== 200 || typeof id !== "string") { - if (created.status === 0 || created.status === 503) { - return { kind: "fallback", reason: created.error ?? `the extension is not ready (${created.browser.state})` } - } + if (created.status === 0 || created.status === 503) return fallback(created) throw new Error(`The Redrob extension refused the task (HTTP ${created.status}).`) } state.used = "extension" @@ -253,6 +309,24 @@ export async function asker(env: NodeJS.ProcessEnv = process.env): Promise Promise) | undefined> { + const local = Channels.current()?.desk + if (local) return async () => (await local.status()).browser + const record = await Registry.read(env).catch(() => undefined) + if (!record || !Registry.alive(record.pid)) return undefined + return async () => { + const response = await fetch(`${record.url}/desk/status`, { + headers: Registry.headers(record.secret), + signal: AbortSignal.timeout(5_000), + }) + if (!response.ok) return undefined + return ((await response.json()) as Channels.DeskStatus).browser + } +} + /** Whether the person's browser may be reachable at all, so the tool is offered without a Chromium. */ export async function reachable(env: NodeJS.ProcessEnv = process.env): Promise { if (Channels.current()?.desk?.browser) return true @@ -260,10 +334,15 @@ export async function reachable(env: NodeJS.ProcessEnv = process.env): Promise Socket readonly announceWaitMs?: number + readonly pairTtlMs?: number + readonly repeatWaitMs?: number readonly log?: (message: string) => void } = {}, ) {} @@ -222,9 +230,20 @@ export class ExtensionLink { let browser = this.pick() if (!browser?.peerId) throw new BrowserUnavailable(this.status()) if (browser.link === "denied") throw new BrowserUnavailable(this.status()) - // The person is asked to Allow once per request in Chrome: a request after one that already - // waited out the Allow does not wait again (the Allow still links this engine when it comes). - const wait = browser.waited ? 0 : pairMs + // A request the extension has forgotten by now (its pairing requests last ten minutes), even + // when its `desk.denied expired` never reached us: start a fresh one with a new key and code. + const ttl = this.options.pairTtlMs ?? PAIR_TTL_MS + if (browser.link === "pending" && browser.askedAt !== undefined && Date.now() - browser.askedAt >= ttl) { + this.log(`the request to ${browser.name} (code ${browser.code}) expired; asking again`) + if (browser.device === this.candidate) this.candidate = undefined + browser.link = "idle" + browser.waited = false + browser.code = undefined + browser.askedAt = undefined + } + // The person is asked to Allow once per request in Chrome: a call after one that already + // waited the full time waits only briefly (the Allow still links this engine when it comes). + const wait = browser.waited ? Math.min(pairMs, this.options.repeatWaitMs ?? REPEAT_WAIT_MS) : pairMs const deadline = Date.now() + wait if (browser.link === "idle") await this.hello(browser) await this.until( @@ -510,6 +529,8 @@ export class ExtensionLink { return this.hello(browser) } browser.link = "pending" + // A hello again with the same key keeps the extension's first deadline, so keep ours too. + browser.askedAt ??= Date.now() this.log(`waiting for Allow in ${browser.name}: "${this.ownName}" asks, code ${browser.code}`) this.notify() } @@ -541,6 +562,7 @@ export class ExtensionLink { browser.link = "ready" browser.code = undefined browser.waited = false + browser.askedAt = undefined this.log(`${browser.name} allowed "${this.ownName}"; linked`) // Keep the key only now that the extension knows it: the next start says hello without a code. if (this.candidate === device) { @@ -566,6 +588,7 @@ export class ExtensionLink { browser.link = reason === "expired" ? "idle" : "denied" browser.reason = reason browser.waited = false + browser.askedAt = undefined browser.code = undefined this.log(`${browser.name} answered ${reason}`) // That pairing request is over: the next one gets a fresh key and code. diff --git a/packages/redrob/src/tool/browser.ts b/packages/redrob/src/tool/browser.ts index 8656b58d92..56741127cb 100644 --- a/packages/redrob/src/tool/browser.ts +++ b/packages/redrob/src/tool/browser.ts @@ -21,7 +21,16 @@ export const Parameters = Schema.Struct({ type Params = Schema.Schema.Type -type Metadata = { action: Params["action"]; url?: string; handedBack?: string; browser?: "extension" | "chromium" } +type Metadata = { + action: Params["action"] + url?: string + handedBack?: string + browser?: "chrome" | "own" + /** While the call waits for Allow in the person's Chrome. */ + pairing?: { code: string; name: string } + /** No Chrome with the extension was found; the TUI says how to connect (once per session). */ + connect?: boolean +} const need = (value: A | undefined, message: string): A => { if (value === undefined || value === "") throw new Error(message) @@ -68,19 +77,32 @@ export const BrowserTool = Tool.define( if (params.action === "open" && !/^https?:\/\//i.test(params.url ?? "")) throw new Error("open takes an http:// or https:// URL.") // The person's own Chrome through the Redrob extension when it is linked; else our Chromium. + // While the first call waits for Allow in Chrome, the row shows the code (metadata.pairing). const user = yield* Effect.promise(() => - UserBrowser.act(params, { sessionID: ctx.sessionID, signal: ctx.abort }), + UserBrowser.act(params, { + sessionID: ctx.sessionID, + signal: ctx.abort, + onPairing: (pairing) => + void Effect.runPromise( + ctx.metadata({ metadata: { action: params.action, ...(pairing ? { pairing } : {}) } }), + ).catch(() => undefined), + }), ) if (user.kind === "extension") return { title: `${params.action} ${user.url} (your Chrome)`, output: `${UserBrowser.line(user)}\n${user.text}`, - metadata: { action: params.action, url: user.url, browser: "extension" as const }, + metadata: { action: params.action, url: user.url, browser: "chrome" as const }, } const browserLine = UserBrowser.line(user) const handedBack = yield* Effect.promise(() => RunnerBrowser.turn(ctx.abort)) const lead = `${browserLine}\n${handedBack ? `${handedBack}\n\n` : ""}` - const meta = { action: params.action, browser: "chromium" as const, ...(handedBack ? { handedBack } : {}) } + const meta = { + action: params.action, + browser: "own" as const, + ...(handedBack ? { handedBack } : {}), + ...(user.connect ? { connect: true } : {}), + } if (params.action === "screenshot") { const shot = yield* Effect.promise(() => RunnerBrowser.screenshot()) diff --git a/packages/redrob/test/browser/extension.test.ts b/packages/redrob/test/browser/extension.test.ts index b9ba9f2510..e6a9264dc9 100644 --- a/packages/redrob/test/browser/extension.test.ts +++ b/packages/redrob/test/browser/extension.test.ts @@ -89,7 +89,7 @@ describe("UserBrowser.act", () => { test("not reaching the extension falls back with the reason, and the session stays on Chromium", async () => { const { ask, calls } = fake({ - create: { status: 0, browser: { state: "none" }, error: "no Chrome with the Redrob extension is connected" }, + create: { status: 0, browser: { state: "found", name: "Chrome" }, error: "no Chrome with the Redrob extension is connected" }, }) const result = await UserBrowser.act({ action: "open", url: "https://example.com" }, { sessionID: "ses_2", ask }) expect(result).toEqual({ kind: "fallback", reason: "no Chrome with the Redrob extension is connected" }) @@ -138,3 +138,60 @@ test("chat ids keep the extension's id shape", () => { expect(UserBrowser.chatId("ses_abc.1")).toBe("code-ses_abc_1") expect(/^[A-Za-z0-9_-]{1,128}$/.test(UserBrowser.chatId("x".repeat(300)))).toBe(true) }) + +describe("the person's side while a call waits", () => { + test("shows the pairing code while the request waits for Allow, and clears it once it is over", async () => { + const seen: Array = [] + let state: { state: "pairing" | "ready"; name: string; code?: string } = { + state: "pairing", + name: "Chrome on Windows", + code: "197159", + } + const { ask: base } = fake() + // The first request blocks like the daemon's while it waits for Allow, then Allow comes. + const ask = async (request: DeskBrowserRequest) => { + if (request.path.startsWith("/browser/task?chatId=")) { + await Bun.sleep(1_300) + state = { state: "ready", name: "Chrome on Windows" } + } + return base(request) + } + const result = await UserBrowser.act( + { action: "open", url: "https://example.com" }, + { sessionID: "ses_p1", ask, observe: async () => state, onPairing: (p) => seen.push(p?.code) }, + ) + expect(result.kind).toBe("extension") + expect(seen).toEqual(["197159", undefined]) + }) + + test("clears the notice when the call falls back (expired, denied or not reached)", async () => { + const seen: Array = [] + const ask = async (): Promise => { + await Bun.sleep(200) + return { status: 0, browser: { state: "pairing", name: "Chrome", code: "111111" }, error: "waiting" } + } + const result = await UserBrowser.act( + { action: "open", url: "https://example.com" }, + { + sessionID: "ses_p2", + ask, + observe: async () => ({ state: "pairing", name: "Chrome", code: "111111" }), + onPairing: (p) => seen.push(p?.code), + }, + ) + expect(result.kind).toBe("fallback") + expect(seen).toEqual(["111111", undefined]) + }) + + test("says how to connect once per session when no Chrome with the extension is found", async () => { + const none = { status: 0, browser: { state: "none" as const }, error: "no Chrome with the Redrob extension is connected" } + const ask = async (): Promise => none + const first = await UserBrowser.act({ action: "open", url: "https://example.com" }, { sessionID: "ses_c", ask }) + expect(first).toMatchObject({ kind: "fallback", connect: true }) + expect(UserBrowser.line(first)).toContain("install the Redrob extension in Chrome") + const again = await UserBrowser.act({ action: "open", url: "https://example.org" }, { sessionID: "ses_c", ask }) + expect(again.kind).toBe("fallback") + expect("connect" in again && again.connect).toBeFalsy() + expect(UserBrowser.line(again)).not.toContain("install the Redrob extension") + }) +}) diff --git a/packages/redrob/test/desk/extension.test.ts b/packages/redrob/test/desk/extension.test.ts index de6d38c62e..de129da68a 100644 --- a/packages/redrob/test/desk/extension.test.ts +++ b/packages/redrob/test/desk/extension.test.ts @@ -174,7 +174,7 @@ function link(room: Room, store = secrets()) { const created = new ExtensionLink( { relayTicket: async () => ({ url: "wss://relay.test/r", ticket: "t", expiresAt: "", role: "owner", peerId: "code-peer", room: "desk:u" }) }, store, - { name: "Redrob Code on test", socket: () => new FakeSocket(room, "code-peer"), announceWaitMs: 200 }, + { name: "Redrob Code on test", socket: () => new FakeSocket(room, "code-peer"), announceWaitMs: 200, repeatWaitMs: 100 }, ) links.push(created) created.start() @@ -257,7 +257,7 @@ describe("ExtensionLink", () => { expect(extension.requests.at(-1)).toEqual({ method: "POST", path: "/browser/task", body: { task: "x" } }) }) - test("no Allow in time: unavailable as pairing, and the next request does not wait again", async () => { + test("no Allow in time: unavailable as pairing, and the next request only waits briefly", async () => { const room = new Room() await FakeExtension.create(room) const subject = link(room) @@ -302,6 +302,33 @@ describe("ExtensionLink", () => { expect((await subject.request("GET", "/desk/capabilities")).status).toBe(200) }) + test("a request the extension forgot is asked again with a fresh code, without a manual step", async () => { + const room = new Room() + const extension = await FakeExtension.create(room) + const created = new ExtensionLink( + { relayTicket: async () => ({ url: "wss://relay.test/r", ticket: "t", expiresAt: "", role: "owner", peerId: "code-peer", room: "desk:u" }) }, + secrets(), + { name: "Redrob Code on test", socket: () => new FakeSocket(room, "code-peer"), announceWaitMs: 200, pairTtlMs: 400, repeatWaitMs: 100 }, + ) + links.push(created) + created.start() + await created.request("GET", "/desk/capabilities", undefined, { pairMs: 200 }).catch(() => undefined) + const first = created.status() + const firstPub = DeskCrypto.b64url(extension.lastHello!.pub) + // The same request still going: a later call does not ask again, it waits briefly. + await created.request("GET", "/desk/capabilities", undefined, { pairMs: 5_000 }).catch(() => undefined) + expect(DeskCrypto.b64url(extension.lastHello!.pub)).toBe(firstPub) + // Its ten minutes (here 400 ms) are over, and no `desk.denied expired` came. + await Bun.sleep(450) + const pending = created.request("GET", "/desk/capabilities", undefined, { pairMs: 5_000 }) + await Bun.sleep(150) + expect(DeskCrypto.b64url(extension.lastHello!.pub)).not.toBe(firstPub) + const second = created.status() + expect(first.state === "pairing" && second.state === "pairing" && first.code !== second.code).toBe(true) + await extension.approve() + expect((await pending).status).toBe(200) + }) + test("Deny in Chrome: unavailable as denied", async () => { const room = new Room() const extension = await FakeExtension.create(room) diff --git a/packages/tui/src/i18n/en.ts b/packages/tui/src/i18n/en.ts index 72a7dbed33..e70b5ac528 100644 --- a/packages/tui/src/i18n/en.ts +++ b/packages/tui/src/i18n/en.ts @@ -517,6 +517,21 @@ export const dict = { "session.tool.delegating": "Delegating...", "session.tool.patch": "Patch", "session.tool.skill": 'Skill "{{name}}"', + "session.tool.browsing": "Using the browser...", + "session.tool.browser": "Browser {{action}} {{target}}", + "session.tool.browser.chrome": "in your Chrome", + "session.tool.browser.own": "in the built-in browser", + "session.tool.browser.action.open": "open", + "session.tool.browser.action.read": "read", + "session.tool.browser.action.click": "click", + "session.tool.browser.action.type": "type", + "session.tool.browser.action.scroll": "scroll", + "session.tool.browser.action.wait": "wait", + "session.tool.browser.action.screenshot": "screenshot", + "session.tool.browser.pairing": + "Press Allow in Chrome (code {{code}}). It shows in a system notification and in the Redrob extension's side panel.", + "session.tool.browser.connect": + "To use your own Chrome, install the Redrob extension in Chrome and connect it with the same Redrob account (extension Settings → Redrob chat → Connect).", "session.tool.questions_asked.one": "Asked {{count}} question", "session.tool.questions_asked.other": "Asked {{count}} questions", "session.patch.deleted": "# Deleted {{path}}", diff --git a/packages/tui/src/i18n/ko.ts b/packages/tui/src/i18n/ko.ts index 190a5f7078..22b24b104b 100644 --- a/packages/tui/src/i18n/ko.ts +++ b/packages/tui/src/i18n/ko.ts @@ -496,6 +496,20 @@ export const dict: Record = { "session.tool.delegating": "맡기는 중...", "session.tool.patch": "패치", "session.tool.skill": '스킬 "{{name}}"', + "session.tool.browsing": "브라우저를 쓰는 중...", + "session.tool.browser": "브라우저 {{action}} {{target}}", + "session.tool.browser.chrome": "내 크롬에서", + "session.tool.browser.own": "내장 브라우저에서", + "session.tool.browser.action.open": "열기", + "session.tool.browser.action.read": "읽기", + "session.tool.browser.action.click": "클릭", + "session.tool.browser.action.type": "입력", + "session.tool.browser.action.scroll": "스크롤", + "session.tool.browser.action.wait": "기다리기", + "session.tool.browser.action.screenshot": "스크린샷", + "session.tool.browser.pairing": "크롬에서 허용을 눌러 주세요 (코드 {{code}}). 컴퓨터 알림과 레드롭 확장 옆 패널에 떠요.", + "session.tool.browser.connect": + "내 크롬을 쓰려면 크롬에 레드롭 확장을 설치하고, 확장 설정의 레드롭 챗에서 같은 레드롭 계정으로 연결해 주세요.", "session.tool.questions_asked.one": "질문 {{count}}개 끝", "session.tool.questions_asked.other": "질문 {{count}}개 끝", "session.patch.deleted": "# 지움 {{path}}", diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index 6fe985f51d..3fb910e182 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -39,7 +39,7 @@ import type { } from "@redrob-labs/sdk/v2" import { useLocal } from "../../context/local" import { Locale } from "../../util/locale" -import { webSearchProviderLabel } from "../../util/tool-display" +import { browserLabel, browserNotice, webSearchProviderLabel } from "../../util/tool-display" import { useRenderer, useTerminalDimensions, type JSX } from "@opentui/solid" import { useSDK } from "../../context/sdk" import { useEditorContext } from "../../context/editor" @@ -1896,6 +1896,9 @@ function ToolPart(props: { last: boolean; part: ToolPart; message: AssistantMess + + + @@ -2382,6 +2385,29 @@ function WebFetch(props: ToolProps) { ) } +function Browser(props: ToolProps) { + const language = useLanguage() + const { theme } = useTheme() + const notice = createMemo(() => browserNotice(language.t, props.part.state.status, props.metadata)) + return ( + + + {browserLabel(language.t, props.input, props.metadata)} + + + + {notice()} + + + + ) +} + function WebSearch(props: ToolProps) { const language = useLanguage() return ( @@ -2857,6 +2883,7 @@ const toolDisplays = new Set([ "grep", "webfetch", "websearch", + "browser", "write", "edit", "task", diff --git a/packages/tui/src/util/tool-display.ts b/packages/tui/src/util/tool-display.ts index 18e256bd34..a7aab438db 100644 --- a/packages/tui/src/util/tool-display.ts +++ b/packages/tui/src/util/tool-display.ts @@ -11,3 +11,59 @@ export function toolDisplayMetadata(state: unknown): Record { if (Array.isArray(state.structured)) return {} return state.structured as Record } + +const BROWSER_ACTIONS = ["open", "read", "click", "type", "scroll", "wait", "screenshot"] as const +type BrowserAction = (typeof BROWSER_ACTIONS)[number] + +type BrowserKey = + | "session.tool.browser" + | "session.tool.browser.chrome" + | "session.tool.browser.own" + | "session.tool.browser.pairing" + | "session.tool.browser.connect" + | `session.tool.browser.action.${BrowserAction}` + +/** + * The line under a `browser` row for the person: while the call waits for Allow in their Chrome, + * which code to allow; once no Chrome with the extension was found (the engine says so once per + * session), how to connect one. Nothing otherwise, so it goes away on Allow, deny, expiry or fallback. + */ +export function browserNotice( + t: (key: BrowserKey, params?: Record) => string, + status: string, + metadata: Record, +): string | undefined { + const pairing = metadata.pairing + if (status === "running" && pairing && typeof pairing === "object" && !Array.isArray(pairing)) { + const code = (pairing as Record).code + if (typeof code === "string" && code) return t("session.tool.browser.pairing", { code }) + } + if (status === "completed" && metadata.connect === true) return t("session.tool.browser.connect") + return undefined +} + +/** + * The row for a `browser` tool call: the action in the person's language, what it acted on + * (the page or the element), and, once the engine said so in `metadata.browser`, which browser + * ran it: the person's own Chrome through the Redrob extension, or the engine's built-in one. + */ +export function browserLabel( + t: (key: BrowserKey, params?: Record) => string, + input: Record, + metadata: Record, +): string { + const action = BROWSER_ACTIONS.find((name) => name === input.action) + const text = (value: unknown) => (typeof value === "string" && value ? value : undefined) + const target = text(input.url) ?? text(metadata.url) ?? text(input.ref) ?? text(input.text) ?? "" + const row = t("session.tool.browser", { + action: action ? t(`session.tool.browser.action.${action}`) : text(input.action) ?? "", + target, + }).trim() + const where = + metadata.browser === "chrome" + ? t("session.tool.browser.chrome") + : metadata.browser === "own" + ? t("session.tool.browser.own") + : undefined + return where ? `${row} (${where})` : row +} diff --git a/packages/tui/test/util/tool-display.test.ts b/packages/tui/test/util/tool-display.test.ts index f77f54cbab..4bf782387c 100644 --- a/packages/tui/test/util/tool-display.test.ts +++ b/packages/tui/test/util/tool-display.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test" -import { toolDisplayMetadata, webSearchProviderLabel } from "../../src/util/tool-display" +import { browserLabel, browserNotice, toolDisplayMetadata, webSearchProviderLabel } from "../../src/util/tool-display" +import { createTranslator } from "../../src/i18n" describe("webSearchProviderLabel", () => { test("labels known providers", () => { @@ -38,3 +39,62 @@ describe("toolDisplayMetadata", () => { expect(toolDisplayMetadata(undefined)).toEqual({}) }) }) + +describe("browserLabel", () => { + const en = createTranslator("en").t + const ko = createTranslator("ko").t + + test("names the person's Chrome in English and Korean", () => { + const input = { action: "open", url: "https://example.com" } + const metadata = { browser: "chrome", url: "https://example.com/" } + expect(browserLabel(en, input, metadata)).toBe("Browser open https://example.com (in your Chrome)") + expect(browserLabel(ko, input, metadata)).toBe("브라우저 열기 https://example.com (내 크롬에서)") + }) + + test("names the built-in browser, with the page from metadata when the input has none", () => { + const metadata = { browser: "own", url: "https://example.com/" } + expect(browserLabel(en, { action: "read" }, metadata)).toBe("Browser read https://example.com/ (in the built-in browser)") + expect(browserLabel(ko, { action: "read" }, metadata)).toBe("브라우저 읽기 https://example.com/ (내장 브라우저에서)") + }) + + test("leaves the browser out until the engine says which one ran it", () => { + expect(browserLabel(ko, { action: "click", ref: "e5" }, {})).toBe("브라우저 클릭 e5") + expect(browserLabel(en, { action: "scroll" }, {})).toBe("Browser scroll") + }) + + test("every action has a Korean word", () => { + for (const action of ["open", "read", "click", "type", "scroll", "wait", "screenshot"]) { + expect(browserLabel(ko, { action }, {})).toMatch(/^브라우저 [\uAC00-\uD7A3]+$/) + } + }) +}) + +describe("browserNotice", () => { + const en = createTranslator("en").t + const ko = createTranslator("ko").t + const pairing = { pairing: { code: "197159", name: "Chrome on Windows" } } + + test("while the call waits for Allow, names the code in both languages", () => { + expect(browserNotice(ko, "running", pairing)).toBe( + "크롬에서 허용을 눌러 주세요 (코드 197159). 컴퓨터 알림과 레드롭 확장 옆 패널에 떠요.", + ) + expect(browserNotice(en, "running", pairing)).toBe( + "Press Allow in Chrome (code 197159). It shows in a system notification and in the Redrob extension's side panel.", + ) + }) + + test("goes away once the call is over or the request is gone", () => { + expect(browserNotice(ko, "completed", pairing)).toBeUndefined() + expect(browserNotice(ko, "error", pairing)).toBeUndefined() + expect(browserNotice(ko, "running", { action: "open" })).toBeUndefined() + expect(browserNotice(ko, "running", { pairing: { code: "" } })).toBeUndefined() + }) + + test("says how to connect when no Chrome with the extension was found", () => { + expect(browserNotice(ko, "completed", { connect: true })).toBe( + "내 크롬을 쓰려면 크롬에 레드롭 확장을 설치하고, 확장 설정의 레드롭 챗에서 같은 레드롭 계정으로 연결해 주세요.", + ) + expect(browserNotice(en, "completed", { connect: true })).toContain("install the Redrob extension in Chrome") + expect(browserNotice(en, "completed", { browser: "own" })).toBeUndefined() + }) +}) From 5cc1a097a64ad7c0fac905ede13af78d7ca2a3d2 Mon Sep 17 00:00:00 2001 From: Janghoon Lee <44862514+its-janghoon@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:56:16 +0900 Subject: [PATCH 2/2] fix(browser): say when a site blocks the built-in browser (#261) A bot check or access block in the headless browser (Access Denied, 403, captcha, robot check, Cloudflare) is detected by browser/blocked.ts. The page is retried in the person's Chrome when it is linked; otherwise the result starts with Blocked: and tells the model to tell the person, to say an answer from search results is not from the site, and once per session how to connect Chrome. The browser and web search descriptions say shops, bookings, sign-ins and bot checks need the browser tool. --- packages/redrob/src/browser/blocked.ts | 29 +++++++ packages/redrob/src/browser/extension.ts | 48 ++++++++++++ packages/redrob/src/browser/tool.ts | 6 +- packages/redrob/src/tool/browser.ts | 15 +++- packages/redrob/src/tool/browser.txt | 4 + packages/redrob/src/tool/websearch.txt | 1 + packages/redrob/test/browser/blocked.test.ts | 81 ++++++++++++++++++++ 7 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 packages/redrob/src/browser/blocked.ts create mode 100644 packages/redrob/test/browser/blocked.test.ts diff --git a/packages/redrob/src/browser/blocked.ts b/packages/redrob/src/browser/blocked.ts new file mode 100644 index 0000000000..31a7576480 --- /dev/null +++ b/packages/redrob/src/browser/blocked.ts @@ -0,0 +1,29 @@ +/** + * Whether a page is a bot check or an access block rather than the site itself: shops and + * ticketing sites (Coupang, for one) answer a headless browser with "Access Denied", a 403, a + * captcha or a Cloudflare challenge. Read from the page's title and the start of its text only; + * a long page that merely mentions a captcha is the site. + */ +const SIGNS: ReadonlyArray = [ + [/\baccess denied\b/i, "Access Denied"], + [/\b403\b.{0,20}\bforbidden\b|\bforbidden\b.{0,20}\b403\b|^\s*403\b/i, "403 Forbidden"], + [/just a moment|checking your browser|cf-chl|attention required.{0,20}cloudflare|cloudflare ray id/i, "a Cloudflare check"], + [/\bcaptcha\b|recaptcha|hcaptcha|보안 ?문자|자동입력 ?방지/i, "a captcha"], + [/are you (?:a )?(?:human|robot)|not a robot|verify (?:you are|that you're) (?:a )?human|unusual traffic|로봇이 아닙니다/i, "a robot check"], + [/request (?:was )?blocked|pardon our interruption|bot detected|접근이 (?:차단|거부)/i, "a block page"], +] + +/** Only a page this short can be a block page; the site's own pages are longer. */ +const MAX_BLOCK_TEXT = 3_000 + +/** What blocked the page ("Access Denied", "a captcha", ...), or undefined when it is the site. */ +export function blocked(page: { readonly title: string; readonly text: string }): string | undefined { + const text = page.text.length > MAX_BLOCK_TEXT ? undefined : page.text + for (const [sign, name] of SIGNS) { + if (sign.test(page.title)) return name + if (text !== undefined && sign.test(text)) return name + } + return undefined +} + +export * as BlockedPage from "./blocked" diff --git a/packages/redrob/src/browser/extension.ts b/packages/redrob/src/browser/extension.ts index 9806f93709..1070ef75b1 100644 --- a/packages/redrob/src/browser/extension.ts +++ b/packages/redrob/src/browser/extension.ts @@ -15,6 +15,7 @@ */ import * as Channels from "@redrob-code/server/channels" import * as Registry from "@/daemon/registry" +import { BlockedPage } from "./blocked" export type Action = "open" | "read" | "click" | "type" | "scroll" | "wait" | "screenshot" @@ -339,6 +340,53 @@ export const CONNECT = "Tell the person once, in their language: to let Redrob Code use their own Chrome with their sign-ins, they install the Redrob extension in Chrome and connect it with the same Redrob account (extension Settings → Redrob chat → Connect). The next browser step then asks them to press Allow in Chrome." /** The first line of every result: which browser the action ran in, and why not the person's own. */ +/** + * A reading from the built-in browser that turned out to be a block page: try the person's Chrome + * once when it is linked now (`retried`), else the note for the result (`blockedNote`). Undefined + * when the page is the site. + */ +export async function onBlocked( + reading: { readonly url: string; readonly title: string; readonly text: string }, + context: Context, +): Promise<{ retried: Result & { kind: "extension" } } | { note: string; connect: boolean } | undefined> { + const sign = BlockedPage.blocked(reading) + if (!sign) return undefined + if (await linked(context.observe)) { + const again = await act({ action: "open", url: reading.url }, context).catch(() => undefined) + if (again?.kind === "extension") return { retried: again } + } + const { text, connect } = blockedNote(context.sessionID, sign, reading.url) + return { note: text, connect } +} + +/** The person's Chrome is linked to the engine service right now. */ +export async function linked(observe?: () => Promise): Promise { + const read = observe ?? (await observer().catch(() => undefined)) + return (await read?.().catch(() => undefined))?.state === "ready" +} + +/** + * The lines a blocked page in the built-in browser gets (BlockedPage.blocked): nothing on it came + * from the site; the person is told their own Chrome is needed, with how to connect it once per + * session; and an answer from search results has to say so. `connect` is true when the + * how-to-connect line was given this time. + */ +export function blockedNote(sessionID: string, sign: string, url: string): { text: string; connect: boolean } { + const state = session(sessionID) + const connect = !state.hinted + state.hinted = true + let host = url + try { + host = new URL(url).hostname + } catch {} + const text = [ + `Blocked: ${host} showed ${sign} to this headless browser, so nothing below is the site's content. Tell the person the site blocked the built-in browser and that their own Chrome is needed for it.`, + `If you answer from search results instead, say so plainly: the answer comes from search results, not from ${host}, and may be out of date.`, + ...(connect ? [CONNECT] : []), + ].join("\n") + return { text, connect } +} + export function line(result: Result): string { if (result.kind === "extension") return `Browser: the person's own Chrome (${result.browser}), through the Redrob extension.` const base = `Browser: Redrob Code's own Chromium on this machine (headless).${result.reason ? ` The person's Chrome is not used: ${result.reason}.` : ""}` diff --git a/packages/redrob/src/browser/tool.ts b/packages/redrob/src/browser/tool.ts index c17c0f68e7..7c13b1bf7b 100644 --- a/packages/redrob/src/browser/tool.ts +++ b/packages/redrob/src/browser/tool.ts @@ -77,7 +77,11 @@ async function act(input: Input, sessionID: string): Promise( } const reading = yield* Effect.promise(() => run(params)) + // A bot check or block page: the person's Chrome when it is linked now, else say so plainly. + const block = yield* Effect.promise(() => + UserBrowser.onBlocked(reading, { sessionID: ctx.sessionID, signal: ctx.abort }), + ) + if (block && "retried" in block) + return { + title: `${params.action} ${block.retried.url} (your Chrome)`, + output: `${UserBrowser.line(block.retried)}\n${block.retried.text}`, + metadata: { action: params.action, url: block.retried.url, browser: "chrome" as const }, + } + const note = block ? `${block.note}\n\n` : "" return { title: `${params.action} ${reading.url}`, - output: `${lead}URL: ${reading.url}\nTitle: ${reading.title}\n\n${reading.text}`, - metadata: { ...meta, url: reading.url }, + output: `${lead}${note}URL: ${reading.url}\nTitle: ${reading.title}\n\n${reading.text}`, + metadata: { ...meta, url: reading.url, ...(block?.connect ? { connect: true } : {}) }, } // A rejected promise dies with its error, which becomes the tool call's error text. }).pipe(Effect.orDie), diff --git a/packages/redrob/src/tool/browser.txt b/packages/redrob/src/tool/browser.txt index ae407d7889..131ac54513 100644 --- a/packages/redrob/src/tool/browser.txt +++ b/packages/redrob/src/tool/browser.txt @@ -11,4 +11,8 @@ Actions: Every action except screenshot returns a fresh reading. Element numbers belong to the reading they came from: after the page changes (navigation, a new tab, the user taking over), read again before using a number. +Use this tool, not web search, when the answer has to come from a site itself: prices and stock in a shop, bookings, anything behind a sign-in, or a site with a bot check. It runs in the person's own Chrome whenever that is linked. A result starting with `Blocked:` means the site showed a bot check or access block to the headless browser: nothing on that page is the site's content. Tell the person, and pass on how to connect their Chrome when the result says it. + +If the site blocked you and you answer from search results instead, say so plainly: the answer comes from search results, not from the site, and may be out of date. + The user may watch the browser and take it over. While they hold it, your next browser action waits until they hand it back (up to 10 minutes), and the result starts with a line saying what they did. Password fields are hidden in pictures. In the person's Chrome, sign-ins, payments and anything their approval setting asks about are their step there: the action waits for them. diff --git a/packages/redrob/src/tool/websearch.txt b/packages/redrob/src/tool/websearch.txt index ad5238cbd5..b38aeee25d 100644 --- a/packages/redrob/src/tool/websearch.txt +++ b/packages/redrob/src/tool/websearch.txt @@ -9,6 +9,7 @@ Usage notes: - Search types when available: 'auto' (balanced), 'fast' (quick results), 'deep' (comprehensive search) - Configurable context length for optimal LLM integration - Domain filtering and advanced search options available + - Results are snippets about a site, not the site itself. For prices or stock in a shop, bookings, or anything behind a sign-in, use the browser tool. When you answer from search results because a site blocked the browser or could not be opened, say so plainly: the answer comes from search results, not from the site, and may be out of date. The current year is {{year}}. You MUST use this year when searching for recent information or current events - Example: If the current year is 2026 and the user asks for "latest AI news", search for "AI news 2026", NOT "AI news 2025" diff --git a/packages/redrob/test/browser/blocked.test.ts b/packages/redrob/test/browser/blocked.test.ts new file mode 100644 index 0000000000..64a02fe35c --- /dev/null +++ b/packages/redrob/test/browser/blocked.test.ts @@ -0,0 +1,81 @@ +import { afterEach, describe, expect, test } from "bun:test" +import { BlockedPage } from "../../src/browser/blocked" +import { UserBrowser } from "../../src/browser/extension" +import BROWSER from "../../src/tool/browser.txt" +import WEBSEARCH from "../../src/tool/websearch.txt" + +afterEach(() => UserBrowser.reset()) + +describe("BlockedPage.blocked", () => { + const cases: Array<[string, string, string]> = [ + ["Access Denied", "- heading \"Access Denied\" [level=1]\n- text: You don't have permission to access", "Access Denied"], + ["403 Forbidden", "- heading \"403 Forbidden\"", "403 Forbidden"], + ["Just a moment...", "- text: Checking your browser before accessing", "a Cloudflare check"], + ["", "- text: Please complete the CAPTCHA to continue", "a captcha"], + ["Robot check", "- text: Are you a robot?", "a robot check"], + ["네이버", "- text: 자동입력 방지 문자를 입력해 주세요", "a captcha"], + ["Pardon Our Interruption", "- text: something about your browser", "a block page"], + ] + for (const [title, text, sign] of cases) + test(`sees ${sign} (${title || text.slice(0, 20)})`, () => { + expect(BlockedPage.blocked({ title, text })).toBe(sign) + }) + + test("the site itself is not a block, even a long page that mentions a captcha", () => { + expect(BlockedPage.blocked({ title: "쿠팡! | 에어팟 프로 3", text: "- link \"에어팟 프로 3\"\n- text: 329,000원" })).toBeUndefined() + expect(BlockedPage.blocked({ title: "Example Domain", text: "This domain is for use in documentation examples." })).toBeUndefined() + const long = `${"- text: product row\n".repeat(300)}- text: we use reCAPTCHA on sign-in` + expect(BlockedPage.blocked({ title: "Shop", text: long })).toBeUndefined() + }) +}) + +describe("a blocked page in the built-in browser", () => { + const page = { url: "https://www.coupang.com/np/search?q=airpods", title: "Access Denied", text: "- heading \"Access Denied\"" } + const off = async () => ({ state: "none" as const }) + + test("says the site blocked it, tells the model to say so, and how to connect once per session", async () => { + const first = await UserBrowser.onBlocked(page, { sessionID: "ses_b", observe: off }) + expect(first && "note" in first).toBe(true) + if (!first || !("note" in first)) return + expect(first.note.startsWith("Blocked: www.coupang.com showed Access Denied")).toBe(true) + expect(first.note).toContain("the answer comes from search results, not from www.coupang.com") + expect(first.note).toContain(UserBrowser.CONNECT) + expect(first.connect).toBe(true) + const again = await UserBrowser.onBlocked(page, { sessionID: "ses_b", observe: off }) + if (!again || !("note" in again)) throw new Error("expected a note") + expect(again.note).not.toContain(UserBrowser.CONNECT) + expect(again.connect).toBe(false) + }) + + test("a page that is the site gets nothing", async () => { + expect(await UserBrowser.onBlocked({ ...page, title: "쿠팡", text: "- text: 329,000원" }, { sessionID: "ses_c", observe: off })).toBeUndefined() + }) + + test("retries in the person's Chrome when it is linked", async () => { + const answer = "URL: https://www.coupang.com/np/search?q=airpods\nTitle: 쿠팡\n\n- text: 329,000원" + let n = 0 + const ask = async (request: { method: string; path: string }) => { + if (request.path.startsWith("/browser/task?chatId=")) return { status: 404, browser: { state: "ready" as const, name: "Chrome" } } + if (request.method === "POST") return { status: 200, body: { id: `t${++n}` }, browser: { state: "ready" as const, name: "Chrome" } } + return { status: 200, body: { id: "t1", state: "done", events: [], answer }, browser: { state: "ready" as const, name: "Chrome" } } + } + const result = await UserBrowser.onBlocked(page, { + sessionID: "ses_r", + ask, + observe: async () => ({ state: "ready" as const, name: "Chrome" }), + }) + expect(result && "retried" in result && result.retried.text).toBe(answer) + }) +}) + +describe("the tool descriptions", () => { + test("the browser tool is for shops, bookings, sign-ins and bot checks, and a block is said", () => { + expect(BROWSER).toContain("prices and stock in a shop, bookings, anything behind a sign-in, or a site with a bot check") + expect(BROWSER).toContain("the answer comes from search results, not from the site") + }) + + test("web search says snippets are not the site, and to say so", () => { + expect(WEBSEARCH).toContain("use the browser tool") + expect(WEBSEARCH).toContain("the answer comes from search results, not from the site") + }) +})