Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions packages/redrob/src/browser/blocked.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/**
* Whether a page is a bot check or an access block rather than the site itself: shops and
* ticketing sites (Coupang, for one) answer a headless browser with "Access Denied", a 403, a
* captcha or a Cloudflare challenge. Read from the page's title and the start of its text only;
* a long page that merely mentions a captcha is the site.
*/
const SIGNS: ReadonlyArray<readonly [RegExp, string]> = [
[/\baccess denied\b/i, "Access Denied"],
[/\b403\b.{0,20}\bforbidden\b|\bforbidden\b.{0,20}\b403\b|^\s*403\b/i, "403 Forbidden"],
[/just a moment|checking your browser|cf-chl|attention required.{0,20}cloudflare|cloudflare ray id/i, "a Cloudflare check"],
[/\bcaptcha\b|recaptcha|hcaptcha|보안 ?문자|자동입력 ?방지/i, "a captcha"],
[/are you (?:a )?(?:human|robot)|not a robot|verify (?:you are|that you're) (?:a )?human|unusual traffic|로봇이 아닙니다/i, "a robot check"],
[/request (?:was )?blocked|pardon our interruption|bot detected|접근이 (?:차단|거부)/i, "a block page"],
]

/** Only a page this short can be a block page; the site's own pages are longer. */
const MAX_BLOCK_TEXT = 3_000

/** What blocked the page ("Access Denied", "a captcha", ...), or undefined when it is the site. */
export function blocked(page: { readonly title: string; readonly text: string }): string | undefined {
const text = page.text.length > MAX_BLOCK_TEXT ? undefined : page.text
for (const [sign, name] of SIGNS) {
if (sign.test(page.title)) return name
if (text !== undefined && sign.test(text)) return name
}
return undefined
}

export * as BlockedPage from "./blocked"
177 changes: 152 additions & 25 deletions packages/redrob/src/browser/extension.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
import * as Channels from "@redrob-code/server/channels"
import * as Registry from "@/daemon/registry"
import { BlockedPage } from "./blocked"

export type Action = "open" | "read" | "click" | "type" | "scroll" | "wait" | "screenshot"

Expand All @@ -33,7 +34,25 @@ export type Ask = (request: Channels.DeskBrowserRequest, signal?: AbortSignal) =

export type Result =
| { readonly kind: "extension"; readonly browser: string; readonly url: string; readonly text: string }
| { readonly kind: "fallback"; readonly reason: string }
| {
readonly kind: "fallback"
readonly reason: string
/** No Chrome with the Redrob extension in this account, said once per session: tell the person how to connect. */
readonly connect?: boolean
}

/** A pairing request waiting for Allow in the person's Chrome while a tool call waits on it; undefined when none. */
export type Pairing = { readonly code: string; readonly name: string } | undefined

export type Context = {
readonly sessionID: string
readonly signal?: AbortSignal
readonly ask?: Ask
/** Where the daemon's link stands; polled while a call waits, for `onPairing`. */
readonly observe?: () => Promise<Channels.DeskBrowserState | undefined>
/** Called with the request while one waits for Allow, and with undefined once it is over. */
readonly onPairing?: (pairing: Pairing) => void
}

const POLL_MS = 1_000
/** A task waits at most this long, as long as a hand-over to the person may last. */
Expand Down Expand Up @@ -61,6 +80,8 @@ type Session = {
elements: Map<string, string>
/** The browser this session last used; a session stays in it until the next `open`. */
used?: "extension" | "chromium"
/** The how-to-connect line was given in this session. */
hinted?: boolean
}

const sessions = new Map<string, Session>()
Expand Down Expand Up @@ -138,19 +159,43 @@ const sleep = (ms: number, signal?: AbortSignal) =>
* Runs one action in the person's Chrome. A `fallback` result means it never reached the extension
* (the tool uses its own Chromium and says why); anything that went wrong after that throws.
*/
export async function act(
input: Input,
context: { readonly sessionID: string; readonly signal?: AbortSignal; readonly ask?: Ask },
): Promise<Result> {
export async function act(input: Input, context: Context): Promise<Result> {
const result = await attempt(input, context)
if (result.kind === "fallback") session(context.sessionID).used = "chromium"
return result
}

async function attempt(
input: Input,
context: { readonly sessionID: string; readonly signal?: AbortSignal; readonly ask?: Ask },
): Promise<Result> {
/**
* Polls the link while `run` waits and reports a pairing request waiting for Allow through
* `onPairing`; it is cleared when `run` settles (allowed, denied, expired or fallen back).
*/
async function watching<A>(context: Context, run: () => Promise<A>): Promise<A> {
const onPairing = context.onPairing
const observe = onPairing ? (context.observe ?? (await observer())) : undefined
if (!observe || !onPairing) return run()
let shown: string | undefined
let done = false
const tick = async () => {
const state = await observe().catch(() => undefined)
if (done) return
const code = state?.state === "pairing" && state.code ? state.code : undefined
if (code === shown) return
shown = code
onPairing(code ? { code, name: state?.name ?? "Chrome" } : undefined)
}
const timer = setInterval(() => void tick(), 1_000)
timer.unref?.()
void tick()
try {
return await run()
} finally {
done = true
clearInterval(timer)
if (shown) onPairing(undefined)
}
}

async function attempt(input: Input, context: Context): Promise<Result> {
const state = session(context.sessionID)
if (input.action !== "open" && state.used === "chromium") return { kind: "fallback", reason: "" }
const ask = context.ask ?? (await asker())
Expand All @@ -166,24 +211,36 @@ async function attempt(

// The same Code session goes on in the same conversation and tab, also after an engine restart.
const chat = chatId(context.sessionID)
if (!state.task) {
const found = await ask({ method: "GET", path: `/browser/task?chatId=${chat}` }, context.signal).catch(() => undefined)
const id = (found?.body as { id?: unknown } | undefined)?.id
if (found?.status === 200 && typeof id === "string") state.task = id
const fallback = (response: Channels.DeskBrowserResponse): Result => {
const connect = response.browser.state === "none" && !state.hinted
if (connect) state.hinted = true
return {
kind: "fallback",
reason: response.error ?? `the extension is not ready (${response.browser.state})`,
...(connect ? { connect } : {}),
}
}
const created = await ask(
{
method: "POST",
path: "/browser/task",
body: { task: text, origin: "code", chatId: chat, ...(state.task ? { followUp: state.task } : {}) },
},
context.signal,
)
// The first request may wait for Allow in Chrome; the person sees the code while it does.
const created = await watching(context, async () => {
if (!state.task) {
const found = await ask({ method: "GET", path: `/browser/task?chatId=${chat}` }, context.signal).catch(() => undefined)
// Not reaching the extension: no second wait for the same answer.
if (found?.status === 0) return found
const id = (found?.body as { id?: unknown } | undefined)?.id
if (found?.status === 200 && typeof id === "string") state.task = id
}
return ask(
{
method: "POST",
path: "/browser/task",
body: { task: text, origin: "code", chatId: chat, ...(state.task ? { followUp: state.task } : {}) },
},
context.signal,
)
})
const id = (created.body as { id?: unknown } | undefined)?.id
if (created.status !== 200 || typeof id !== "string") {
if (created.status === 0 || created.status === 503) {
return { kind: "fallback", reason: created.error ?? `the extension is not ready (${created.browser.state})` }
}
if (created.status === 0 || created.status === 503) return fallback(created)
throw new Error(`The Redrob extension refused the task (HTTP ${created.status}).`)
}
state.used = "extension"
Expand Down Expand Up @@ -253,17 +310,87 @@ export async function asker(env: NodeJS.ProcessEnv = process.env): Promise<Ask |
}
}

/** The daemon link's state: in this process when it is the daemon, else from its `GET /desk/status`. */
export async function observer(
env: NodeJS.ProcessEnv = process.env,
): Promise<(() => Promise<Channels.DeskBrowserState | undefined>) | undefined> {
const local = Channels.current()?.desk
if (local) return async () => (await local.status()).browser
const record = await Registry.read(env).catch(() => undefined)
if (!record || !Registry.alive(record.pid)) return undefined
return async () => {
const response = await fetch(`${record.url}/desk/status`, {
headers: Registry.headers(record.secret),
signal: AbortSignal.timeout(5_000),
})
if (!response.ok) return undefined
return ((await response.json()) as Channels.DeskStatus).browser
}
}

/** Whether the person's browser may be reachable at all, so the tool is offered without a Chromium. */
export async function reachable(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
if (Channels.current()?.desk?.browser) return true
const record = await Registry.read(env).catch(() => undefined)
return !!record && Registry.alive(record.pid)
}

/** Said once per session when no Chrome with the extension is connected, for the model to pass on. */
export const CONNECT =
"Tell the person once, in their language: to let Redrob Code use their own Chrome with their sign-ins, they install the Redrob extension in Chrome and connect it with the same Redrob account (extension Settings → Redrob chat → Connect). The next browser step then asks them to press Allow in Chrome."

/** The first line of every result: which browser the action ran in, and why not the person's own. */
/**
* A reading from the built-in browser that turned out to be a block page: try the person's Chrome
* once when it is linked now (`retried`), else the note for the result (`blockedNote`). Undefined
* when the page is the site.
*/
export async function onBlocked(
reading: { readonly url: string; readonly title: string; readonly text: string },
context: Context,
): Promise<{ retried: Result & { kind: "extension" } } | { note: string; connect: boolean } | undefined> {
const sign = BlockedPage.blocked(reading)
if (!sign) return undefined
if (await linked(context.observe)) {
const again = await act({ action: "open", url: reading.url }, context).catch(() => undefined)
if (again?.kind === "extension") return { retried: again }
}
const { text, connect } = blockedNote(context.sessionID, sign, reading.url)
return { note: text, connect }
}

/** The person's Chrome is linked to the engine service right now. */
export async function linked(observe?: () => Promise<Channels.DeskBrowserState | undefined>): Promise<boolean> {
const read = observe ?? (await observer().catch(() => undefined))
return (await read?.().catch(() => undefined))?.state === "ready"
}

/**
* The lines a blocked page in the built-in browser gets (BlockedPage.blocked): nothing on it came
* from the site; the person is told their own Chrome is needed, with how to connect it once per
* session; and an answer from search results has to say so. `connect` is true when the
* how-to-connect line was given this time.
*/
export function blockedNote(sessionID: string, sign: string, url: string): { text: string; connect: boolean } {
const state = session(sessionID)
const connect = !state.hinted
state.hinted = true
let host = url
try {
host = new URL(url).hostname
} catch {}
const text = [
`Blocked: ${host} showed ${sign} to this headless browser, so nothing below is the site's content. Tell the person the site blocked the built-in browser and that their own Chrome is needed for it.`,
`If you answer from search results instead, say so plainly: the answer comes from search results, not from ${host}, and may be out of date.`,
...(connect ? [CONNECT] : []),
].join("\n")
return { text, connect }
}

export function line(result: Result): string {
if (result.kind === "extension") return `Browser: the person's own Chrome (${result.browser}), through the Redrob extension.`
return `Browser: Redrob Code's own Chromium on this machine (headless).${result.reason ? ` The person's Chrome is not used: ${result.reason}.` : ""}`
const base = `Browser: Redrob Code's own Chromium on this machine (headless).${result.reason ? ` The person's Chrome is not used: ${result.reason}.` : ""}`
return result.connect ? `${base}\n${CONNECT}` : base
}

/** Forget a session's state (tests). */
Expand Down
6 changes: 5 additions & 1 deletion packages/redrob/src/browser/tool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,11 @@ async function act(input: Input, sessionID: string): Promise<Schema.Schema.Type<
return RunnerBrowser.wait({ text: input.text, seconds: input.seconds })
}
})()
return { url: reading.url, text: `${lead}URL: ${reading.url}\nTitle: ${reading.title}\n\n${reading.text}` }
const block = await UserBrowser.onBlocked(reading, { sessionID })
if (block && "retried" in block)
return { url: block.retried.url, text: `${UserBrowser.line(block.retried)}\n${block.retried.text}` }
const note = block ? `${block.note}\n\n` : ""
return { url: reading.url, text: `${lead}${note}URL: ${reading.url}\nTitle: ${reading.title}\n\n${reading.text}` }
}

const layer = Layer.effectDiscard(
Expand Down
29 changes: 26 additions & 3 deletions packages/redrob/src/desk/extension.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ const decoder = new TextDecoder()
export const ANNOUNCE_WAIT_MS = 4_000
/** How long a request waits for the person to press Allow in Chrome the first time. */
export const PAIR_WAIT_MS = 2 * 60_000
/** How long the extension keeps a pairing request (its `PENDING_MS`). */
export const PAIR_TTL_MS = 10 * 60_000
/** A later call while the same request still waits: long enough to show the code and catch an Allow. */
export const REPEAT_WAIT_MS = 20_000
const HANDSHAKE_WAIT_MS = 15_000
const REQUEST_TIMEOUT_MS = 30_000

Expand Down Expand Up @@ -93,6 +97,8 @@ type Browser = {
reason?: string
/** A request already waited out the Allow for the current pairing request. */
waited?: boolean
/** When the extension said the current pairing request is pending. */
askedAt?: number
/** The key this engine said hello with. */
device?: DeskCrypto.KeyPair
/** It left in the middle of a handshake: say hello again when it is back. */
Expand Down Expand Up @@ -132,6 +138,8 @@ export class ExtensionLink {
readonly name?: string
readonly socket?: (url: string) => Socket
readonly announceWaitMs?: number
readonly pairTtlMs?: number
readonly repeatWaitMs?: number
readonly log?: (message: string) => void
} = {},
) {}
Expand Down Expand Up @@ -222,9 +230,20 @@ export class ExtensionLink {
let browser = this.pick()
if (!browser?.peerId) throw new BrowserUnavailable(this.status())
if (browser.link === "denied") throw new BrowserUnavailable(this.status())
// The person is asked to Allow once per request in Chrome: a request after one that already
// waited out the Allow does not wait again (the Allow still links this engine when it comes).
const wait = browser.waited ? 0 : pairMs
// A request the extension has forgotten by now (its pairing requests last ten minutes), even
// when its `desk.denied expired` never reached us: start a fresh one with a new key and code.
const ttl = this.options.pairTtlMs ?? PAIR_TTL_MS
if (browser.link === "pending" && browser.askedAt !== undefined && Date.now() - browser.askedAt >= ttl) {
this.log(`the request to ${browser.name} (code ${browser.code}) expired; asking again`)
if (browser.device === this.candidate) this.candidate = undefined
browser.link = "idle"
browser.waited = false
browser.code = undefined
browser.askedAt = undefined
}
// The person is asked to Allow once per request in Chrome: a call after one that already
// waited the full time waits only briefly (the Allow still links this engine when it comes).
const wait = browser.waited ? Math.min(pairMs, this.options.repeatWaitMs ?? REPEAT_WAIT_MS) : pairMs
const deadline = Date.now() + wait
if (browser.link === "idle") await this.hello(browser)
await this.until(
Expand Down Expand Up @@ -510,6 +529,8 @@ export class ExtensionLink {
return this.hello(browser)
}
browser.link = "pending"
// A hello again with the same key keeps the extension's first deadline, so keep ours too.
browser.askedAt ??= Date.now()
this.log(`waiting for Allow in ${browser.name}: "${this.ownName}" asks, code ${browser.code}`)
this.notify()
}
Expand Down Expand Up @@ -541,6 +562,7 @@ export class ExtensionLink {
browser.link = "ready"
browser.code = undefined
browser.waited = false
browser.askedAt = undefined
this.log(`${browser.name} allowed "${this.ownName}"; linked`)
// Keep the key only now that the extension knows it: the next start says hello without a code.
if (this.candidate === device) {
Expand All @@ -566,6 +588,7 @@ export class ExtensionLink {
browser.link = reason === "expired" ? "idle" : "denied"
browser.reason = reason
browser.waited = false
browser.askedAt = undefined
browser.code = undefined
this.log(`${browser.name} answered ${reason}`)
// That pairing request is over: the next one gets a fresh key and code.
Expand Down
Loading
Loading