From eaf0d9f462dd3efdc4833ba19325e850dba8fbc2 Mon Sep 17 00:00:00 2001 From: Charlie Gleason Date: Wed, 23 Sep 2026 14:04:50 -0700 Subject: [PATCH] fix(cloudflare): sync reasoning controls from model schemas --- packages/core/src/sync/index.ts | 14 +- .../sync/providers/cloudflare-workers-ai.ts | 135 +++++++-- .../test/cloudflare-workers-ai-sync.test.ts | 280 ++++++++++++++++++ 3 files changed, 397 insertions(+), 32 deletions(-) create mode 100644 packages/core/test/cloudflare-workers-ai-sync.test.ts diff --git a/packages/core/src/sync/index.ts b/packages/core/src/sync/index.ts index 031341b56cd..009d04b0ca1 100644 --- a/packages/core/src/sync/index.ts +++ b/packages/core/src/sync/index.ts @@ -93,6 +93,8 @@ export interface SyncProvider { /** Replace existing leading comments with translateModel.header. */ authoritativeHeaders?: boolean; sameModel?(current: ExistingModel, desired: SyncedModel): boolean; + /** Refresh provider-owned comments while preserving unrelated authored notes. */ + updateHeader?(current: string, generated: string): string; missingNotice?(paths: string[]): string[]; /** * Remote ID to report when translateModel skips a source model. Return @@ -368,12 +370,15 @@ export async function syncProvider( throw parsed.error; } + const currentHeader = existing.get(relativePath)?.header ?? ""; const translatedHeader = translated.header === undefined ? undefined : leadingComments(translated.header); - const header = provider.authoritativeHeaders - ? translatedHeader ?? "" - : (existing.get(relativePath)?.header || translatedHeader) ?? ""; + const header = translatedHeader !== undefined && provider.updateHeader !== undefined + ? provider.updateHeader(currentHeader, translatedHeader) + : provider.authoritativeHeaders + ? translatedHeader ?? "" + : currentHeader || translatedHeader || ""; desired.set(relativePath, { model: parsed.data, content: header + formatToml(parsed.data), @@ -448,7 +453,8 @@ export async function syncProvider( continue; } - const headerChanged = provider.authoritativeHeaders && current.header !== file.header; + const headerChanged = (provider.authoritativeHeaders || provider.updateHeader !== undefined) + && current.header !== file.header; if ( headerChanged || !(provider.sameModel?.(current.authored, file.model) diff --git a/packages/core/src/sync/providers/cloudflare-workers-ai.ts b/packages/core/src/sync/providers/cloudflare-workers-ai.ts index 0ed0e391513..7b7fc7a5b69 100644 --- a/packages/core/src/sync/providers/cloudflare-workers-ai.ts +++ b/packages/core/src/sync/providers/cloudflare-workers-ai.ts @@ -1,7 +1,8 @@ import { z } from "zod"; -import { readFileSync, readdirSync } from "node:fs"; +import { readdirSync } from "node:fs"; import path from "node:path"; +import { ReasoningOption } from "../../schema.js"; import type { ExistingModel, SyncedFullModel, SyncedModel, SyncProvider } from "../index.js"; import { MissingReasoningOptionsError } from "../missing-reasoning-options.js"; import { buildOpenRouterModel, OpenRouterModel } from "./openrouter.js"; @@ -34,7 +35,10 @@ const METADATA_PUBLISHERS: Record = { }; const WorkersAiReasoning = OpenRouterModel.shape.reasoning.unwrap().partial({ mandatory: true }); -const WorkersAiModel = OpenRouterModel.extend({ reasoning: WorkersAiReasoning.optional() }); +const WorkersAiModel = OpenRouterModel.extend({ + reasoning: WorkersAiReasoning.optional(), + input_schema: z.record(z.unknown()).optional(), +}); type WorkersAiModel = z.infer; const CloudflareModel = z.object({ @@ -54,6 +58,7 @@ const CloudflareModel = z.object({ supported_sampling_parameters: z.array(z.string()).optional(), // Validate reasoning separately so malformed controls do not discard the model. reasoning: z.unknown().optional(), + input_schema: z.record(z.unknown()).optional(), }).passthrough(); const CloudflareResponse = z.object({ @@ -85,7 +90,14 @@ export const cloudflareWorkersAi = { name: "Cloudflare Workers AI", modelsDir: "providers/cloudflare-workers-ai/models", deleteMissing: false, - authoritativeHeaders: true, + updateHeader(current, generated) { + const notes = current.split("\n").filter((line) => { + const trimmed = line.trim(); + return trimmed !== "" + && !/^# (?:Reasoning controls declared by Cloudflare's model input schema\.|Toggle:|Effort:|Budget:)/.test(trimmed); + }); + return [...new Set([...generated.trim().split("\n"), ...notes])].join("\n") + "\n"; + }, async fetchModels() { const accountID = process.env.CLOUDFLARE_WORKERS_AI_SYNC_ACCOUNT_ID; const token = process.env.CLOUDFLARE_WORKERS_AI_SYNC_API_TOKEN; @@ -104,7 +116,14 @@ export const cloudflareWorkersAi = { if (response !== undefined) rows.push(...modelRows(response)); } - return { data: rows }; + const models = parseCloudflareModels({ data: rows }); + for (let i = 0; i < models.length; i += 4) { + await Promise.all(models.slice(i, i + 4).map(async (model) => { + model.input_schema = await fetchSchema(accountID, token, model.id.replace(/^workers-ai\//, "")); + })); + } + + return { data: models }; }, parseModels(raw) { const models = parseCloudflareModels(raw); @@ -114,35 +133,46 @@ export const cloudflareWorkersAi = { translateModel(model, context) { const id = model.id; const existing = context.existing(id); - if (existing === undefined && hasReasoning(model) && workersAiReasoningOptions(model) === undefined) { + const apiControls = workersAiReasoningOptions(model); + const controls = apiControls === undefined + && model.supported_parameters.some((value) => value === "reasoning" || value === "include_reasoning") + ? schemaReasoningOptions(model.input_schema) + : undefined; + if ( + existing === undefined + && model.input_schema !== undefined + && hasReasoning(model) + && controls === undefined + && apiControls === undefined + ) { throw new MissingReasoningOptionsError(id, "Workers AI Search does not specify concrete reasoning controls; manual authoring is needed"); } - const translated = buildWorkersAiModel(model, existing); - // Only read paths already present in the sync runner's catalogue map. - const header = existing === undefined ? "" : modelHeader(path.resolve(ROOT_DIR, this.modelsDir, `${id}.toml`)); - if (header === undefined) return undefined; - // Remove only the exact generated line; preserve every other leading comment. - const preservedHeader = header.replace(TOGGLE_HEADER, ""); - // Respect curated model-specific wire instructions, including Qwen's split lines. - const hasToggleWire = /\b(?:thinking\.type|chat_template_kwargs\.thinking|enable_thinking)\b/.test(preservedHeader); + const translated = { + ...buildWorkersAiModel(model, existing), + ...(controls && { reasoning_options: controls.options }), + }; + if (controls !== undefined) return { id, model: translated, header: controls.header }; + const toggleHeader = ENABLE_THINKING_MODELS.has(id) ? TOGGLE_HEADER : ""; - if (translated.reasoning_options?.some((option) => option.type === "toggle") && !hasToggleWire && !toggleHeader) { - if (existing === undefined) { - throw new MissingReasoningOptionsError(id, "Workers AI toggle wire path needs manual verification"); - } + if ( + translated.reasoning_options?.some((option) => option.type === "toggle") + && model.input_schema !== undefined + && !toggleHeader + ) { + if (existing === undefined) throw new MissingReasoningOptionsError(id, "Workers AI toggle wire path needs manual verification"); console.warn(`Keeping catalogue reasoning for ${id}: Workers AI toggle wire path is unknown`); // Keep the reasoning boundary small: other usable properties still update. return { id, model: buildWorkersAiModel({ ...model, reasoning: undefined }, existing), - header, }; } return { id, model: translated, - header: (translated.reasoning_options?.some((option) => option.type === "toggle") && !hasToggleWire ? toggleHeader : "") - + preservedHeader, + header: translated.reasoning_options?.some((option) => option.type === "toggle") + ? toggleHeader || undefined + : undefined, }; }, } satisfies SyncProvider; @@ -153,15 +183,64 @@ function hasReasoning(model: WorkersAiModel) { || model.reasoning?.supports_max_tokens === true; } -function modelHeader(file: string) { - try { - const lines = readFileSync(file, "utf8").split("\n"); - const firstKey = lines.findIndex((line) => line.trim() !== "" && !line.trim().startsWith("#")); - return lines.slice(0, firstKey === -1 ? lines.length : firstKey).join("\n") + "\n"; - } catch (error) { - console.warn(`Skipping Workers AI model with an unreadable catalogue header: ${file}`, error); - return undefined; +const SchemaNode = z.object({ + type: z.string().optional(), + enum: z.array(z.unknown()).optional(), + const: z.unknown().optional(), + properties: z.record(z.unknown()).optional(), + anyOf: z.array(z.unknown()).optional(), + oneOf: z.array(z.unknown()).optional(), +}); + +// Follow request properties and alternatives, not arbitrary nested message/tool schemas. +function schemaFields(input: unknown, path: string[]): z.infer[] { + const parsed = SchemaNode.safeParse(input); + if (!parsed.success) return []; + const node = parsed.data; + const fields = path.length === 0 ? [node] : schemaFields(node.properties?.[path[0]!], path.slice(1)); + return fields.concat((node.anyOf ?? []).concat(node.oneOf ?? []).flatMap((branch) => schemaFields(branch, path))); +} + +function schemaReasoningOptions(input: unknown) { + const options: z.infer[] = []; + const paths = [["reasoning_effort"], ["reasoning", "effort"]]; + const values = [...new Set(paths.flatMap((path) => schemaFields(input, path)) + .flatMap((node) => node.enum ?? []).filter((value) => typeof value === "string"))]; + const effort = ReasoningOption.safeParse({ type: "effort", values }); + const toggle = schemaFields(input, ["chat_template_kwargs", "enable_thinking"]) + .some((node) => node.type === "boolean" && node.const === undefined && + (node.enum === undefined || (node.enum.includes(true) && node.enum.includes(false)))); + const comments = ["# Reasoning controls declared by Cloudflare's model input schema."]; + if (toggle && !values.includes("none")) { + options.push({ type: "toggle" }); + comments.push("# Toggle: chat_template_kwargs.enable_thinking = true|false"); + } + if (effort.success && values.some((value) => value !== "none")) { + options.push(effort.data); + for (const path of paths) { + if (schemaFields(input, path).some((node) => node.enum?.length)) { + comments.push(`# Effort: ${path.join(".")} = ${values.join("|")}`); + } + } + } + if (options.length === 0) return undefined; + return { options, header: comments.join("\n") + "\n" }; +} + +async function fetchSchema(accountID: string, token: string, model: string) { + const url = new URL(`${API_BASE}/${accountID}/ai/models/schema`); + url.searchParams.set("model", model); + const response = await fetch(url, { + headers: { Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) { + throw new Error(`Cloudflare Workers AI schema request failed for ${model}: ${response.status}`); } + return z.object({ + success: z.literal(true), + result: z.object({ input: z.record(z.unknown()) }), + }).parse(await response.json()).result.input; } export function buildWorkersAiModel( diff --git a/packages/core/test/cloudflare-workers-ai-sync.test.ts b/packages/core/test/cloudflare-workers-ai-sync.test.ts new file mode 100644 index 00000000000..5622e4176bc --- /dev/null +++ b/packages/core/test/cloudflare-workers-ai-sync.test.ts @@ -0,0 +1,280 @@ +import { expect, spyOn, test } from "bun:test"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; + +import { formatToml, syncProvider, type ExistingModel } from "../src/sync/index.js"; +import { cloudflareWorkersAi } from "../src/sync/providers/cloudflare-workers-ai.js"; + +// Synthetic metadata exercises the sync contract, not a real model's controls. +const model = { + id: "@cf/example/reasoner", + name: "Example Reasoner", + created: 1_787_702_400, + hugging_face_id: null, + knowledge_cutoff: null, + context_length: 128_000, + architecture: { input_modalities: ["text"], output_modalities: ["text"] }, + pricing: { prompt: "0.000001", completion: "0.000002" }, + top_provider: { context_length: 128_000, max_completion_tokens: 16_000 }, + supported_parameters: ["reasoning", "tools", "temperature"], +}; + +function translate(raw: unknown, existing?: ExistingModel) { + const [parsed] = cloudflareWorkersAi.parseModels(raw); + return cloudflareWorkersAi.translateModel(parsed!, { + existing: () => existing, + authored: () => existing, + }).model; +} + +const inputSchema = { + anyOf: [{ oneOf: [{ + type: "object", + properties: { + reasoning_effort: { anyOf: [{ type: "string", enum: ["low", "medium", "high"] }, { type: "null" }] }, + chat_template_kwargs: { + type: "object", + properties: { enable_thinking: { type: "boolean", default: true } }, + }, + max_tokens: { type: "integer", maximum: 16_000 }, + }, + }] }], +}; + +test("derives effort and toggle controls from the model input schema", () => { + const synced = translate({ data: [{ ...model, input_schema: inputSchema }] }, { reasoning_options: [] }); + expect(synced.reasoning_options).toEqual([ + { type: "toggle" }, + { type: "effort", values: ["low", "medium", "high"] }, + ]); + + const [parsed] = cloudflareWorkersAi.parseModels({ data: [{ ...model, input_schema: inputSchema }] }); + const translated = cloudflareWorkersAi.translateModel(parsed!, { + existing: () => undefined, + authored: () => undefined, + }); + expect(translated.header).toContain("chat_template_kwargs.enable_thinking = true|false"); +}); + +test("reads nested Responses API effort enums without inventing a toggle", () => { + const synced = translate({ data: [{ ...model, input_schema: { + oneOf: [{ type: "object", properties: { reasoning: { + type: "object", properties: { effort: { type: "string", enum: ["low", "high"] } }, + } } }], + } }] }); + expect(synced.reasoning_options).toEqual([{ type: "effort", values: ["low", "high"] }]); +}); + +test("refreshes serialized control comments without losing authored notes or resyncing forever", async () => { + const directory = await mkdtemp("/tmp/opencode/workers-schema-"); + const raw = { data: [{ ...model, input_schema: inputSchema }] }; + const file = `${directory}/${model.id}.toml`; + try { + await mkdir(`${directory}/@cf/example`, { recursive: true }); + await Bun.write(file, "# Context limit and reasoning behavior verified separately.\n# Toggle: thinking.type = enabled|disabled\n# Effort: reasoning_effort = high|max\n" + formatToml({ id: model.id, ...translate(raw) })); + const provider = { ...cloudflareWorkersAi, modelsDir: directory, fetchModels: async () => raw }; + expect((await syncProvider(provider)).updated).toBe(1); + const written = await Bun.file(file).text(); + expect(written).toContain("# Context limit and reasoning behavior verified separately."); + expect(written).toContain("# Toggle: chat_template_kwargs.enable_thinking = true|false"); + expect(written).toContain("# Effort: reasoning_effort = low|medium|high"); + expect(written).not.toContain("thinking.type"); + expect(written).not.toContain("high|max"); + expect((await syncProvider(provider)).updated).toBe(0); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("schema effort none replaces a separate toggle", () => { + const synced = translate({ data: [{ ...model, input_schema: { properties: { + reasoning_effort: { type: "string", enum: ["none", "low", "high"] }, + chat_template_kwargs: { properties: { enable_thinking: { type: "boolean" } } }, + } } }] }); + expect(synced.reasoning_options).toEqual([{ type: "effort", values: ["none", "low", "high"] }]); +}); + +test("does not emit a separate toggle header when effort includes none", () => { + const [parsed] = cloudflareWorkersAi.parseModels({ data: [{ + ...model, + id: "@cf/google/gemma-4-26b-a4b-it", + input_schema: {}, + reasoning: { mandatory: false, supported_efforts: ["none", "high"] }, + }] }); + const translated = cloudflareWorkersAi.translateModel(parsed!, { + existing: () => undefined, + authored: () => undefined, + }); + + expect(translated.header).toBeUndefined(); + expect(translated.model.reasoning_options).toEqual([{ type: "effort", values: ["none", "high"] }]); +}); + +test("prefers model-specific reasoning metadata over generic schema controls", () => { + const synced = translate({ data: [{ + ...model, + input_schema: inputSchema, + reasoning: { mandatory: false, supported_efforts: ["none", "high"] }, + }] }); + + expect(synced.reasoning_options).toEqual([{ type: "effort", values: ["none", "high"] }]); +}); + +test.each([ + {}, + { properties: { reasoning_effort: { type: "string" }, max_tokens: { type: "integer", maximum: 1000 } } }, + { properties: { reasoning_effort: { enum: [null, "unsupported"] } } }, + { properties: { chat_template_kwargs: { properties: { enable_thinking: { type: "boolean", const: true } } } } }, +])("preserves curated options when schema has no usable controls: %j", (schema) => { + const options = [{ type: "effort", values: ["high", "max"] }] satisfies ExistingModel["reasoning_options"]; + expect(translate({ data: [{ ...model, input_schema: schema }] }, { reasoning_options: options }).reasoning_options).toEqual(options); +}); + +test("does not infer reasoning capability from generic schema fields", () => { + const synced = translate({ data: [{ ...model, supported_parameters: ["tools"], input_schema: inputSchema }] }); + expect(synced.reasoning).toBe(false); + expect(synced.reasoning_options).toBeUndefined(); +}); + +test.each([ + { name: "success", status: 200, body: { success: true, result: { input: inputSchema } } }, + { name: "HTTP failure", status: 403, body: {} }, + { name: "API failure", status: 200, body: { success: false } }, + { name: "missing input schema", status: 200, body: { success: true, result: {} } }, +])("fetches each model schema and fails closed on $name", async ({ status, body }) => { + const account = process.env.CLOUDFLARE_WORKERS_AI_SYNC_ACCOUNT_ID; + const token = process.env.CLOUDFLARE_WORKERS_AI_SYNC_API_TOKEN; + process.env.CLOUDFLARE_WORKERS_AI_SYNC_ACCOUNT_ID = "test-account"; + process.env.CLOUDFLARE_WORKERS_AI_SYNC_API_TOKEN = "test-token"; + const requested: string[] = []; + const fetch = spyOn(globalThis, "fetch").mockImplementation((async (input: string | URL | Request) => { + const url = new URL(String(input)); + if (url.pathname.endsWith("/search")) { + return Response.json({ data: [model, { ...model, id: "@cf/example/other" }] }); + } + requested.push(url.searchParams.get("model")!); + return Response.json(body, { status }); + }) as typeof globalThis.fetch); + try { + if (status !== 200 || !("result" in body) || !body.result || !("input" in body.result)) { + await expect(cloudflareWorkersAi.fetchModels()).rejects.toThrow(); + return; + } + const raw = await cloudflareWorkersAi.fetchModels(); + expect(requested.toSorted()).toEqual(["@cf/example/other", model.id]); + expect(translate(raw).reasoning_options).toEqual([ + { type: "toggle" }, { type: "effort", values: ["low", "medium", "high"] }, + ]); + } finally { + fetch.mockRestore(); + if (account === undefined) delete process.env.CLOUDFLARE_WORKERS_AI_SYNC_ACCOUNT_ID; + else process.env.CLOUDFLARE_WORKERS_AI_SYNC_ACCOUNT_ID = account; + if (token === undefined) delete process.env.CLOUDFLARE_WORKERS_AI_SYNC_API_TOKEN; + else process.env.CLOUDFLARE_WORKERS_AI_SYNC_API_TOKEN = token; + } +}); + +test.each([ + ["direct", (value: unknown) => ({ data: [value] })], + ["wrapped", (value: unknown) => ({ result: { data: [value] } })], + ["wrapped array", (value: unknown) => ({ result: [value] })], +] as const)("imports explicit Workers AI reasoning metadata from %s responses", (_, wrap) => { + const synced = translate(wrap({ + ...model, + reasoning: { mandatory: true, supported_efforts: ["low", "high", "max"] }, + })); + + expect(synced.reasoning_options).toEqual([ + { type: "effort", values: ["low", "high", "max"] }, + ]); +}); + +test.each([ + { name: "empty", options: [] }, + { name: "stale", options: [{ type: "effort", values: ["low", "medium", "high"] }] }, +] satisfies { name: string; options: ExistingModel["reasoning_options"] }[])("refreshes $name Workers AI options from API metadata", ({ options }) => { + const synced = translate({ data: [{ + ...model, + reasoning: { mandatory: true, supported_efforts: ["high", "max"] }, + }] }, { + reasoning_options: options, + name: "Curated name", + limit: { context: 128_000, output: 8_000 }, + }); + + expect(synced.reasoning_options).toEqual([{ type: "effort", values: ["high", "max"] }]); + expect(synced.name).toBe("Curated name"); + expect(synced.limit?.output).toBe(8_000); +}); + +test("preserves curated Workers AI options when API metadata is absent", () => { + const options = [{ type: "toggle" }, { type: "budget_tokens", min: 1_024 }] satisfies ExistingModel["reasoning_options"]; + const synced = translate({ data: [model] }, { reasoning_options: options }); + + expect(synced.reasoning_options).toEqual(options); + expect(translate({ data: [model] }).reasoning_options).toBeUndefined(); +}); + +test("imports explicit toggle and budget support without treating completion limits as budgets", () => { + const synced = translate({ data: [{ + ...model, + reasoning: { + mandatory: false, + supported_efforts: ["low", "high"], + supports_max_tokens: true, + }, + }] }); + + expect(synced.reasoning_options).toEqual([ + { type: "toggle" }, + { type: "effort", values: ["low", "high"] }, + { type: "budget_tokens" }, + ]); +}); + +test("retains reasoning metadata when normalizing the native Cloudflare response", () => { + const synced = translate({ data: [{ + id: model.id, + name: model.name, + created: model.created, + context_length: model.context_length, + max_output_length: 16_000, + pricing: model.pricing, + supported_features: ["reasoning", "tools"], + supported_sampling_parameters: ["temperature"], + reasoning: { mandatory: false, supported_efforts: ["none", "low", "high"] }, + }] }); + + expect(synced.reasoning_options).toEqual([ + { type: "effort", values: ["none", "low", "high"] }, + ]); +}); + +test("does not expand null Cloudflare efforts into OpenRouter's full effort enum", () => { + const raw = { result: { data: [{ + ...model, + reasoning: { mandatory: true, supported_efforts: null }, + }] } }; + const options = [{ type: "effort", values: ["low", "high"] }] satisfies ExistingModel["reasoning_options"]; + + expect(translate(raw, { reasoning_options: options }).reasoning_options).toEqual(options); + expect(translate(raw).reasoning_options).toBeUndefined(); +}); + +test.each([ + { reasoning: null }, + { reasoning: { supported_efforts: ["low", "high"] } }, +])("preserves curated options for unusable native reasoning metadata: %j", ({ reasoning }) => { + const options = [{ type: "toggle" }] satisfies ExistingModel["reasoning_options"]; + const synced = translate({ data: [{ + id: model.id, + name: model.name, + created: model.created, + context_length: model.context_length, + max_output_length: 16_000, + pricing: model.pricing, + supported_features: ["reasoning"], + reasoning, + }] }, { reasoning_options: options }); + + expect(synced.reasoning_options).toEqual(options); +});