diff --git a/apps/desktop/e2e/specs/gbc.e2e.ts b/apps/desktop/e2e/specs/gbc.e2e.ts new file mode 100644 index 0000000..3f8a22e --- /dev/null +++ b/apps/desktop/e2e/specs/gbc.e2e.ts @@ -0,0 +1,224 @@ +/** + * 看图拆区域: 拆成区域 in a picture's menu fills a new 画面 with the regions + * KohakuGBC finds, and 设置 › 功能 sets it up. + */ +import { describe, expect, it } from 'vitest' +import type { Page } from 'playwright-core' +import type { ProjectCanvas, Settings } from '@shared/types' +import { settle, useApp, type E2eApp } from '../support/app' + +const PICTURE = '/fixture/projects/project-1/assets/cat-1600x1200.png' + +interface Setup { + settings?: Partial + /** GBC is installed and runs. */ + ready?: boolean + /** The 助手 has a model that can read pictures. */ + provider?: boolean + theme?: 'light' | 'dark' +} + +/** A project with one imported picture (img-1) and one 画面 (canvas-1). */ +async function openBoard(app: E2eApp, setup: Setup = {}): Promise { + const page = await app.open({ + scenario: 'ready', + route: '/studio', + defer: true, + theme: setup.theme, + settings: { gbcEnabled: true, ...setup.settings }, + }) + await page.evaluate( + ({ picturePath, ready, provider }) => { + const { canvasFixture, projectFixture, aiProviderFixture, gbcStatusFixture } = window.__mock.fixtures + const picture = canvasFixture({ + id: 'img-1', + kind: 'image', + title: '小猫照片', + imagePath: picturePath, + imageSize: { width: 1600, height: 1200 }, + x: 0, + y: 0, + width: 1024, + height: 768, + }) + const layout = canvasFixture({ id: 'canvas-1', title: '画面 1', prompt: '一只橘猫坐在窗台上', x: 0, y: 300 }) + window.__mock.state.projects = [ + projectFixture({ id: 'project-1', name: '小猫', canvases: [picture, layout], selectedCanvasId: 'canvas-1' }), + ] + window.__mock.state.aiProviders = provider + ? [ + aiProviderFixture({ id: 'local', kind: 'ollama', name: 'Ollama' }), + aiProviderFixture({ id: 'claude', kind: 'anthropic', name: 'Claude' }), + ] + : [] + window.__mock.state.gbc.status = gbcStatusFixture(ready) + }, + { picturePath: PICTURE, ready: setup.ready ?? true, provider: setup.provider ?? true }, + ) + await app.start() + await page.locator('article[data-canvas-card="img-1"]').waitFor() + await settle(page, 150) + return page +} + +const card = (page: Page, id: string) => page.locator(`article[data-canvas-card="${id}"]`) + +async function pictureMenu(page: Page): Promise { + const body = (await card(page, 'img-1').locator('[data-card-body]').boundingBox())! + await page.mouse.click(body.x + body.width / 2, body.y + body.height / 2, { button: 'right' }) + const menu = page.getByRole('menu', { name: '小猫照片 操作' }) + await menu.waitFor() + return menu.getByRole('menuitem').allTextContents() +} + +async function cardIds(page: Page): Promise { + return page.locator('article[data-canvas-card]').evaluateAll((items) => items.map((item) => item.getAttribute('data-canvas-card')!)) +} + +async function saved(page: Page, id: string): Promise { + return page.evaluate((canvasId) => window.__mock.state.projects[0]?.canvases.find((canvas) => canvas.id === canvasId), id) +} + +describe('拆成区域', () => { + const app = useApp() + + it('is not offered while 看图拆区域 is off', async () => { + const page = await openBoard(app, { settings: { gbcEnabled: false } }) + expect(await pictureMenu(page)).toEqual(['用作起始图', '放大', '复制', '删除']) + expect(await app.calls('gbc:status')).toEqual([]) + }) + + it('fills a new 画面 beside the picture with its regions and description', async () => { + const page = await openBoard(app) + expect(await pictureMenu(page)).toEqual(['用作起始图', '拆成区域', '放大', '复制', '删除']) + await app.screenshot('gbc-picture-menu-light') + await page.getByRole('menuitem', { name: '拆成区域' }).click() + + await expect.poll(async () => (await cardIds(page)).length).toBe(3) + const created = (await cardIds(page)).find((id) => id !== 'img-1' && id !== 'canvas-1')! + const call = (await app.calls<{ runId: string; path: string }>('gbc:parse'))[0]! + expect(call.path).toBe(PICTURE) + await expect.poll(() => card(page, created).locator('[data-region-id]').count()).toBe(2) + await expect.poll(() => page.getByText('已拆出 2 个区域').isVisible()).toBe(true) + + await expect.poll(() => saved(page, created).then((item) => item?.layers.length), { timeout: 10_000 }).toBe(2) + const canvas = await saved(page, created) + expect(canvas!.prompt).toBe('午后的窗台,阳光照在一只睡着的橘猫身上') + expect(canvas!.layers.map((region) => [region.kind, region.caption])).toEqual([ + ['object', '趴着睡觉的橘猫'], + ['object', '窗外的晴天'], + ]) + // Shaped like the 4:3 picture, and only its layout: no wire from the picture. + expect(canvas!.width / canvas!.height).toBeCloseTo(4 / 3, 1) + expect(canvas!.inputs).toBeUndefined() + expect(await page.locator(`[data-wire-chip^="input:${created}:"]`).count()).toBe(0) + const source = (await card(page, 'img-1').boundingBox())! + const fresh = (await card(page, created).boundingBox())! + expect(fresh.x).toBeGreaterThan(source.x + source.width) + expect(await card(page, created).getAttribute('aria-current')).toBe('true') + await app.screenshot('gbc-regions-light') + }) + + it('shows the picture faintly while reading; 停止 takes the empty 画面 away', async () => { + const page = await openBoard(app, { theme: 'dark' }) + // Reading takes as long as the test wants; 停止 ends it the way main does, answering null. + await page.evaluate(() => { + const mock = window.__mock as typeof window.__mock & { release?: (value: null) => void } + mock.handle(mock.Ipc.gbcParse, (payload, { emit }) => { + emit(mock.Events.gbcParse, { runId: (payload as { runId: string }).runId, phase: 'download', percent: 40 }) + return new Promise((resolve) => { + mock.release = resolve + }) + }) + mock.handle(mock.Ipc.gbcCancel, () => { + mock.release?.(null) + return true + }) + }) + await pictureMenu(page) + await page.getByRole('menuitem', { name: '拆成区域' }).click() + const status = page.locator('[data-card-status="parsing"]') + await status.waitFor() + await expect.poll(() => status.textContent()).toContain('正在下载看图模型') + expect(await status.textContent()).toContain('40% · 只有第一次') + await app.screenshot('gbc-reading-dark') + + await status.getByRole('button', { name: '停止' }).click() + const runId = (await app.calls<{ runId: string }>('gbc:parse'))[0]!.runId + expect(await app.calls('gbc:cancel')).toEqual([{ runId }]) + await expect.poll(() => cardIds(page)).toEqual(['img-1', 'canvas-1']) + }) + + it('a failure says why and takes the empty 画面 away', async () => { + const page = await openBoard(app) + await page.evaluate(() => window.__mock.fail(window.__mock.Ipc.gbcParse, { code: 'E_GBC', message: '没在这张图里找到物体' })) + await pictureMenu(page) + await page.getByRole('menuitem', { name: '拆成区域' }).click() + await expect.poll(() => page.getByText('没拆成:没在这张图里找到物体').isVisible()).toBe(true) + await expect.poll(() => cardIds(page)).toEqual(['img-1', 'canvas-1']) + }) + + it('says what is missing, and leads to 设置', async () => { + const page = await openBoard(app, { provider: false }) + await pictureMenu(page) + const entry = page.getByRole('menuitem', { name: /拆成区域/ }) + expect(await entry.textContent()).toBe('拆成区域 先给助手加一个模型') + + const notInstalled = await openBoard(app, { ready: false }) + await pictureMenu(notInstalled) + expect(await notInstalled.getByRole('menuitem', { name: /拆成区域/ }).textContent()).toBe('拆成区域 看图还没装好') + await notInstalled.getByRole('menuitem', { name: /拆成区域/ }).click() + await expect.poll(() => notInstalled.evaluate(() => window.location.hash)).toBe('#/settings/features') + expect(await app.calls('gbc:parse')).toEqual([]) + }) +}) + +describe('设置 › 功能 › 看图拆区域', () => { + const app = useApp() + + async function openSettings(theme: 'light' | 'dark' = 'light'): Promise { + const page = await app.open({ route: '/settings/features', defer: true, theme }) + await page.evaluate(() => { + const { aiProviderFixture } = window.__mock.fixtures + window.__mock.state.aiProviders = [ + aiProviderFixture({ id: 'local', kind: 'ollama', name: 'Ollama', model: 'llava' }), + aiProviderFixture({ id: 'claude', kind: 'anthropic', name: 'Claude', model: 'claude-sonnet' }), + aiProviderFixture({ id: 'gpt', kind: 'openai', name: 'GPT', model: 'gpt-x' }), + ] + }) + await app.start() + return page + } + + it('turning it on shows its setup; 安装 makes it ready', async () => { + const page = await openSettings() + expect(await page.getByRole('heading', { name: '看图拆区域' }).count()).toBe(0) + await page.getByRole('switch', { name: '看图拆区域' }).click() + await expect.poll(() => app.calls('settings:update').then((calls) => calls.at(-1))).toEqual({ gbcEnabled: true }) + await page.getByRole('heading', { name: '看图拆区域' }).waitFor() + + // Only models that can read pictures are offered. + const model = page.getByRole('combobox', { name: '看图的模型' }) + await expect.poll(() => model.locator('option').allTextContents()).toEqual(['Claude · claude-sonnet', 'GPT · gpt-x']) + await model.selectOption('gpt') + await expect.poll(() => app.calls('settings:update').then((calls) => calls.at(-1))).toEqual({ gbcProviderId: 'gpt' }) + + await page.getByRole('button', { name: '安装' }).waitFor() + await app.screenshot('gbc-settings-light') + await page.getByRole('button', { name: '安装' }).click() + await page.getByText('可以用', { exact: true }).waitFor() + expect(await app.calls('gbc:setup')).toHaveLength(1) + + await page.getByRole('button', { name: /^高级/ }).click() + await expect.poll(() => page.getByText('正在使用').isVisible()).toBe(true) + await page.getByRole('combobox', { name: '找物体的方式' }).selectOption('sam3') + await expect.poll(() => app.calls('settings:update').then((calls) => calls.at(-1))).toEqual({ gbcDetector: 'sam3' }) + }) + + it('says to add a model first when the 助手 has none that reads pictures', async () => { + const page = await app.open({ route: '/settings/features', settings: { gbcEnabled: true }, theme: 'dark' }) + await page.getByText('先在助手里添加一个 OpenAI 兼容或 Anthropic 的模型,并填好 API Key。').waitFor() + expect(await page.getByRole('combobox', { name: '看图的模型' }).count()).toBe(0) + await app.screenshot('gbc-settings-dark') + }) +}) diff --git a/apps/desktop/e2e/specs/visual.e2e.ts b/apps/desktop/e2e/specs/visual.e2e.ts index 68f1c0f..aa09d50 100644 --- a/apps/desktop/e2e/specs/visual.e2e.ts +++ b/apps/desktop/e2e/specs/visual.e2e.ts @@ -18,7 +18,7 @@ interface Stop { prepare?: (page: Page) => Promise } -const ALL_FEATURES = { trainingEnabled: true, remoteHostsEnabled: true, lanWebEnabled: true } +const ALL_FEATURES = { trainingEnabled: true, remoteHostsEnabled: true, gbcEnabled: true, lanWebEnabled: true } /** A project whose canvas has a description, two regions and a finished run with four outputs. */ async function seedStudio(page: Page): Promise { diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 64df92c..c420daa 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -8,7 +8,7 @@ import { runStudioCanvasUiTests } from './studioCanvasUi.test' import { ensureDirs, paths } from './lib/paths' import { getBackendInfo, getWebAccessInfo, startBackend, stopBackend } from './services/backend' import { listProviders, removeProvider, saveProvider } from './services/ai' -import { gbcStatus } from './services/gbc' +import { gbcStatus, stopGbcProcesses } from './services/gbc' import type { JobRecord } from '@shared/types' import { localNotificationsSupported, pushTestNotification } from './services/desktopNotifications' import { checkBareMirror } from './services/gitHealth' @@ -2167,6 +2167,7 @@ app.on('window-all-closed', () => { }) app.on('before-quit', () => { + stopGbcProcesses() closeAllSshTerminals() stopUpdater() stopBackend() diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index db2c329..4c2f9b1 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -47,7 +47,7 @@ import { renameConversation, saveProvider, } from './services/ai' -import { gbcStatus, installGbcEnv, listGbcImages, runGbc } from './services/gbc' +import { cancelParse, forgetGbcStatus, gbcStatus, installGbc, installGbcEnv, listGbcImages, parseRegions, runGbc } from './services/gbc' import { nativeImage } from 'electron' import { createGbcVersion, listGbcBranches, listGbcCommits, listGbcVersions, removeGbcVersion } from './services/gbcGit' import { closeSshTerminal, inputSshTerminal, listSshTerminals, openSshTerminal } from './services/sshSessions' @@ -232,9 +232,18 @@ export function registerIpc(): void { }), ) - handle(Ipc.gbcStatus, schemas.gbcStatusQuery, ({ versionId }) => gbcStatus(versionId)) + handle(Ipc.gbcStatus, schemas.gbcStatusQuery, ({ versionId, fresh }) => gbcStatus(versionId, fresh)) - handle(Ipc.gbcDoctor, schemas.gbcStatusQuery, ({ versionId }) => gbcStatus(versionId)) + handle(Ipc.gbcDoctor, schemas.gbcStatusQuery, ({ versionId }) => gbcStatus(versionId, true)) + + handle(Ipc.gbcSetup, null, () => installGbc()) + + handle(Ipc.gbcParse, schemas.gbcParse, ({ runId, path }) => parseRegions(runId, path)) + + handle(Ipc.gbcCancel, schemas.gbcCancel, ({ runId }) => { + cancelParse(runId) + return true + }) handle(Ipc.gbcInstallEnv, schemas.gbcInstallEnv, ({ versionId }) => installGbcEnv(versionId)) @@ -269,6 +278,8 @@ export function registerIpc(): void { handle(Ipc.gbcVersionRemove, schemas.gbcVersionRemove, async ({ id }) => { await removeGbcVersion(getSettings(), id) + if (getSettings().gbcVersionId === id) emit(Events.settingsChanged, updateSettings({ gbcVersionId: null })) + forgetGbcStatus() return true }) diff --git a/apps/desktop/src/main/lib/run.ts b/apps/desktop/src/main/lib/run.ts index 4dd4fe1..85a44dc 100644 --- a/apps/desktop/src/main/lib/run.ts +++ b/apps/desktop/src/main/lib/run.ts @@ -5,6 +5,10 @@ export interface RunOptions { env?: NodeJS.ProcessEnv timeoutMs?: number input?: string + /** Ends the process (`runStream` only); the result reports it as `aborted`. */ + signal?: AbortSignal + /** `runStream` only: a lone carriage return also ends a line, so progress bars arrive as they redraw. */ + splitCarriageReturn?: boolean } export interface RunResult { @@ -12,6 +16,7 @@ export interface RunResult { stdout: string stderr: string timedOut: boolean + aborted?: boolean } export function run(command: string, args: string[], options: RunOptions = {}): Promise { @@ -84,6 +89,16 @@ export function runStream( timedOut = true child.kill('SIGKILL') }, options.timeoutMs ?? 3_600_000) + const abort = (): void => { + child.kill('SIGKILL') + } + if (options.signal?.aborted) abort() + else options.signal?.addEventListener('abort', abort, { once: true }) + const done = (result: RunResult): void => { + clearTimeout(timer) + options.signal?.removeEventListener('abort', abort) + resolve(options.signal?.aborted ? { ...result, aborted: true } : result) + } const pipe = (stream: 'stdout' | 'stderr', source: NodeJS.ReadableStream | null): void => { let buffer = '' @@ -92,7 +107,7 @@ export function runStream( if (stream === 'stdout') stdout += chunk else stderr += chunk buffer += chunk - const parts = buffer.split(/\r?\n/) + const parts = buffer.split(options.splitCarriageReturn ? /\r\n|\r|\n/ : /\r?\n/) buffer = parts.pop() ?? '' for (const line of parts) { if (line.trim()) onLine(line, stream) @@ -108,14 +123,8 @@ export function runStream( child.stdin?.on('error', () => undefined) child.stdin?.end(options.input) - child.on('error', (error) => { - clearTimeout(timer) - resolve({ code: -1, stdout, stderr: stderr || String(error), timedOut }) - }) - child.on('close', (code) => { - clearTimeout(timer) - resolve({ code: code ?? -1, stdout, stderr, timedOut }) - }) + child.on('error', (error) => done({ code: -1, stdout, stderr: stderr || String(error), timedOut })) + child.on('close', (code) => done({ code: code ?? -1, stdout, stderr, timedOut })) }) } diff --git a/apps/desktop/src/main/services/gbc.ts b/apps/desktop/src/main/services/gbc.ts index 1f2aecc..31f759a 100644 --- a/apps/desktop/src/main/services/gbc.ts +++ b/apps/desktop/src/main/services/gbc.ts @@ -1,13 +1,28 @@ import { existsSync, readFileSync } from 'node:fs' -import { mkdir, readdir, writeFile } from 'node:fs/promises' +import { mkdir, readdir, rm, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { randomUUID } from 'node:crypto' -import type { GbcCheck, GbcGraph, GbcRunInput, GbcRunResult, GbcStatus } from '@shared/gbc' +import { Events } from '@shared/channels' +import { + gbcCanUse, + regionsFromGraph, + type GbcCheck, + type GbcGraph, + type GbcParseProgress, + type GbcParseResult, + type GbcRunInput, + type GbcRunResult, + type GbcSetupProgress, + type GbcStatus, +} from '@shared/gbc' +import type { Settings } from '@shared/types' +import { emit } from '../lib/events' import { paths } from '../lib/paths' -import { firstLine, run, tail } from '../lib/run' -import { getProviderSecret } from './ai' -import { listGbcVersions } from './gbcGit' -import { getSettings } from './settings' +import { firstLine, run, runStream, tail } from '../lib/run' +import { getProviderSecret, listProviders } from './ai' +import { createGbcVersion, listGbcBranches, listGbcVersions } from './gbcGit' +import { pickPython, resolveTorchIndex } from './runtimes' +import { getSettings, updateSettings } from './settings' function detectorModel(detector: GbcRunInput['detector']): string { return detector === 'sam3' ? 'facebook/sam3' : 'google/owlv2-base-patch16-ensemble' @@ -44,7 +59,50 @@ function versionPython(version: { path: string }): string { return process.platform === 'win32' ? join(version.path, '.venv', 'Scripts', 'python.exe') : join(version.path, '.venv', 'bin', 'python') } -export async function gbcStatus(versionId?: string): Promise { +/** KohakuGBC's pyproject: `requires-python = ">=3.11"`. */ +const MIN_PYTHON_MINOR = 11 + +/** Every GBC process this app started: 停止 and quitting end them. */ +const owned = new Set() +/** 拆成区域 runs by the id the renderer gave them, for 停止. */ +const parses = new Map() +let setup: Promise | null = null + +function ownedController(): AbortController { + const controller = new AbortController() + owned.add(controller) + return controller +} + +/** Quitting ends every GBC run and install still going. */ +export function stopGbcProcesses(): void { + for (const controller of owned) controller.abort() + owned.clear() + parses.clear() +} + +/** The checks spawn Python and import torch, which takes seconds; they are kept until something changes. */ +const statusCache = new Map>() + +export function forgetGbcStatus(): void { + statusCache.clear() +} + +/** + * Whether GBC can run. `versionId` defaults to the version 设置 › 功能 uses; + * `fresh` checks again instead of answering from the last check. + */ +export function gbcStatus(versionId?: string, fresh = false): Promise { + const id = versionId ?? getSettings().gbcVersionId ?? '' + const cached = statusCache.get(id) + if (cached && !fresh) return cached + const next = checkGbc(id || undefined) + statusCache.set(id, next) + next.catch(() => statusCache.delete(id)) + return next +} + +async function checkGbc(versionId?: string): Promise { const settings = getSettings() const version = versionId ? (await listGbcVersions(settings)).find((item) => item.id === versionId) : null const bundled = version ? versionBinary(version) : '' @@ -92,30 +150,94 @@ export async function gbcStatus(versionId?: string): Promise { } } -export async function installGbcEnv(versionId: string): Promise<{ ok: boolean; output: string; status: GbcStatus }> { +/** + * The Python environment of one version: a venv beside its code, PyTorch from + * the index the trainer uses (PyPI's Windows wheel is CPU only), then the + * version itself with its torch extra. KohakuGBC is not on PyPI, so it is + * always installed from its code. + */ +export async function installGbcEnv(versionId: string, signal?: AbortSignal): Promise<{ ok: boolean; output: string; status: GbcStatus }> { const settings = getSettings() const version = (await listGbcVersions(settings)).find((item) => item.id === versionId) if (!version) throw new Error('GBC 版本不存在') - const base = settings.pythonPath || (process.platform === 'win32' ? 'python' : 'python3') + if (!existsSync(join(version.path, 'pyproject.toml'))) throw new Error('这个 GBC 版本缺少 pyproject.toml,无法安装') + const controller = signal ? null : ownedController() + const abort = signal ?? controller!.signal const venv = join(version.path, '.venv') const py = versionPython(version) + const env = { PIP_CACHE_DIR: join(paths.cache(), 'pip') } const lines: string[] = [] const runStep = async (command: string, args: string[], timeoutMs: number): Promise => { - const result = await run(command, args, { cwd: version.path, timeoutMs }) lines.push(`$ ${command} ${args.join(' ')}`) - if (result.stdout) lines.push(result.stdout.trim()) - if (result.stderr) lines.push(result.stderr.trim()) + const result = await runStream(command, args, { cwd: version.path, timeoutMs, env, signal: abort }, (line) => lines.push(line)) + if (result.aborted) throw new Error('安装已停止') if (result.code !== 0) throw new Error(`安装步骤失败:${tail(result.stderr || result.stdout, 30)}`) } - if (!existsSync(py)) await runStep(base, ['-m', 'venv', venv], 180_000) - await runStep(py, ['-m', 'pip', 'install', '--upgrade', 'pip'], 300_000) - const local = await run(py, ['-c', 'import pathlib; print(pathlib.Path("pyproject.toml").exists())'], { - cwd: version.path, - timeoutMs: 5000, - }) - if (local.stdout.trim() === 'True') await runStep(py, ['-m', 'pip', 'install', '-e', '.[torch]'], 30 * 60_000) - else await runStep(py, ['-m', 'pip', 'install', 'kohakugbc[torch]'], 30 * 60_000) - return { ok: true, output: lines.join('\n').slice(-12000), status: await gbcStatus(versionId) } + try { + if (!existsSync(py)) { + const base = await pickPython(settings) + const minor = Number.parseInt(base.version.split('.')[1] ?? '0', 10) + if (minor < MIN_PYTHON_MINOR) throw new Error(`看图需要 Python 3.${MIN_PYTHON_MINOR} 或更新版本,现在找到的是 ${base.version}`) + await runStep(base.path, ['-m', 'venv', venv], 180_000) + } + await runStep(py, ['-m', 'pip', 'install', '--upgrade', 'pip'], 300_000) + const torchIndex = await resolveTorchIndex(settings) + if (torchIndex) await runStep(py, ['-m', 'pip', 'install', '--index-url', torchIndex, 'torch', 'torchvision'], 60 * 60_000) + const pipIndex = settings.pipIndex ? ['--index-url', settings.pipIndex] : [] + await runStep(py, ['-m', 'pip', 'install', ...pipIndex, '-e', '.[torch]'], 60 * 60_000) + } finally { + if (controller) owned.delete(controller) + forgetGbcStatus() + } + return { ok: true, output: lines.join('\n').slice(-12000), status: await gbcStatus(versionId, true) } +} + +function setupProgress(progress: GbcSetupProgress): void { + emit(Events.gbcSetup, progress) +} + +/** + * 安装 in 设置 › 功能: the version chosen there, or the newest commit of + * KohakuGBC's default branch when none is, made ready to run. A second click + * while it runs waits for the same install. + */ +export function installGbc(): Promise { + if (setup) return setup + const controller = ownedController() + setup = (async () => { + try { + setupProgress({ phase: 'code' }) + const settings = getSettings() + const versions = await listGbcVersions(settings) + let version = versions.find((item) => item.id === settings.gbcVersionId) + if (!version) { + const branches = await listGbcBranches(settings) + const branch = branches.find((item) => item.isDefault) ?? branches[0]! + version = await createGbcVersion(settings, branch.name, branch.sha) + emit(Events.settingsChanged, updateSettings({ gbcVersionId: version.id })) + } + if (controller.signal.aborted) throw new Error('安装已停止') + setupProgress({ phase: 'packages' }) + const { status } = await installGbcEnv(version.id, controller.signal) + setupProgress({ phase: 'done' }) + return status + } catch (error) { + setupProgress({ phase: 'failed', error: error instanceof Error ? error.message : String(error) }) + throw error + } finally { + owned.delete(controller) + setup = null + } + })() + return setup +} + +/** The 助手 model 设置 › 功能 chose, or the first one GBC can use. */ +function readingProvider(settings: Settings): string { + const usable = listProviders().filter(gbcCanUse) + const chosen = usable.find((provider) => provider.id === settings.gbcProviderId) ?? usable[0] + if (!chosen) throw new Error('先给助手加一个能用的模型') + return chosen.id } function configFor( @@ -148,17 +270,30 @@ sink = {"kind": "graphs", "path": ${JSON.stringify(join(out, 'graphs.jsonl'))}} ` } -export async function runGbc(input: GbcRunInput): Promise { +/** A run ended by 停止 or by quitting; not an error to show. */ +export class GbcStopped extends Error { + constructor() { + super('已停止') + } +} + +export interface GbcRunControl { + signal?: AbortSignal + /** Every output line, progress bars included (they redraw with a carriage return). */ + onLine?: (line: string) => void +} + +export async function runGbc(input: GbcRunInput, control: GbcRunControl = {}): Promise { const selectedFiles = (input.files ?? []).filter((path) => path.trim()) const folder = input.folder?.trim() ?? '' if (!selectedFiles.length && (!folder || !existsSync(folder))) throw new Error('GBC 图片目录/文件不存在') const provider = getProviderSecret(input.providerId) - if (!provider) throw new Error('请先在 Agent 设置中配置一个 AI Provider') - if (!provider.apiKey) throw new Error('GBC 的 API Provider 需要 API Key') + if (!provider) throw new Error('先给助手加一个能用的模型') + if (!provider.apiKey) throw new Error('看图的模型还没有填 API Key') const gpu = (await run('nvidia-smi', ['-L'], { timeoutMs: 5000 })).code === 0 const device = gpu ? 'cuda:0' : 'cpu' const id = `gbc-${randomUUID().slice(0, 8)}` - const out = join(paths.root(), 'gbc', id) + const out = join(paths.root(), 'gbc', 'runs', id) await mkdir(out, { recursive: true }) const keyEnv = 'MOSSTUDIO_GBC_API_KEY' const sourceMap = new Map() @@ -180,16 +315,34 @@ export async function runGbc(input: GbcRunInput): Promise { const version = input.versionId ? (await listGbcVersions(getSettings())).find((item) => item.id === input.versionId) : null const bundled = version ? versionBinary(version) : '' const command = bundled && existsSync(bundled) ? bundled : process.env['CLOVER_GBC_BIN'] || 'gbc' - const result = await run(command, ['run', configPath], { - cwd: version?.path ?? out, - env: { [keyEnv]: provider.apiKey, PYTHONUNBUFFERED: '1' }, - timeoutMs: 30 * 60_000, - }) + const controller = ownedController() + const stop = (): void => controller.abort() + control.signal?.addEventListener('abort', stop, { once: true }) + if (control.signal?.aborted) stop() + let result + try { + result = await runStream( + command, + ['run', configPath], + { + cwd: version?.path ?? out, + env: { [keyEnv]: provider.apiKey, PYTHONUNBUFFERED: '1' }, + timeoutMs: 30 * 60_000, + signal: controller.signal, + splitCarriageReturn: true, + }, + (line) => control.onLine?.(line), + ) + } finally { + owned.delete(controller) + control.signal?.removeEventListener('abort', stop) + } + if (result.aborted) throw new GbcStopped() const graphsFile = join(out, 'graphs.jsonl') if (result.code !== 0) { const failures = join(out, 'graphs.failures.jsonl') const detail = tail(result.stderr || result.stdout, 20) || (existsSync(failures) ? tail(readFileSync(failures, 'utf8'), 10) : '') - throw new Error(`GBC 运行失败:${detail || `exit ${result.code}`}`) + throw new Error(result.timedOut ? 'GBC 运行超过 30 分钟,已停止' : `GBC 运行失败:${detail || `exit ${result.code}`}`) } if (!existsSync(graphsFile)) throw new Error('GBC 没有输出 graph') const lines = readFileSync(graphsFile, 'utf8').split('\n').filter(Boolean) @@ -261,3 +414,73 @@ export async function runGbc(input: GbcRunInput): Promise { sidecarFiles, } } + +/** + * What a run's output line says about its progress. The first run downloads + * the detector and embedder from Hugging Face; their progress bars carry a + * percentage. Anything else means the picture is being read. + */ +export function parseProgressLine(line: string): Omit | null { + const percent = /(\d{1,3})%\|/.exec(line) + if (percent && /\.(safetensors|bin|json|model|txt)|download/i.test(line)) { + return { phase: 'download', percent: Math.min(100, Number.parseInt(percent[1]!, 10)) } + } + if (/download/i.test(line)) return { phase: 'download' } + return null +} + +/** How long after the last download line the card says 正在看图 again. */ +const READING_AFTER_MS = 3000 + +/** + * 拆成区域: GBC reads one picture with the version, model and detector chosen + * in 设置 › 功能. Resolves to null when 停止 ended it. Progress goes out as + * `Events.gbcParse` under the renderer's `runId`. + */ +export async function parseRegions(runId: string, path: string): Promise { + if (!existsSync(path)) throw new Error('找不到这张图片') + const settings = getSettings() + const providerId = readingProvider(settings) + const controller = new AbortController() + parses.set(runId, controller) + let last = '' + let lastDownload = 0 + const report = (progress: Omit): void => { + if (progress.phase === 'download') lastDownload = Date.now() + // Log lines between two download bars must not flip the card back to 正在看图. + else if (Date.now() - lastDownload < READING_AFTER_MS) return + const key = `${progress.phase}:${progress.percent ?? ''}` + if (key === last) return + last = key + emit(Events.gbcParse, { runId, ...progress } satisfies GbcParseProgress) + } + report({ phase: 'reading' }) + try { + const result = await runGbc( + { files: [path], providerId, detector: settings.gbcDetector, versionId: settings.gbcVersionId ?? undefined }, + { + signal: controller.signal, + onLine: (line) => { + const progress = parseProgressLine(line) + report(progress ?? { phase: 'reading' }) + }, + }, + ) + // Only the graph is needed; the run folder holds the picture's key and a copy of the config. + void rm(result.outputDir, { recursive: true, force: true }).catch(() => undefined) + const graph = result.images[0]?.graph + const parsed = graph ? regionsFromGraph(graph) : { summary: '', regions: [] } + if (!parsed.regions.length && !parsed.summary) throw new Error('没在这张图里找到物体') + return parsed + } catch (error) { + if (error instanceof GbcStopped) return null + throw error + } finally { + parses.delete(runId) + } +} + +/** 停止 on a 画面 that is still being read. */ +export function cancelParse(runId: string): void { + parses.get(runId)?.abort() +} diff --git a/apps/desktop/src/main/services/settings.ts b/apps/desktop/src/main/services/settings.ts index 34666ba..58f9b62 100644 --- a/apps/desktop/src/main/services/settings.ts +++ b/apps/desktop/src/main/services/settings.ts @@ -61,6 +61,10 @@ export const DEFAULT_SETTINGS: Settings = { deviceLabels: {}, remoteHostsEnabled: false, trainingEnabled: false, + gbcEnabled: false, + gbcProviderId: null, + gbcVersionId: null, + gbcDetector: 'owlv2', engineAutoUpdate: true, } diff --git a/apps/desktop/src/renderer/src/lib/features.ts b/apps/desktop/src/renderer/src/lib/features.ts index 1915288..427f24d 100644 --- a/apps/desktop/src/renderer/src/lib/features.ts +++ b/apps/desktop/src/renderer/src/lib/features.ts @@ -1,7 +1,7 @@ import type { Settings } from '@shared/types' /** Optional features. Each is off by default and turned on in 设置 › 功能. */ -export type FeatureKey = 'remoteHostsEnabled' | 'trainingEnabled' | 'lanWebEnabled' +export type FeatureKey = 'remoteHostsEnabled' | 'trainingEnabled' | 'gbcEnabled' | 'lanWebEnabled' export interface FeatureInfo { key: FeatureKey @@ -20,6 +20,11 @@ export const FEATURES: FeatureInfo[] = [ label: '训练', description: '用自己的图片训练 LoRA,训练好的 LoRA 可以直接在创作中使用。', }, + { + key: 'gbcEnabled', + label: '看图拆区域', + description: '把一张图拆成带描述的区域,照它的布局再画一张。', + }, { key: 'lanWebEnabled', label: '局域网访问', @@ -35,11 +40,13 @@ export function featureEnabled(settings: Pick | null | und export function featureVisibility(settings: Pick | null | undefined): { remoteHosts: boolean training: boolean + gbc: boolean lan: boolean } { return { remoteHosts: featureEnabled(settings, 'remoteHostsEnabled'), training: featureEnabled(settings, 'trainingEnabled'), + gbc: featureEnabled(settings, 'gbcEnabled'), lan: featureEnabled(settings, 'lanWebEnabled'), } } diff --git a/apps/desktop/src/renderer/src/lib/gbc.ts b/apps/desktop/src/renderer/src/lib/gbc.ts new file mode 100644 index 0000000..8e2b20e --- /dev/null +++ b/apps/desktop/src/renderer/src/lib/gbc.ts @@ -0,0 +1,59 @@ +import { useCallback, useEffect, useState } from 'react' +import { Events, Ipc } from '@shared/channels' +import type { AiProviderInfo } from '@shared/ai' +import { gbcBlocker, gbcCanUse, type GbcSetupProgress, type GbcStatus } from '@shared/gbc' +import { call, onEvent } from './api' + +export interface GbcState { + /** undefined while checking; null when the check itself failed. */ + status: GbcStatus | null | undefined + /** The 助手 models GBC can read pictures with. */ + providers: AiProviderInfo[] + /** Why 拆成区域 cannot run, or null when it can. */ + blocker: string | null + /** Check again; `fresh` re-runs the checks instead of taking the last answer. */ + refresh: (fresh?: boolean) => Promise +} + +/** + * Whether 拆成区域 can run, for the version and model 设置 › 功能 chose. + * Nothing is asked while the feature is off. An install finishing elsewhere, + * or the window coming back (a model added in 助手), checks again. + */ +export function useGbcState(enabled: boolean, versionId: string | null): GbcState { + const [status, setStatus] = useState(undefined) + const [providers, setProviders] = useState([]) + + const refresh = useCallback( + async (fresh = false) => { + if (!enabled) return + const [nextStatus, nextProviders] = await Promise.all([ + call(Ipc.gbcStatus, { fresh }).catch(() => null), + call(Ipc.aiProvidersList).catch(() => []), + ]) + setStatus(nextStatus) + setProviders(nextProviders.filter(gbcCanUse)) + }, + [enabled], + ) + + useEffect(() => { + setStatus(undefined) + void refresh() + }, [refresh, versionId]) + + useEffect(() => { + if (!enabled) return + const offSetup = onEvent(Events.gbcSetup, (progress) => { + if (progress.phase === 'done' || progress.phase === 'failed') void refresh() + }) + const onFocus = (): void => void refresh() + window.addEventListener('focus', onFocus) + return () => { + offSetup() + window.removeEventListener('focus', onFocus) + } + }, [enabled, refresh]) + + return { status, providers, blocker: gbcBlocker(status, providers), refresh } +} diff --git a/apps/desktop/src/renderer/src/pages/settings/FeaturesSection.tsx b/apps/desktop/src/renderer/src/pages/settings/FeaturesSection.tsx index 353d34c..d6aa24b 100644 --- a/apps/desktop/src/renderer/src/pages/settings/FeaturesSection.tsx +++ b/apps/desktop/src/renderer/src/pages/settings/FeaturesSection.tsx @@ -10,6 +10,7 @@ import { useApp } from '../../store' import { PathValue } from '../../components/common/PathValue' import { Button, Group, LoadingRow, Row, Section, useToast } from '../../components/ui' import { SectionBody, SwitchRow, TextSetting, errorText, useSaveSettings } from './fields' +import { GbcSettings } from './GbcSettings' export function FeaturesSection(): React.JSX.Element { const settings = useApp((state) => state.settings)! @@ -30,6 +31,7 @@ export function FeaturesSection(): React.JSX.Element { ))} + {settings.gbcEnabled ? : null} {settings.lanWebEnabled ? : null} ) diff --git a/apps/desktop/src/renderer/src/pages/settings/GbcSettings.tsx b/apps/desktop/src/renderer/src/pages/settings/GbcSettings.tsx new file mode 100644 index 0000000..edf686e --- /dev/null +++ b/apps/desktop/src/renderer/src/pages/settings/GbcSettings.tsx @@ -0,0 +1,308 @@ +import { Trash2 } from 'lucide-react' +import { useEffect, useState } from 'react' +import { Events, Ipc } from '@shared/channels' +import type { GbcSetupProgress } from '@shared/gbc' +import type { BranchInfo, CommitInfo, GbcVersion, Settings } from '@shared/types' +import { call, onEvent } from '../../lib/api' +import { dateTime } from '../../lib/format' +import { useGbcState } from '../../lib/gbc' +import { useApp } from '../../store' +import { Select } from '../../components/form' +import { + Advanced, + Button, + ConfirmDialog, + FieldLabel, + Group, + IconButton, + Pill, + ProgressBar, + Row, + Section, + Spinner, + useToast, +} from '../../components/ui' +import { errorText, useSaveSettings } from './fields' + +const SETUP_STEP: Record<'code' | 'packages', string> = { + code: '正在下载 KohakuGBC', + packages: '正在安装 Python 包,第一次要一段时间', +} + +const DETECTORS: Array<{ value: Settings['gbcDetector']; label: string }> = [ + { value: 'owlv2', label: 'OWLv2(照片)' }, + { value: 'sam3', label: 'SAM 3(插画)' }, +] + +/** 设置 › 功能 › 看图拆区域: which model reads the picture, and the KohakuGBC it runs on. */ +export function GbcSettings(): React.JSX.Element { + const settings = useApp((state) => state.settings)! + const save = useSaveSettings() + const toast = useToast() + const gbc = useGbcState(true, settings.gbcVersionId) + const [setup, setSetup] = useState(null) + const installing = setup?.phase === 'code' || setup?.phase === 'packages' + + useEffect( + () => + onEvent(Events.gbcSetup, (progress) => { + setSetup(progress) + if (progress.phase === 'done') toast.push('看图拆区域可以用了', 'success') + }), + [toast], + ) + + async function install(): Promise { + setSetup({ phase: 'code' }) + try { + await call(Ipc.gbcSetup) + } catch (error) { + setSetup({ phase: 'failed', error: errorText(error) }) + } + } + + const providerId = gbc.providers.some((provider) => provider.id === settings.gbcProviderId) + ? settings.gbcProviderId! + : (gbc.providers[0]?.id ?? '') + + return ( +
+ + + {gbc.providers.length ? ( + void save({ gbcDetector: value as Settings['gbcDetector'] })} + /> + + + + 版本 + + {versions.length === 0 ?

还没有版本,安装时会用最新的。

: null} + {versions.map((version) => { + const current = version.id === settings.gbcVersionId + return ( + + {current ? ( + 正在使用 + ) : ( + + )} + setRemoveTarget(version)} + > + {working === version.id ? : } + + + ) + })} + +
+ {branches.length === 0 ? ( + + ) : ( + <> + ({ value: item.sha, label: `${item.short} · ${item.subject.slice(0, 60)}` })) + : [{ value: '', label: '这个分支没有提交' }] + } + /> + + + )} +
+ + setRemoveTarget(null)} + onConfirm={() => removeTarget && void removeVersion(removeTarget)} + /> + + ) +} diff --git a/apps/desktop/src/renderer/src/pages/settings/diagnosticsReport.ts b/apps/desktop/src/renderer/src/pages/settings/diagnosticsReport.ts index ae5b0c0..c4d201d 100644 --- a/apps/desktop/src/renderer/src/pages/settings/diagnosticsReport.ts +++ b/apps/desktop/src/renderer/src/pages/settings/diagnosticsReport.ts @@ -43,7 +43,7 @@ export function diagnosticsReport(input: DiagnosticsInput, now = new Date()): st .join(', ')}` : '' }`, - `功能 远程主机 ${settings.remoteHostsEnabled ? '开' : '关'} · 训练 ${settings.trainingEnabled ? '开' : '关'} · 局域网 ${settings.lanWebEnabled ? '开' : '关'}`, + `功能 远程主机 ${settings.remoteHostsEnabled ? '开' : '关'} · 训练 ${settings.trainingEnabled ? '开' : '关'} · 看图 ${settings.gbcEnabled ? '开' : '关'} · 局域网 ${settings.lanWebEnabled ? '开' : '关'}`, ) } const failing = env?.checks.filter((check) => check.status === 'error' || check.status === 'warn') ?? [] diff --git a/apps/desktop/src/renderer/src/pages/studio/Board.tsx b/apps/desktop/src/renderer/src/pages/studio/Board.tsx index 48d913c..28d680e 100644 --- a/apps/desktop/src/renderer/src/pages/studio/Board.tsx +++ b/apps/desktop/src/renderer/src/pages/studio/Board.tsx @@ -24,6 +24,8 @@ import { type View, } from './boardLayout' import { CanvasCard, type CardMenuAction, type CardStrip } from './CanvasCard' +import type { ParseView } from './CardStatus' +import { PARSE_CHECKING } from '@shared/gbc' import { useShell } from '../../lib/shell' import { newCanvasMenu } from './connect' import { isImageFile } from './imageNodes' @@ -136,6 +138,15 @@ export interface BoardProps { /** 导入图片… / 从作品选择… from the right-click menu: pictures placed at a board point. */ onImportPictures: (at: Point) => void onPicturesFromGallery: (at: Point) => void + /** + * 拆成区域 on a card's picture. `parseBlocker` is undefined while the feature + * is off (no entry), null when it can run, else why not. + */ + parseBlocker: string | null | undefined + onParseRegions: (sourceId: string) => void + /** 画面 that 拆成区域 is filling, by id. */ + parsing: Readonly> + onStopParse: (canvasId: string) => void } /** Things to try on a new board's first 画面. */ @@ -402,12 +413,26 @@ export function Board(props: BoardProps): React.JSX.Element { // 放大 is offered when the engine takes a start picture and the picture can still grow. const canUpscale = (canvas: ProjectCanvas): boolean => props.engineFeatures?.imageInput !== false && !upscaleProblem(props.canvases, pictureLink(props.canvases, { canvasId: canvas.id })) + // 拆成区域 stays in the menu while it cannot run, saying why; choosing it then leads to 设置. + const parseAction = (canvasId: string): CardMenuAction[] => + props.parseBlocker === undefined + ? [] + : [ + { + label: '拆成区域', + hint: props.parseBlocker ?? undefined, + title: props.parseBlocker ?? undefined, + disabled: props.parseBlocker === PARSE_CHECKING, + onSelect: () => props.onParseRegions(canvasId), + }, + ] // The ⋯ menu of a card: what can be done with the picture it shows. const cardActions = (canvas: ProjectCanvas, run: ProjectCanvas, item: StripItem | undefined): CardMenuAction[] => { const actions: CardMenuAction[] = [] if (canvas.kind === 'layout') { if (!item) return actions if (item.path) actions.push({ label: '用作起始图', onSelect: () => props.onNewFromPicture(canvas.id, 'init') }) + if (item.path) actions.push(...parseAction(canvas.id)) if (item.path && canUpscale(canvas)) actions.push({ label: '放大', onSelect: () => props.onUpscale(canvas.id) }) if (folded.has(run.id) && item.path) actions.push({ label: '固定到画板', onSelect: () => props.onPinRun(run.id) }) if (run.kind === 'result') actions.push({ label: '用这次的设置', onSelect: () => props.onUseRunSettings(canvas.id, run.id) }) @@ -418,6 +443,7 @@ export function Board(props: BoardProps): React.JSX.Element { } if (givesPicture(canvas, props.canvases)) { actions.push({ label: '用作起始图', onSelect: () => props.onNewFromPicture(canvas.id, 'init') }) + actions.push(...parseAction(canvas.id)) if (canUpscale(canvas)) actions.push({ label: '放大', onSelect: () => props.onUpscale(canvas.id) }) } // A pinned result goes back into its 画面's filmstrip, unless it feeds another canvas. @@ -730,6 +756,8 @@ export function Board(props: BoardProps): React.JSX.Element { } : undefined } + parsing={props.parsing[canvas.id]} + onStopParse={() => props.onStopParse(canvas.id)} connecting={Boolean(draft && canvas.id !== draft.sourceId)} wireOut={draft?.sourceId === canvas.id} onPortDown={(event) => connect.start(canvas, event)} diff --git a/apps/desktop/src/renderer/src/pages/studio/BoardMenu.tsx b/apps/desktop/src/renderer/src/pages/studio/BoardMenu.tsx index 3ca2f25..bbab046 100644 --- a/apps/desktop/src/renderer/src/pages/studio/BoardMenu.tsx +++ b/apps/desktop/src/renderer/src/pages/studio/BoardMenu.tsx @@ -2,7 +2,17 @@ import { useEffect, useLayoutEffect, useRef, useState } from 'react' import { MenuGap, MenuItem } from '../../components/common/Popover' export type BoardMenuEntry = - { label: string; onSelect: () => void; tone?: 'danger'; disabled?: boolean; title?: string; selected?: boolean } | { gap: true } + | { + label: string + onSelect: () => void + tone?: 'danger' + disabled?: boolean + title?: string + selected?: boolean + /** A few words after the label, such as why the entry cannot do its job yet. */ + hint?: string + } + | { gap: true } const MENU_WIDTH = 200 @@ -85,6 +95,7 @@ export function BoardMenu({ disabled={entry.disabled} selected={entry.selected} title={entry.title} + hint={entry.hint} onSelect={() => { onClose() entry.onSelect() diff --git a/apps/desktop/src/renderer/src/pages/studio/CanvasCard.tsx b/apps/desktop/src/renderer/src/pages/studio/CanvasCard.tsx index 0db2a72..bd8dad1 100644 --- a/apps/desktop/src/renderer/src/pages/studio/CanvasCard.tsx +++ b/apps/desktop/src/renderer/src/pages/studio/CanvasCard.tsx @@ -8,7 +8,7 @@ import { tidyEntries, type BoardMenuEntry } from './BoardMenu' import type { LinkRole } from './links' import { OutputImage } from './OutputImage' import { CARD_FOOTER, CARD_HEADER, cardBodyHeight, cardWidth } from './boardLayout' -import { LivePreview, ProgressStrip, StatusOverlay } from './CardStatus' +import { LivePreview, ParseOverlay, ProgressStrip, StatusOverlay, type ParseView } from './CardStatus' import type { CardJobView } from './jobStatus' import { MaskLayer, usePictureSize, type MaskBrush, type MaskCommand } from './MaskLayer' import type { Size } from './mask' @@ -113,6 +113,9 @@ export interface CanvasCardProps { onRerun: (seed: 'new' | 'same') => void onCancel: () => void onShowActivity: () => void + /** 拆成区域 is filling this 画面. */ + parsing?: ParseView + onStopParse?: () => void } /** @@ -245,7 +248,14 @@ export function CanvasCard(props: CanvasCardProps): React.JSX.Element { // A 风格 always shows its description. const blankFace = isStyle || - (canvas.kind === 'layout' && !preview && !current && !painting && !props.initPicture && !canvas.backgroundPath && !view.active) + (canvas.kind === 'layout' && + !preview && + !current && + !painting && + !props.initPicture && + !canvas.backgroundPath && + !view.active && + !props.parsing) // The header speaks only while something is happening or went wrong. const loud = view.active || view.state === 'failed' || view.state === 'lost' const statusLabel = isImage || isStyle || !loud ? '' : view.label @@ -319,6 +329,9 @@ export function CanvasCard(props: CanvasCardProps): React.JSX.Element { { close() action.onSelect() @@ -442,6 +455,7 @@ export function CanvasCard(props: CanvasCardProps): React.JSX.Element { /> ) : null} + {props.parsing ? props.onStopParse?.()} /> : null} {isImage || isStyle ? null : } {view.active && (view.phase === 'denoise' || view.phase === 'decode' || view.phase === 'save' || view.phase === 'text') ? ( diff --git a/apps/desktop/src/renderer/src/pages/studio/CardStatus.tsx b/apps/desktop/src/renderer/src/pages/studio/CardStatus.tsx index 8162878..595c016 100644 --- a/apps/desktop/src/renderer/src/pages/studio/CardStatus.tsx +++ b/apps/desktop/src/renderer/src/pages/studio/CardStatus.tsx @@ -1,7 +1,9 @@ import { useEffect, useState } from 'react' -import { Spinner, cn } from '../../components/ui' +import { Button, Spinner, cn } from '../../components/ui' import { cloverUrl } from '../../lib/clover' +import { duration } from '../../lib/format' import type { CardJobView } from './jobStatus' +import { OutputImage } from './OutputImage' /** The engine rewrites one preview file; `seq` makes every frame a new URL. */ export function previewSrc(path: string, seq: number): string { @@ -122,3 +124,54 @@ export function StatusOverlay({ view, hasPicture }: { view: CardJobView; hasPict } return null } + +/** A 画面 that 拆成区域 is filling: the picture being read, how it is going, and 停止. */ +export interface ParseView { + runId: string + picture: string + phase: 'download' | 'reading' + percent?: number + /** ISO time the run started. */ + startedAt: string +} + +/** + * The face of a 画面 while its picture is read: the picture faintly, until + * its regions arrive all at once. The first run downloads the detector first. + */ +export function ParseOverlay({ parse, onStop }: { parse: ParseView; onStop: () => void }): React.JSX.Element { + const [now, setNow] = useState(() => new Date().toISOString()) + useEffect(() => { + const timer = window.setInterval(() => setNow(new Date().toISOString()), 1000) + return () => window.clearInterval(timer) + }, []) + const downloading = parse.phase === 'download' + const detail = downloading + ? parse.percent === undefined + ? '只有第一次' + : `${parse.percent}% · 只有第一次` + : `已用 ${duration(parse.startedAt, now) || '1 秒'}` + return ( +
+ +
+ +
+ {downloading ? '正在下载看图模型' : '正在看图…'} + {detail} +
+ +
+
+ ) +} diff --git a/apps/desktop/src/renderer/src/pages/studio/StudioPage.tsx b/apps/desktop/src/renderer/src/pages/studio/StudioPage.tsx index 90a1c88..36ac7da 100644 --- a/apps/desktop/src/renderer/src/pages/studio/StudioPage.tsx +++ b/apps/desktop/src/renderer/src/pages/studio/StudioPage.tsx @@ -62,6 +62,7 @@ import { resolveJobInputs } from './jobInputs' import { useGenerate } from './useGenerate' import { useInputImport } from './useInputImport' import { useStudioProject } from './useStudioProject' +import { usePictureRegions } from './usePictureRegions' const BUNDLE_KEY = 'mossstudio.modelBundleId' const HOST_KEY = 'mossstudio.studioHostId' @@ -376,6 +377,8 @@ export function StudioPage(): React.JSX.Element { const latestSettings = (): ReturnType => settingsFrom([...project.canvasesRef.current].reverse().find((canvas) => canvas.kind === 'layout' || canvas.kind === 'result')) + const regions = usePictureRegions(project, { latestSettings, reveal: (canvasId) => board.current?.reveal(canvasId) }) + /** 放大: a 放大 node fed by a card's picture, to its right (or where a wire was dropped). */ const upscaleFrom = (sourceId: string, at?: { x: number; y: number }): void => { const list = project.canvasesRef.current @@ -905,6 +908,10 @@ export function StudioPage(): React.JSX.Element { onRecipe={(recipe, at) => void startRecipe(recipe, at)} onUpscale={upscaleFrom} onAddStyle={(at) => addStyle(undefined, undefined, at)} + parseBlocker={regions.blocker} + onParseRegions={regions.parse} + parsing={regions.parsing} + onStopParse={regions.stop} /> diff --git a/apps/desktop/src/renderer/src/pages/studio/pictureRegions.ts b/apps/desktop/src/renderer/src/pages/studio/pictureRegions.ts new file mode 100644 index 0000000..106381e --- /dev/null +++ b/apps/desktop/src/renderer/src/pages/studio/pictureRegions.ts @@ -0,0 +1,59 @@ +import type { GbcParseResult } from '@shared/gbc' +import type { ProjectCanvas } from '@shared/types' +import { addRegion, createLayoutCanvas, settingsFrom, type CanvasSettings } from './canvases' +import { clampDimension } from './defaults' +import { linkSourceImage } from './links' +import { pictureSize } from './nodes' +import { canvasDescription, withDescription } from './prompt' +import { givesPicture, pictureLink, spotBeside } from './runs' + +/** The size of a 画面 shaped like the picture, with as many pixels as `settings` asks for. */ +export function sizeLike( + picture: { width: number; height: number }, + settings: Pick, +): { + width: number + height: number +} { + const area = settings.width * settings.height + const aspect = picture.width / picture.height + return { width: clampDimension(Math.sqrt(area * aspect)), height: clampDimension(Math.sqrt(area / aspect)) } +} + +/** + * 拆成区域: an empty 画面 beside the card whose picture is read, shaped like + * that picture; its regions come in when GBC is done. It is not wired to the + * picture: it keeps only the layout, where 用作起始图 keeps the pixels. + * `path` is the picture to read (a 画面 gives the run on its face). + */ +export function regionsCanvasFrom( + canvases: readonly ProjectCanvas[], + sourceId: string, + options: { fallback: CanvasSettings; id?: string }, +): { canvases: ProjectCanvas[]; created: ProjectCanvas; path: string } | null { + const source = canvases.find((canvas) => canvas.id === sourceId) + if (!source || !givesPicture(source, canvases)) return null + const link = pictureLink(canvases, { canvasId: sourceId }) + const path = linkSourceImage(link, canvases) + if (!path) return null + const settings = source.kind === 'image' ? options.fallback : settingsFrom(source) + const picture = pictureSize(canvases, link) + const placed = createLayoutCanvas(canvases, picture ? { ...settings, ...sizeLike(picture, settings) } : settings, options.id) + const created = { ...placed, ...spotBeside(source, placed, canvases) } + return { canvases: [...canvases, created], created, path } +} + +/** + * The regions GBC found, added to the 画面. A description typed while it was + * reading is kept; otherwise the picture's caption becomes the description. + */ +export function withParsedRegions(canvas: ProjectCanvas, result: GbcParseResult): ProjectCanvas { + let next = canvasDescription(canvas).trim() || !result.summary ? canvas : withDescription(canvas, result.summary) + for (const region of result.regions) next = addRegion(next, { kind: 'object', caption: region.caption, bbox: region.bbox }).canvas + return next +} + +/** Still as 拆成区域 made it: nothing typed, drawn or generated, so 停止 or a failure may take it away again. */ +export function untouched(canvas: ProjectCanvas): boolean { + return !canvas.layers.length && !canvasDescription(canvas).trim() && !canvas.jobId && !canvas.outputs?.length && !canvas.inputs +} diff --git a/apps/desktop/src/renderer/src/pages/studio/usePictureRegions.ts b/apps/desktop/src/renderer/src/pages/studio/usePictureRegions.ts new file mode 100644 index 0000000..dada80a --- /dev/null +++ b/apps/desktop/src/renderer/src/pages/studio/usePictureRegions.ts @@ -0,0 +1,119 @@ +import { useEffect, useRef, useState } from 'react' +import { useNavigate } from 'react-router-dom' +import { Events, Ipc } from '@shared/channels' +import type { GbcParseProgress, GbcParseResult } from '@shared/gbc' +import type { ProjectCanvas } from '@shared/types' +import { call, onEvent } from '../../lib/api' +import { useGbcState } from '../../lib/gbc' +import { useApp } from '../../store' +import { useToast } from '../../components/ui' +import { sectionPath } from '../settings/sections' +import { removeCanvasFrom, type CanvasSettings } from './canvases' +import type { ParseView } from './CardStatus' +import { regionsCanvasFrom, untouched, withParsedRegions } from './pictureRegions' + +interface ProjectAccess { + canvases: readonly ProjectCanvas[] + canvasesRef: { readonly current: ProjectCanvas[] } + setCanvases: (next: ProjectCanvas[] | ((previous: ProjectCanvas[]) => ProjectCanvas[])) => void + updateCanvas: (id: string, update: (canvas: ProjectCanvas) => ProjectCanvas) => void + select: (id: string | null) => void +} + +/** + * 拆成区域 on the board: a new 画面 beside the picture that fills with the + * regions GBC finds. 停止, a failure, or the 画面 going away ends the run; an + * untouched 画面 is taken away again. + */ +export function usePictureRegions( + project: ProjectAccess, + options: { latestSettings: () => CanvasSettings; reveal: (canvasId: string) => void }, +): { + /** undefined while the feature is off; null when it can run; else why not. */ + blocker: string | null | undefined + parsing: Readonly> + parse: (sourceId: string) => void + stop: (canvasId: string) => void +} { + const settings = useApp((state) => state.settings) + const enabled = Boolean(settings?.gbcEnabled) + const gbc = useGbcState(enabled, settings?.gbcVersionId ?? null) + const navigate = useNavigate() + const toast = useToast() + const [parsing, setParsing] = useState>({}) + const parsingRef = useRef(parsing) + parsingRef.current = parsing + + useEffect( + () => + onEvent(Events.gbcParse, (progress) => + setParsing((previous) => { + const entry = Object.entries(previous).find(([, view]) => view.runId === progress.runId) + if (!entry) return previous + const [canvasId, view] = entry + return { ...previous, [canvasId]: { ...view, phase: progress.phase, percent: progress.percent } } + }), + ), + [], + ) + + // A 画面 deleted, or a project switched, while its picture is read: the run has nowhere to go. + useEffect(() => { + for (const [canvasId, view] of Object.entries(parsingRef.current)) { + if (!project.canvases.some((canvas) => canvas.id === canvasId)) void call(Ipc.gbcCancel, { runId: view.runId }).catch(() => undefined) + } + }, [project.canvases]) + + const finish = (canvasId: string): void => + setParsing((previous) => { + const { [canvasId]: _done, ...rest } = previous + return rest + }) + + const dropIfUntouched = (canvasId: string): void => + project.setCanvases((previous) => { + const canvas = previous.find((item) => item.id === canvasId) + return canvas && untouched(canvas) ? removeCanvasFrom(previous, canvasId).canvases : previous + }) + + const parse = (sourceId: string): void => { + if (gbc.blocker) { + void navigate(sectionPath('features')) + return + } + const made = regionsCanvasFrom(project.canvasesRef.current, sourceId, { fallback: options.latestSettings() }) + if (!made) return + const canvasId = made.created.id + const runId = crypto.randomUUID() + project.setCanvases(made.canvases) + project.select(canvasId) + requestAnimationFrame(() => options.reveal(canvasId)) + setParsing((previous) => ({ + ...previous, + [canvasId]: { runId, picture: made.path, phase: 'reading', startedAt: new Date().toISOString() }, + })) + void call(Ipc.gbcParse, { runId, path: made.path }) + // The run is over before its 画面 may go, so taking it away does not stop the run a second time. + .then((result) => { + finish(canvasId) + if (!result) { + dropIfUntouched(canvasId) + return + } + project.updateCanvas(canvasId, (canvas) => withParsedRegions(canvas, result)) + toast.push(`已拆出 ${result.regions.length} 个区域`, 'success') + }) + .catch((error: unknown) => { + finish(canvasId) + dropIfUntouched(canvasId) + toast.push(`没拆成:${error instanceof Error ? error.message : String(error)}`, 'error') + }) + } + + const stop = (canvasId: string): void => { + const view = parsingRef.current[canvasId] + if (view) void call(Ipc.gbcCancel, { runId: view.runId }).catch(() => undefined) + } + + return { blocker: enabled ? gbc.blocker : undefined, parsing, parse, stop } +} diff --git a/apps/desktop/src/shared/channels.ts b/apps/desktop/src/shared/channels.ts index 4aebca3..47cecd4 100644 --- a/apps/desktop/src/shared/channels.ts +++ b/apps/desktop/src/shared/channels.ts @@ -101,6 +101,11 @@ export const Ipc = { gbcVersions: 'gbc:versions', gbcVersionCreate: 'gbc:version-create', gbcVersionRemove: 'gbc:version-remove', + /** 拆成区域: `{ runId, path }` → GbcParseResult, or null when 停止 ended it; progress as `Events.gbcParse`. */ + gbcParse: 'gbc:parse', + gbcCancel: 'gbc:cancel', + /** 安装 in 设置 › 功能 → GbcStatus; steps as `Events.gbcSetup`. */ + gbcSetup: 'gbc:setup', announcementsDismiss: 'announcements:dismiss', gitBranches: 'git:branches', gitCommits: 'git:commits', @@ -237,6 +242,10 @@ export const Events = { terminalData: 'event:ssh-terminal-data', terminalExit: 'event:ssh-terminal-exit', aiStream: 'event:ai-stream', + /** GbcParseProgress */ + gbcParse: 'event:gbc-parse', + /** GbcSetupProgress */ + gbcSetup: 'event:gbc-setup', aiToolCall: 'event:ai-tool-call', /** SetupProgress while installing. */ installProgress: 'event:install-progress', diff --git a/apps/desktop/src/shared/gbc.ts b/apps/desktop/src/shared/gbc.ts index dafd50b..33ca161 100644 --- a/apps/desktop/src/shared/gbc.ts +++ b/apps/desktop/src/shared/gbc.ts @@ -6,11 +6,18 @@ export interface GbcBBox { confidence?: number | null } +/** One caption of a vertex; ml-gbc names its kind `label` (short, detail…), older writers `kind`. */ +export interface GbcDescription { + text?: string + label?: string + kind?: string +} + export interface GbcVertex { vertex_id?: string label?: string bbox?: GbcBBox - descs?: Array<{ text?: string; kind?: string }> + descs?: GbcDescription[] in_edges?: unknown[] out_edges?: unknown[] } @@ -68,3 +75,102 @@ export interface GbcStatus { checks: GbcCheck[] versionId?: string } + +/** A region found in a picture: its words and its box, normalised to the picture (0–1). */ +export interface GbcRegion { + caption: string + bbox: { x: number; y: number; w: number; h: number } +} + +/** 拆成区域: what a picture becomes on its new 画面. */ +export interface GbcParseResult { + /** The whole picture's caption, the 画面's description. */ + summary: string + regions: GbcRegion[] +} + +/** While 拆成区域 runs: the first run downloads the detector, later ones only read. */ +export interface GbcParseProgress { + runId: string + phase: 'download' | 'reading' + /** 0–100 while downloading, when the download says how far it is. */ + percent?: number +} + +/** 安装 in 设置 › 功能: fetching KohakuGBC's code, then its Python packages. */ +export interface GbcSetupProgress { + phase: 'code' | 'packages' | 'done' | 'failed' + error?: string +} + +/** More regions than this crowd a 画面; the largest are kept. */ +export const GBC_MAX_REGIONS = 12 +/** A box covering nearly the whole picture says nothing about the layout. */ +const WHOLE_PICTURE = 0.9 + +function descKind(desc: GbcDescription): string { + return (desc.label ?? desc.kind ?? '').toLowerCase() +} + +function caption(vertex: GbcVertex, prefer: RegExp): string { + const descs = (vertex.descs ?? []).filter((desc) => desc.text?.trim()) + const pick = descs.find((desc) => prefer.test(descKind(desc))) ?? descs[0] + return pick?.text?.trim() ?? '' +} + +function edgeSource(edge: unknown): string | undefined { + if (!edge || typeof edge !== 'object') return undefined + const source = (edge as { source?: unknown }).source + return typeof source === 'string' ? source : undefined +} + +/** + * The regions and description a GBC graph gives a 画面. Regions are the + * objects the picture's caption names (entity vertices under the image vertex); + * their parts and the relation and composition vertices would only crowd it. + * A graph that does not say which entities are top level keeps every entity. + */ +export function regionsFromGraph(graph: GbcGraph): GbcParseResult { + const vertices = graph.vertices ?? [] + const image = vertices.find((vertex) => vertex.label === 'image') + const imageId = image?.vertex_id ?? '' + const entities = vertices.filter((vertex) => vertex.label === 'entity' && vertex.bbox) + const topLevel = entities.filter((vertex) => (vertex.in_edges ?? []).some((edge) => edgeSource(edge) === imageId)) + const chosen = topLevel.length ? topLevel : entities + const regions = chosen + .map((vertex) => { + const box = vertex.bbox! + const x = Math.min(Math.max(box.left, 0), 1) + const y = Math.min(Math.max(box.top, 0), 1) + const w = Math.min(Math.max(box.right, 0), 1) - x + const h = Math.min(Math.max(box.bottom, 0), 1) - y + return { caption: caption(vertex, /short/), bbox: { x, y, w, h } } + }) + .filter((region) => region.caption && region.bbox.w > 0 && region.bbox.h > 0 && region.bbox.w * region.bbox.h < WHOLE_PICTURE) + .sort((a, b) => b.bbox.w * b.bbox.h - a.bbox.w * a.bbox.h) + .slice(0, GBC_MAX_REGIONS) + return { summary: image ? caption(image, /detail|long/) : '', regions } +} + +/** What the menu says while GBC is still being checked; the entry is off until then. */ +export const PARSE_CHECKING = '正在检查' + +/** The 助手 models GBC can read pictures with: an OpenAI-compatible or Anthropic API with a key. */ +export function gbcCanUse(provider: { kind: string; hasKey: boolean }): boolean { + return provider.hasKey && (provider.kind === 'openai' || provider.kind === 'anthropic') +} + +/** + * Why 拆成区域 cannot run yet, as the menu says it, or null when it can. + * `status` is undefined while it is still being checked. + */ +export function gbcBlocker( + status: Pick | null | undefined, + providers: readonly { kind: string; hasKey: boolean }[], +): string | null { + if (status === undefined) return PARSE_CHECKING + // A chosen model that is gone falls back to the first usable one, so only having none blocks. + if (!providers.some(gbcCanUse)) return '先给助手加一个模型' + if (!status?.available) return '看图还没装好' + return null +} diff --git a/apps/desktop/src/shared/schemas.ts b/apps/desktop/src/shared/schemas.ts index f51d407..fcbf7c3 100644 --- a/apps/desktop/src/shared/schemas.ts +++ b/apps/desktop/src/shared/schemas.ts @@ -367,6 +367,10 @@ export const schemas = { githubToken: z.string().max(200).nullable().optional(), remoteHostsEnabled: z.boolean(), trainingEnabled: z.boolean(), + gbcEnabled: z.boolean(), + gbcProviderId: z.string().min(1).max(120).nullable(), + gbcVersionId: z.string().min(1).max(160).nullable(), + gbcDetector: z.enum(['owlv2', 'sam3']), engineAutoUpdate: z.boolean(), }) .partial(), @@ -678,7 +682,9 @@ export const schemas = { }), gbcVersionRemove: z.object({ id: z.string().min(1).max(160) }), - gbcStatusQuery: z.object({ versionId: z.string().max(160).optional() }), + gbcStatusQuery: z.object({ versionId: z.string().max(160).optional(), fresh: z.boolean().optional() }), + gbcParse: z.object({ runId: z.string().min(1).max(80), path: z.string().min(1).max(4096) }), + gbcCancel: z.object({ runId: z.string().min(1).max(80) }), gbcListImages: z.object({ folder: z.string().min(1).max(4096) }), gbcThumbnails: z.object({ paths: z.array(z.string().max(4096)).max(120), width: z.number().int().min(32).max(1600).default(160) }), gbcInstallEnv: z.object({ versionId: z.string().min(1).max(160) }), diff --git a/apps/desktop/src/shared/types.ts b/apps/desktop/src/shared/types.ts index c497653..266a0ee 100644 --- a/apps/desktop/src/shared/types.ts +++ b/apps/desktop/src/shared/types.ts @@ -199,6 +199,14 @@ export interface Settings { remoteHostsEnabled: boolean /** Optional feature: LoRA training. Off by default. */ trainingEnabled: boolean + /** Optional feature: 拆成区域 with KohakuGBC. Off by default. */ + gbcEnabled: boolean + /** The 助手 model GBC reads pictures with; null = the first usable one. */ + gbcProviderId: string | null + /** The installed KohakuGBC version 拆成区域 runs; null = none installed yet. */ + gbcVersionId: string | null + /** How GBC finds objects: OWLv2 for photos, SAM 3 for illustrations. */ + gbcDetector: 'owlv2' | 'sam3' /** Keep the inference engine on the latest release of its channel automatically. */ engineAutoUpdate: boolean } diff --git a/apps/desktop/test/main/lib/run.test.ts b/apps/desktop/test/main/lib/run.test.ts index 0c9ecf6..8a57f2e 100644 --- a/apps/desktop/test/main/lib/run.test.ts +++ b/apps/desktop/test/main/lib/run.test.ts @@ -84,6 +84,35 @@ describe('runStream', () => { const result = await runStream('/nonexistent/definitely-not-a-binary', [], {}, () => undefined) expect(result.code).toBe(-1) }) + + it('ends the process when its signal aborts, and says so', async () => { + const controller = new AbortController() + const started = Date.now() + const pending = runStream(node, script('console.log("up"); setTimeout(() => {}, 10000)'), { signal: controller.signal }, () => + controller.abort(), + ) + const result = await pending + expect(result.aborted).toBe(true) + expect(result.timedOut).toBe(false) + expect(Date.now() - started).toBeLessThan(5000) + }) + + it('does not start work for a signal already aborted', async () => { + const controller = new AbortController() + controller.abort() + const result = await runStream(node, script('setTimeout(() => {}, 10000)'), { signal: controller.signal }, () => undefined) + expect(result.aborted).toBe(true) + }) + + it('splits progress bars on a lone carriage return only when asked', async () => { + const source = 'process.stdout.write("10%|#\\r50%|###\\r100%|#####\\n")' + const split: string[] = [] + await runStream(node, script(source), { splitCarriageReturn: true }, (line) => split.push(line)) + expect(split).toEqual(['10%|#', '50%|###', '100%|#####']) + const whole: string[] = [] + await runStream(node, script(source), {}, (line) => whole.push(line)) + expect(whole).toEqual(['10%|#\r50%|###\r100%|#####']) + }) }) describe('firstLine / tail', () => { diff --git a/apps/desktop/test/main/services/gbc.test.ts b/apps/desktop/test/main/services/gbc.test.ts new file mode 100644 index 0000000..965b30e --- /dev/null +++ b/apps/desktop/test/main/services/gbc.test.ts @@ -0,0 +1,204 @@ +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname } from 'node:path' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { RunOptions, RunResult } from '@main/lib/run' +import type { GbcGraph, GbcParseProgress } from '@shared/gbc' +import { useTempDir } from '@test/helpers/tempDir' +import { settingsFixture } from '@test/mocks/backend/fixtures' + +const root = useTempDir('gbc-run-') + +type Behaviour = (args: string[], options: RunOptions, onLine: (line: string) => void) => Promise> +const state = { + settings: settingsFixture({ gbcEnabled: true, gbcDetector: 'sam3' }), + providers: [ + { id: 'local', name: 'Ollama', kind: 'ollama', baseUrl: '', model: 'llava', temperature: 0.7, hasKey: true }, + { + id: 'claude', + name: 'Claude', + kind: 'anthropic', + baseUrl: 'https://api.anthropic.com', + model: 'claude-sonnet', + temperature: 0.7, + hasKey: true, + }, + ], + emitted: [] as Array<[string, unknown]>, + configs: [] as string[], + behaviour: (async () => ({})) as Behaviour, +} + +/** What GBC writes for one picture: the image vertex and one object under it. */ +const graph: GbcGraph = { + vertices: [ + { vertex_id: '', label: 'image', descs: [{ text: 'a cat on a sill', label: 'detail' }] }, + { + vertex_id: 'cat', + label: 'entity', + bbox: { left: 0.2, top: 0.4, right: 0.6, bottom: 0.8 }, + descs: [{ text: 'a sleeping cat', label: 'short' }], + in_edges: [{ source: '' }], + }, + ], +} + +/** The graphs file the generated machine config names. */ +function sinkOf(config: string): string { + return /"kind": "graphs", "path": "([^"]+)"/.exec(readFileSync(config, 'utf8'))![1]!.replace(/\\\\/g, '\\') +} + +const writes = + (written: GbcGraph | null): Behaviour => + async (args) => { + if (written) writeFileSync(sinkOf(args[1]!), `${JSON.stringify(written)}\n`) + return {} + } + +vi.mock('@main/lib/run', async (importOriginal) => ({ + ...(await importOriginal()), + // No NVIDIA here: GBC runs on the CPU. + run: async () => ({ code: 1, stdout: '', stderr: '', timedOut: false }), + runStream: async (_command: string, args: string[], options: RunOptions, onLine: (line: string) => void) => { + state.configs.push(readFileSync(args[1]!, 'utf8')) + const result = await state.behaviour(args, options, onLine) + return { code: 0, stdout: '', stderr: '', timedOut: false, ...result } + }, +})) +vi.mock('@main/lib/paths', () => ({ paths: { root: () => root(), cache: () => root() } })) +vi.mock('@main/lib/events', () => ({ emit: (channel: string, payload: unknown) => state.emitted.push([channel, payload]) })) +vi.mock('@main/services/settings', () => ({ getSettings: () => state.settings, updateSettings: vi.fn() })) +vi.mock('@main/services/ai', () => ({ + listProviders: () => state.providers, + getProviderSecret: (id: string) => { + const provider = state.providers.find((item) => item.id === id) + return provider ? { kind: provider.kind, baseUrl: provider.baseUrl, model: provider.model, apiKey: 'sk-test' } : null + }, +})) +vi.mock('@main/services/gbcGit', () => ({ listGbcVersions: async () => [], listGbcBranches: vi.fn(), createGbcVersion: vi.fn() })) +vi.mock('@main/services/runtimes', () => ({ pickPython: vi.fn(), resolveTorchIndex: vi.fn() })) + +const load = () => import('@main/services/gbc') +const picture = (): string => { + const path = `${root()}/cat.png` + writeFileSync(path, 'png') + return path +} +const progress = (): GbcParseProgress[] => + state.emitted.filter(([channel]) => channel === 'event:gbc-parse').map(([, payload]) => payload as GbcParseProgress) + +beforeEach(() => { + vi.resetModules() + state.settings = settingsFixture({ gbcEnabled: true, gbcDetector: 'sam3' }) + state.emitted = [] + state.configs = [] + state.behaviour = writes(graph) +}) + +describe('拆成区域', () => { + it('reads one picture into a description and regions, with the chosen detector and the first model that can read pictures', async () => { + const { parseRegions } = await load() + expect(await parseRegions('run-1', picture())).toEqual({ + summary: 'a cat on a sill', + regions: [{ caption: 'a sleeping cat', bbox: { x: 0.2, y: 0.4, w: expect.closeTo(0.4), h: expect.closeTo(0.4) } }], + }) + const config = state.configs[0]! + expect(config).toContain('"protocol": "anthropic"') + expect(config).toContain('"model": "claude-sonnet"') + expect(config).toContain('"model": "facebook/sam3"') + expect(config).toContain('"devices": ["cpu"]') + // The key goes to the process environment, never into the config file. + expect(config).not.toContain('sk-test') + }) + + it('uses the model 设置 chose', async () => { + state.providers.push({ + id: 'gpt', + name: 'GPT', + kind: 'openai', + baseUrl: 'https://api.openai.com/v1', + model: 'gpt-x', + temperature: 1, + hasKey: true, + }) + state.settings = settingsFixture({ gbcEnabled: true, gbcProviderId: 'gpt' }) + const { parseRegions } = await load() + await parseRegions('run-1', picture()) + expect(state.configs[0]).toContain('"model": "gpt-x"') + expect(state.configs[0]).toContain('"model": "google/owlv2-base-patch16-ensemble"') + state.providers.pop() + }) + + it('removes the run folder once the graph is read', async () => { + let folder = '' + state.behaviour = async (args, options, onLine) => { + folder = dirname(args[1]!) + return writes(graph)(args, options, onLine) + } + const { parseRegions } = await load() + await parseRegions('run-1', picture()) + await vi.waitFor(() => expect(existsSync(folder)).toBe(false)) + }) + + it('tells the card when the first run downloads models, then that it reads', async () => { + state.behaviour = async (args, options, onLine) => { + onLine('model.safetensors: 10%|# | 62M/620M') + onLine('model.safetensors: 10%|# | 63M/620M') + onLine('model.safetensors: 55%|##### | 341M/620M') + onLine('loading weights') + return writes(graph)(args, options, onLine) + } + const { parseRegions } = await load() + await parseRegions('run-1', picture()) + // The same step twice is sent once, and a log line right after a download bar does not flip it back. + expect(progress()).toEqual([ + { runId: 'run-1', phase: 'reading' }, + { runId: 'run-1', phase: 'download', percent: 10 }, + { runId: 'run-1', phase: 'download', percent: 55 }, + ]) + }) + + it('resolves to null when 停止 ends the run', async () => { + state.behaviour = (_args, options) => + new Promise((resolve) => options.signal!.addEventListener('abort', () => resolve({ code: -1, aborted: true }))) + const { cancelParse, parseRegions } = await load() + const pending = parseRegions('run-1', picture()) + await vi.waitFor(() => expect(state.configs).toHaveLength(1)) + cancelParse('run-1') + expect(await pending).toBeNull() + }) + + it('ends every run when the app quits', async () => { + state.behaviour = (_args, options) => + new Promise((resolve) => options.signal!.addEventListener('abort', () => resolve({ code: -1, aborted: true }))) + const { parseRegions, stopGbcProcesses } = await load() + const pending = parseRegions('run-1', picture()) + await vi.waitFor(() => expect(state.configs).toHaveLength(1)) + stopGbcProcesses() + expect(await pending).toBeNull() + }) + + it('says why it cannot read', async () => { + const { parseRegions } = await load() + await expect(parseRegions('run-1', `${root()}/gone.png`)).rejects.toThrow('找不到这张图片') + state.behaviour = writes({ vertices: [{ vertex_id: '', label: 'image', descs: [] }] }) + await expect(parseRegions('run-2', picture())).rejects.toThrow('没在这张图里找到物体') + state.behaviour = async () => ({ code: 1, stderr: 'Traceback\nRuntimeError: CUDA out of memory' }) + await expect(parseRegions('run-3', picture())).rejects.toThrow('CUDA out of memory') + state.behaviour = async () => ({ code: -1, timedOut: true }) + await expect(parseRegions('run-4', picture())).rejects.toThrow('超过 30 分钟') + const providers = state.providers + state.providers = providers.filter((provider) => provider.kind === 'ollama') + await expect(parseRegions('run-5', picture())).rejects.toThrow('先给助手加一个能用的模型') + state.providers = providers + }) +}) + +describe('parseProgressLine', () => { + it('reads a download percentage from a Hugging Face progress bar', async () => { + const { parseProgressLine } = await load() + expect(parseProgressLine('model.safetensors: 42%|████▏ | 260M/620M [00:12<00:17]')).toEqual({ phase: 'download', percent: 42 }) + expect(parseProgressLine('Downloading (…)okenizer.json')).toEqual({ phase: 'download' }) + expect(parseProgressLine('vlm 12%|█ | 3/25')).toBeNull() + expect(parseProgressLine('admitted 1 job')).toBeNull() + }) +}) diff --git a/apps/desktop/test/mocks/backend/fixtures.ts b/apps/desktop/test/mocks/backend/fixtures.ts index fb0c199..1052c31 100644 --- a/apps/desktop/test/mocks/backend/fixtures.ts +++ b/apps/desktop/test/mocks/backend/fixtures.ts @@ -6,6 +6,7 @@ */ import releaseCatalog from '../../../resources/models.json' import type { AiConversationInfo, AiProviderInfo } from '@shared/ai' +import type { GbcParseResult, GbcStatus } from '@shared/gbc' import type { UpdateAnnouncement, UpdaterState } from '@shared/update' import type { AdapterEntry, @@ -17,6 +18,7 @@ import type { EngineReadiness, EnvReport, GalleryItem, + GbcVersion, GpuInventoryDevice, GpuInventorySnapshot, GpuTelemetrySnapshot, @@ -99,6 +101,10 @@ export function settingsFixture(patch: Partial = {}): Settings { deviceLabels: {}, remoteHostsEnabled: false, trainingEnabled: false, + gbcEnabled: false, + gbcProviderId: null, + gbcVersionId: null, + gbcDetector: 'owlv2', engineAutoUpdate: true, ...patch, } @@ -657,6 +663,41 @@ export function aiProviderFixture(patch: Partial = {}): AiProvid } } +// --------------------------------------------------------------------------- +// 看图拆区域 (KohakuGBC) + +export function gbcVersionFixture(patch: Partial = {}): GbcVersion { + return { + id: 'gbc-main', + name: 'GBC 9f3c2a1', + branch: 'main', + commit: '9f3c2a1b4d5e', + shortCommit: '9f3c2a1', + subject: 'Fixture GBC', + commitDate: FIXTURE_NOW, + path: '/fixture/gbc/versions/gbc-main', + createdAt: FIXTURE_NOW, + ...patch, + } +} + +export function gbcStatusFixture(available: boolean): GbcStatus { + return available + ? { available: true, version: 'GBC 9f3c2a1 · 9f3c2a1', checks: [], versionId: 'gbc-main' } + : { available: false, error: 'GBC 尚未安装完整环境;请查看下方检查项并安装环境。', checks: [] } +} + +/** What 拆成区域 finds in a picture: a caption and two regions. */ +export function gbcParseFixture(): GbcParseResult { + return { + summary: '午后的窗台,阳光照在一只睡着的橘猫身上', + regions: [ + { caption: '趴着睡觉的橘猫', bbox: { x: 0.25, y: 0.5, w: 0.45, h: 0.25 } }, + { caption: '窗外的晴天', bbox: { x: 0.5, y: 0.08, w: 0.42, h: 0.4 } }, + ], + } +} + export function conversationFixture(patch: Partial = {}): AiConversationInfo { return { id: 'fixture-history', title: '历史对话', providerId: 'fixture-ai', messages: [], createdAt: '', updatedAt: '', ...patch } } diff --git a/apps/desktop/test/mocks/backend/index.ts b/apps/desktop/test/mocks/backend/index.ts index de4ff7b..49e52b2 100644 --- a/apps/desktop/test/mocks/backend/index.ts +++ b/apps/desktop/test/mocks/backend/index.ts @@ -52,6 +52,8 @@ import { controlAdapterFixture, localHostFixture, downloadFixture, + gbcStatusFixture, + gbcVersionFixture, jobLifecycle, submittedJobFixture, updateAvailableFixture, @@ -781,6 +783,38 @@ function defaultHandlers(): Map { return null }) + // 看图拆区域: setup and reading answer at once, with the events main sends on the way. + h.set(Ipc.gbcStatus, (_, { state }) => state.gbc.status) + h.set(Ipc.gbcVersions, (_, { state }) => state.gbc.versions) + h.set(Ipc.gbcSetup, (_, { state, emit }) => { + emit(Events.gbcSetup, { phase: 'code' }) + if (!state.gbc.versions.length) state.gbc.versions = [gbcVersionFixture()] + state.settings = { ...state.settings, gbcVersionId: state.gbc.versions[0]!.id } + emit(Events.settingsChanged, state.settings) + emit(Events.gbcSetup, { phase: 'packages' }) + state.gbc.status = gbcStatusFixture(true) + emit(Events.gbcSetup, { phase: 'done' }) + return state.gbc.status + }) + h.set(Ipc.gbcBranches, () => [{ name: 'main', sha: '9f3c2a1b4d5e', isDefault: true, isStable: true }]) + h.set(Ipc.gbcCommits, () => [{ sha: '9f3c2a1b4d5e', short: '9f3c2a1', date: FIXTURE_NOW, subject: 'Fixture GBC', author: 'fixture' }]) + h.set(Ipc.gbcVersionCreate, (payload, { state }) => { + const { commit } = p<{ branch: string; commit: string }>(payload) + const version = gbcVersionFixture({ id: `gbc-${commit.slice(0, 7)}`, commit, shortCommit: commit.slice(0, 7) }) + state.gbc.versions = [version, ...state.gbc.versions.filter((item) => item.id !== version.id)] + return version + }) + h.set(Ipc.gbcVersionRemove, (payload, { state }) => { + const { id } = p<{ id: string }>(payload) + state.gbc.versions = state.gbc.versions.filter((item) => item.id !== id) + return true + }) + h.set(Ipc.gbcParse, (payload, { state, emit }) => { + emit(Events.gbcParse, { runId: p<{ runId: string }>(payload).runId, phase: 'reading' }) + return state.gbc.parse + }) + h.set(Ipc.gbcCancel, () => true) + // AI h.set(Ipc.aiProvidersList, (_, { state }) => state.aiProviders) h.set(Ipc.aiConversations, (_, { state }) => state.conversations) diff --git a/apps/desktop/test/mocks/backend/scenarios.ts b/apps/desktop/test/mocks/backend/scenarios.ts index 38419d8..5dbe8d1 100644 --- a/apps/desktop/test/mocks/backend/scenarios.ts +++ b/apps/desktop/test/mocks/backend/scenarios.ts @@ -5,6 +5,7 @@ * a test changes what the "main process" knows by editing `backend.state`. */ import type { AiConversationInfo, AiProviderInfo } from '@shared/ai' +import type { GbcParseResult, GbcStatus } from '@shared/gbc' import type { UpdaterState } from '@shared/update' import { cliEngineFeatures } from '@shared/cliEngine' import type { EngineAssetList, EngineMachine } from '@shared/engineChoice' @@ -17,6 +18,7 @@ import type { EnvReport, GalleryDetail, GalleryItem, + GbcVersion, GpuInventorySnapshot, JobLogLine, JobRecord, @@ -47,6 +49,8 @@ import { engineVersionFixture, envReportFixture, galleryItemFixture, + gbcParseFixture, + gbcStatusFixture, GPU_RTX_5090, gpuDeviceFixture, gpuInventoryFixture, @@ -100,6 +104,8 @@ export interface MockBackendState { systemMemoryBytes: number /** Free space models:space reports for a models folder; a folder not listed has plenty. */ diskFree: Record + /** 看图拆区域: whether GBC runs, its versions, and what reading a picture finds. */ + gbc: { status: GbcStatus; versions: GbcVersion[]; parse: GbcParseResult } } export type ScenarioName = @@ -165,6 +171,7 @@ export function scenario(name: ScenarioName = 'ready', patch: Partial { it('lists each optional feature once with copy', () => { - expect(FEATURES.map((feature) => feature.key)).toEqual(['remoteHostsEnabled', 'trainingEnabled', 'lanWebEnabled']) + expect(FEATURES.map((feature) => feature.key)).toEqual(['remoteHostsEnabled', 'trainingEnabled', 'gbcEnabled', 'lanWebEnabled']) expect(FEATURES.every((feature) => feature.label && feature.description)).toBe(true) }) it('every optional feature is off in default settings', () => { - expect(featureVisibility(settingsFixture())).toEqual({ remoteHosts: false, training: false, lan: false }) + expect(featureVisibility(settingsFixture())).toEqual({ remoteHosts: false, training: false, gbc: false, lan: false }) }) it('reads each toggle independently', () => { - expect(featureVisibility(settingsFixture({ trainingEnabled: true }))).toEqual({ remoteHosts: false, training: true, lan: false }) + expect(featureVisibility(settingsFixture({ trainingEnabled: true }))).toEqual({ + remoteHosts: false, + training: true, + gbc: false, + lan: false, + }) expect(featureVisibility(settingsFixture({ remoteHostsEnabled: true, lanWebEnabled: true }))).toEqual({ remoteHosts: true, training: false, + gbc: false, lan: true, }) }) + it('reads 看图拆区域 on its own', () => { + expect(featureVisibility(settingsFixture({ gbcEnabled: true }))).toEqual({ remoteHosts: false, training: false, gbc: true, lan: false }) + }) + it('treats missing settings as everything off', () => { expect(featureEnabled(null, 'trainingEnabled')).toBe(false) expect(featureEnabled(undefined, 'lanWebEnabled')).toBe(false) - expect(featureVisibility(null)).toEqual({ remoteHosts: false, training: false, lan: false }) + expect(featureVisibility(null)).toEqual({ remoteHosts: false, training: false, gbc: false, lan: false }) }) }) diff --git a/apps/desktop/test/renderer/settings/features.test.ts b/apps/desktop/test/renderer/settings/features.test.ts index 859f270..c86234b 100644 --- a/apps/desktop/test/renderer/settings/features.test.ts +++ b/apps/desktop/test/renderer/settings/features.test.ts @@ -3,18 +3,18 @@ import { FEATURES, featureEnabled, featureVisibility } from '@renderer/lib/featu import { settingsFixture } from '@test/mocks/backend/fixtures' describe('optional features', () => { - it('lists remote hosts, training and LAN access with an explanation each', () => { - expect(FEATURES.map((feature) => feature.key)).toEqual(['remoteHostsEnabled', 'trainingEnabled', 'lanWebEnabled']) + it('lists remote hosts, training, 看图拆区域 and LAN access with an explanation each', () => { + expect(FEATURES.map((feature) => feature.key)).toEqual(['remoteHostsEnabled', 'trainingEnabled', 'gbcEnabled', 'lanWebEnabled']) for (const feature of FEATURES) expect(feature.description.length).toBeGreaterThan(8) }) it('hides everything until settings load', () => { - expect(featureVisibility(null)).toEqual({ remoteHosts: false, training: false, lan: false }) + expect(featureVisibility(null)).toEqual({ remoteHosts: false, training: false, gbc: false, lan: false }) expect(featureEnabled(undefined, 'trainingEnabled')).toBe(false) }) it('follows the switches', () => { const settings = settingsFixture({ remoteHostsEnabled: true, trainingEnabled: false, lanWebEnabled: true }) - expect(featureVisibility(settings)).toEqual({ remoteHosts: true, training: false, lan: true }) + expect(featureVisibility(settings)).toEqual({ remoteHosts: true, training: false, gbc: false, lan: true }) }) }) diff --git a/apps/desktop/test/renderer/studio/pictureRegions.test.ts b/apps/desktop/test/renderer/studio/pictureRegions.test.ts new file mode 100644 index 0000000..6aeabd2 --- /dev/null +++ b/apps/desktop/test/renderer/studio/pictureRegions.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest' +import { defaultSettings } from '@renderer/pages/studio/canvases' +import { canvasDescription } from '@renderer/pages/studio/prompt' +import { regionsCanvasFrom, sizeLike, untouched, withParsedRegions } from '@renderer/pages/studio/pictureRegions' +import { canvas, layer } from './fixtures' + +const picture = canvas({ + id: 'p', + kind: 'image', + title: 'IMG_0231', + imagePath: '/pictures/IMG_0231.jpg', + imageSize: { width: 1200, height: 800 }, + width: 1200, + height: 800, +}) +const parsed = { + summary: 'a cat asleep on a sunny sill', + regions: [ + { caption: 'a sleeping cat', bbox: { x: 0.25, y: 0.5, w: 0.5, h: 0.25 } }, + { caption: 'a white mug', bbox: { x: 0.8, y: 0.6, w: 0.1, h: 0.1 } }, + ], +} + +describe('拆成区域 on the board', () => { + it('shapes the 画面 like the picture, with the pixels the settings ask for', () => { + expect(sizeLike({ width: 1200, height: 800 }, { width: 1024, height: 1024 })).toEqual({ width: 1248, height: 832 }) + expect(sizeLike({ width: 800, height: 1200 }, { width: 1024, height: 1024 })).toEqual({ width: 832, height: 1248 }) + expect(sizeLike({ width: 500, height: 500 }, { width: 1024, height: 1024 })).toEqual({ width: 1024, height: 1024 }) + }) + + it('puts an empty 画面 to the right of the picture, not wired to it, and names the file to read', () => { + const made = regionsCanvasFrom([picture], 'p', { fallback: defaultSettings(), id: 'n' })! + expect(made.path).toBe('/pictures/IMG_0231.jpg') + expect(made.created).toMatchObject({ id: 'n', kind: 'layout', title: '画面 1', width: 1248, height: 832, layers: [], prompt: '' }) + expect(made.created.inputs).toBeUndefined() + expect(made.created.x).toBeGreaterThan(picture.x) + expect(made.canvases.map((item) => item.id)).toEqual(['p', 'n']) + expect(untouched(made.created)).toBe(true) + }) + + it('reads the picture a result card shows, and takes its settings', () => { + const result = canvas({ id: 'r', kind: 'result', steps: 8, outputs: [{ path: '/out/r.png', width: 1024, height: 1536, seed: 3 }] }) + const made = regionsCanvasFrom([result], 'r', { fallback: defaultSettings(), id: 'n' })! + expect(made.path).toBe('/out/r.png') + expect(made.created).toMatchObject({ steps: 8, width: 832, height: 1248 }) + }) + + it('offers nothing for a card without a picture', () => { + expect(regionsCanvasFrom([canvas({ id: 'empty' })], 'empty', { fallback: defaultSettings() })).toBeNull() + expect(regionsCanvasFrom([picture], 'missing', { fallback: defaultSettings() })).toBeNull() + }) + + it('fills the 画面 with the regions in its pixels and the picture caption as its description', () => { + const made = regionsCanvasFrom([picture], 'p', { fallback: defaultSettings(), id: 'n' })! + const filled = withParsedRegions(made.created, parsed) + expect(canvasDescription(filled)).toBe('a cat asleep on a sunny sill') + expect(filled.layers.map((region) => [region.kind, region.caption, region.bbox])).toEqual([ + ['object', 'a sleeping cat', [312, 416, 936, 624]], + ['object', 'a white mug', [998, 499, 1123, 582]], + ]) + expect(untouched(filled)).toBe(false) + }) + + it('keeps a description typed while the picture was read, and the boxes drawn meanwhile', () => { + const typed = canvas({ + id: 'n', + prompt: '水彩画', + summary: { caption: '水彩画', meta: [], canvas: 'single' }, + layers: [layer({ id: 'mine' })], + }) + const filled = withParsedRegions(typed, parsed) + expect(canvasDescription(filled)).toBe('水彩画') + expect(filled.layers.map((region) => region.id)[0]).toBe('mine') + expect(filled.layers).toHaveLength(3) + }) + + it('counts a 画面 as touched once anything was typed, drawn or generated', () => { + expect(untouched(canvas())).toBe(true) + expect(untouched(canvas({ prompt: '海边' }))).toBe(false) + expect(untouched(canvas({ layers: [layer()] }))).toBe(false) + expect(untouched(canvas({ jobId: 'j1' }))).toBe(false) + expect(untouched(canvas({ inputs: { init: { link: { canvasId: 'p' }, strength: 0.6 } } }))).toBe(false) + }) +}) diff --git a/apps/desktop/test/shared/gbc.test.ts b/apps/desktop/test/shared/gbc.test.ts new file mode 100644 index 0000000..2ce6b2a --- /dev/null +++ b/apps/desktop/test/shared/gbc.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from 'vitest' +import { GBC_MAX_REGIONS, PARSE_CHECKING, gbcBlocker, gbcCanUse, regionsFromGraph, type GbcGraph, type GbcVertex } from '@shared/gbc' + +const box = (left: number, top: number, right: number, bottom: number): GbcVertex['bbox'] => ({ + left, + top, + right, + bottom, + confidence: null, +}) +const fromImage = [{ source: '', text: 'cat', target: 'cat' }] + +// The shape ml-gbc writes: the image vertex has id "", its captions carry `label`. +const graph: GbcGraph = { + vertices: [ + { + vertex_id: '', + label: 'image', + bbox: box(0, 0, 1, 1), + descs: [ + { text: 'A cat asleep on a sill.', label: 'short' }, + { text: 'An orange cat sleeps on a sunny window sill beside a white mug.', label: 'detail' }, + ], + }, + { + vertex_id: 'cat', + label: 'entity', + bbox: box(0.3, 0.5, 0.75, 0.72), + descs: [ + { text: 'an orange cat curled up, eyes closed, fur lit by the sun', label: 'detail' }, + { text: 'a sleeping orange cat', label: 'short' }, + ], + in_edges: fromImage, + }, + { + vertex_id: 'mug', + label: 'entity', + bbox: box(0.8, 0.6, 0.92, 0.72), + descs: [{ text: 'a white mug', label: 'short' }], + in_edges: [{ source: '', text: 'mug', target: 'mug' }], + }, + // A part of the cat: under the cat, not the image, so it would only crowd the 画面. + { + vertex_id: 'cat/ear', + label: 'entity', + bbox: box(0.32, 0.5, 0.4, 0.56), + descs: [{ text: 'a pointed ear', label: 'short' }], + in_edges: [{ source: 'cat', text: 'ear', target: 'cat/ear' }], + }, + { vertex_id: 'rel', label: 'relation', bbox: box(0.3, 0.5, 0.92, 0.72), descs: [{ text: 'the mug is beside the cat' }] }, + ], +} + +describe('regionsFromGraph', () => { + it('turns the objects the picture names into regions, largest first, with their short captions', () => { + const result = regionsFromGraph(graph) + expect(result.summary).toBe('An orange cat sleeps on a sunny window sill beside a white mug.') + expect(result.regions.map((region) => region.caption)).toEqual(['a sleeping orange cat', 'a white mug']) + expect(result.regions[0]!.bbox.x).toBeCloseTo(0.3) + expect(result.regions[0]!.bbox.y).toBeCloseTo(0.5) + expect(result.regions[0]!.bbox.w).toBeCloseTo(0.45) + expect(result.regions[0]!.bbox.h).toBeCloseTo(0.22) + }) + + it('keeps every entity when the graph does not say which are top level', () => { + const flat: GbcGraph = { vertices: graph.vertices!.map(({ in_edges: _edges, ...vertex }) => vertex) } + expect(regionsFromGraph(flat).regions.map((region) => region.caption)).toEqual([ + 'a sleeping orange cat', + 'a white mug', + 'a pointed ear', + ]) + }) + + it('reads captions written with `kind` and falls back to the first caption', () => { + const result = regionsFromGraph({ + vertices: [ + { vertex_id: 'img', label: 'image', descs: [{ text: 'a room', kind: 'long' }] }, + { + vertex_id: 'lamp', + label: 'entity', + bbox: box(0.1, 0.1, 0.3, 0.5), + descs: [{ text: 'a tall lamp' }], + in_edges: [{ source: 'img' }], + }, + ], + }) + expect(result.summary).toBe('a room') + expect(result.regions).toEqual([{ caption: 'a tall lamp', bbox: { x: 0.1, y: 0.1, w: expect.closeTo(0.2), h: expect.closeTo(0.4) } }]) + }) + + it('drops boxes that cover the whole picture, have no size or no words, and pulls boxes inside the picture', () => { + const result = regionsFromGraph({ + vertices: [ + { vertex_id: 'whole', label: 'entity', bbox: box(0, 0, 1, 0.98), descs: [{ text: 'the scene' }] }, + { vertex_id: 'flat', label: 'entity', bbox: box(0.2, 0.2, 0.2, 0.6), descs: [{ text: 'a line' }] }, + { vertex_id: 'mute', label: 'entity', bbox: box(0.1, 0.1, 0.4, 0.4), descs: [{ text: ' ' }] }, + { vertex_id: 'edge', label: 'entity', bbox: box(-0.2, 0.5, 0.3, 1.4), descs: [{ text: 'a chair' }] }, + ], + }) + expect(result.summary).toBe('') + expect(result.regions).toEqual([{ caption: 'a chair', bbox: { x: 0, y: 0.5, w: 0.3, h: 0.5 } }]) + }) + + it(`keeps at most ${GBC_MAX_REGIONS} regions, the largest`, () => { + const vertices: GbcVertex[] = Array.from({ length: GBC_MAX_REGIONS + 3 }, (_, index) => ({ + vertex_id: `v${index}`, + label: 'entity', + bbox: box(0, 0, 0.05 + index * 0.01, 0.5), + descs: [{ text: `thing ${index}` }], + })) + const regions = regionsFromGraph({ vertices }).regions + expect(regions).toHaveLength(GBC_MAX_REGIONS) + expect(regions[0]!.caption).toBe(`thing ${GBC_MAX_REGIONS + 2}`) + expect(regions.some((region) => region.caption === 'thing 0')).toBe(false) + }) + + it('gives nothing for an empty graph', () => { + expect(regionsFromGraph({})).toEqual({ summary: '', regions: [] }) + }) +}) + +describe('gbcBlocker', () => { + const claude = { kind: 'anthropic', hasKey: true } + + it('only an OpenAI-compatible or Anthropic model with a key can read pictures', () => { + expect(gbcCanUse(claude)).toBe(true) + expect(gbcCanUse({ kind: 'openai', hasKey: true })).toBe(true) + expect(gbcCanUse({ kind: 'openai', hasKey: false })).toBe(false) + expect(gbcCanUse({ kind: 'ollama', hasKey: true })).toBe(false) + expect(gbcCanUse({ kind: 'gemini', hasKey: true })).toBe(false) + }) + + it('says what is missing, in the order the person fixes it', () => { + expect(gbcBlocker(undefined, [])).toBe(PARSE_CHECKING) + expect(gbcBlocker({ available: true }, [{ kind: 'ollama', hasKey: true }])).toBe('先给助手加一个模型') + expect(gbcBlocker({ available: false }, [claude])).toBe('看图还没装好') + expect(gbcBlocker(null, [claude])).toBe('看图还没装好') + expect(gbcBlocker({ available: true }, [claude])).toBeNull() + }) +}) diff --git a/apps/desktop/test/shared/schemas.test.ts b/apps/desktop/test/shared/schemas.test.ts index 13a53cf..8dbf795 100644 --- a/apps/desktop/test/shared/schemas.test.ts +++ b/apps/desktop/test/shared/schemas.test.ts @@ -101,6 +101,8 @@ const cases = { engineDeviceOverrides: { 'GPU-1': 'cu13-sm75-system-cuda', 'vk-1002-744c-0': 'hip', 'vk-8086-56a0-0': 'vulkan' }, }, 'no per-GPU engines': { engineDeviceOverrides: {} }, + '拆成区域 settings': { gbcEnabled: true, gbcProviderId: 'provider-1', gbcVersionId: 'gbc-1', gbcDetector: 'sam3' }, + '拆成区域 defaults': { gbcProviderId: null, gbcVersionId: null }, }, rejects: { 'unknown theme': { theme: 'blue' }, @@ -122,6 +124,8 @@ const cases = { 'non-boolean flag': { autoDownloadUpdates: 'yes' }, 'non-boolean feature toggle': { remoteHostsEnabled: 1 }, 'null engine auto-update': { engineAutoUpdate: null }, + 'unknown GBC detector': { gbcDetector: 'yolo' }, + 'empty GBC model': { gbcProviderId: '' }, 'unknown engine package': { engineDeviceOverrides: { 'GPU-1': 'cu11' } }, 'engine package for no GPU': { engineDeviceOverrides: { '': 'vulkan' } }, '65 per-GPU engines': { engineDeviceOverrides: Object.fromEntries(Array.from({ length: 65 }, (_, i) => [`GPU-${i}`, 'vulkan'])) }, @@ -620,10 +624,21 @@ const cases = { gbcStatusQuery: { valid: {}, - accepts: { version: { versionId: 'v1' }, 'empty version': { versionId: '' } }, - rejects: { 'overlong version': { versionId: long(161) }, undefined: undefined }, + accepts: { version: { versionId: 'v1' }, 'empty version': { versionId: '' }, 'checked again': { fresh: true } }, + rejects: { 'overlong version': { versionId: long(161) }, undefined: undefined, 'fresh as text': { fresh: 'yes' } }, }, + gbcParse: { + valid: { runId: 'r1', path: '/a.png' }, + rejects: { + 'no path': { runId: 'r1', path: '' }, + 'no run id': { runId: '', path: '/a.png' }, + 'overlong run id': { runId: long(81), path: '/a.png' }, + }, + }, + + gbcCancel: { valid: { runId: 'r1' }, rejects: { 'empty run id': { runId: '' } } }, + gbcListImages: { valid: { folder: '/imgs' }, rejects: { 'empty folder': { folder: '' } } }, gbcThumbnails: { diff --git a/docs/how-to/setup.md b/docs/how-to/setup.md index 3fda094..9f478c2 100644 --- a/docs/how-to/setup.md +++ b/docs/how-to/setup.md @@ -107,7 +107,7 @@ CUDA 引擎对 NVIDIA 驱动的最低要求:CUDA 13 需要 Windows 580.88 或 | 「设备与引擎」 | 「显卡」:多块显卡时「设为默认」或「改为自动」,右上角可「重新检测显卡」。「引擎」:「安装引擎」「检查更新」「更新」,以及「自动更新」开关。「远程主机」。「高级」:「显存模式」「性能档位」「连跑时重新加载引擎」「每块显卡的引擎」「渠道」「版本」「添加自定义引擎」「引擎目录」 | | 「存储」 | 「数据文件夹」「模型文件夹」「作品来源」 | | 「网络」 | Hugging Face「访问令牌(Token)」;「高级」里的「下载方式」(自动、直连、镜像)、「Hugging Face 端点」、「同时下载的文件数」、「GitHub 令牌」 | -| 「功能」 | 可选功能的开关:「远程主机」「训练」「局域网访问」,默认都关闭 | +| 「功能」 | 可选功能的开关:「远程主机」「训练」「看图拆区域」「局域网访问」,默认都关闭;打开「看图拆区域」后在下方选看图的模型并安装 | | 「通知与更新」 | 「桌面通知」、「自动更新」、「更新渠道」(正式版、测试版)、「回到上一个版本」 | | 「关于」 | 版本、「复制诊断信息」、「环境检查」;Linux 和 macOS 上还有「卸载 MossStudio」 | diff --git a/docs/reference/architecture.md b/docs/reference/architecture.md index 1116452..d2d367a 100644 --- a/docs/reference/architecture.md +++ b/docs/reference/architecture.md @@ -1,7 +1,7 @@ --- status: current last-verified: 2026-11-08 (fec4fe3) -code: apps/desktop/src/main/index.ts, apps/desktop/src/main/windows.ts, apps/desktop/src/main/ipc.ts, apps/desktop/src/main/lib/paths.ts, apps/desktop/src/main/install/, apps/desktop/src/main/update/, apps/desktop/src/preload/index.ts, apps/desktop/src/renderer/src/App.tsx, apps/desktop/src/renderer/src/components/shell/routes.ts, apps/desktop/src/renderer/src/pages/settings/sections.ts, apps/desktop/src/shared/channels.ts, apps/desktop/src/shared/agent/, apps/desktop/src/backend/index.ts, apps/desktop/src/main/services/backend.ts +code: apps/desktop/src/main/index.ts, apps/desktop/src/main/windows.ts, apps/desktop/src/main/ipc.ts, apps/desktop/src/main/lib/paths.ts, apps/desktop/src/main/install/, apps/desktop/src/main/update/, apps/desktop/src/preload/index.ts, apps/desktop/src/renderer/src/App.tsx, apps/desktop/src/renderer/src/components/shell/routes.ts, apps/desktop/src/renderer/src/pages/settings/sections.ts, apps/desktop/src/shared/channels.ts, apps/desktop/src/shared/agent/, apps/desktop/src/backend/index.ts, apps/desktop/src/main/services/backend.ts, apps/desktop/src/main/services/gbc.ts, apps/desktop/src/shared/gbc.ts --- # Architecture @@ -157,7 +157,13 @@ Generation runs in MossInfer, a native engine installed per GPU: a long-lived `m ## GBC -KohakuGBC is an optional external tool reachable only through IPC (`gbc:*` in `shared/channels.ts`, `shared/gbc.ts`, `main/services/gbc.ts`); no renderer code calls it at fec4fe3. +[KohakuGBC](https://github.com/KohakuBlueleaf/KohakuGBC) is an optional external tool behind the 看图拆区域 feature (`Settings.gbcEnabled`, off by default). It finds the objects in a picture and asks one of the 助手's models (an OpenAI-compatible or Anthropic API with a key) to describe each. The Studio uses it for 拆成区域 ([studio-node-board.md](/docs/reference/studio-node-board.md#6-runs-and-the-filmstrip)). + +- `main/services/gbcGit.ts` keeps a bare mirror of KohakuGBC and checks out each version as a worktree under `gbc/versions/` (or `Settings.gbcRoot`). KohakuGBC is not on PyPI, so a version is always installed from its code. +- `main/services/gbc.ts` installs a version's `.venv` (PyTorch from the trainer's index, then `pip install -e .[torch]`), checks it (`gbcStatus`, cached until an install or `fresh`), and runs `gbc run` on a generated machine config. The API key reaches the process only through its environment. +- Every GBC process is owned there: 停止 (`gbc:cancel`) and quitting (`stopGbcProcesses` on `before-quit`) end it, and a run stops after 30 minutes. +- Channels: `gbc:setup` (安装 in 设置 › 功能) with `event:gbc-setup`, and `gbc:parse` with `event:gbc-parse` progress; the version channels serve 设置 › 功能 › 高级. `shared/gbc.ts` turns a GBC graph into regions (`regionsFromGraph`) and says why it cannot run (`gbcBlocker`). +- `gbc:run`, `gbc:list-images` and `gbc:thumbnails` (folders and sidecar files) have no UI. ## Leftovers diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index a2e1a71..5358070 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -71,3 +71,4 @@ The terms MossStudio coins or uses in a fixed sense, with the Chinese the UI sho | LoRA | LoRA, `role: 'lora'` | A small add-on model applied through a 风格 node. 训练 produces them. | `shared/types.ts`, `renderer/src/pages/studio/nodes.ts` | | 模型文件夹 | models root, `modelsRoot` | The folder models are stored in; ComfyUI layouts are recognised in place. | `renderer/src/pages/models/ModelsPage.tsx`, `shared/types.ts` (`Settings.modelsRoot`) | | 训练 | Training, route `/training`, Trainer window | LoRA training through the external Trainer, behind the `trainingEnabled` feature. | `renderer/src/pages/training/`, `renderer/src/pages/TrainerWindow.tsx`, ADR-0002 | +| 看图拆区域 / 拆成区域 | GBC, `gbcEnabled`, `gbc:parse` | The optional feature that reads a picture with KohakuGBC, and the picture-menu entry that turns it into a new 画面 with regions and a description. | `main/services/gbc.ts`, `shared/gbc.ts`, `renderer/src/pages/studio/usePictureRegions.ts` | diff --git a/docs/reference/studio-node-board.md b/docs/reference/studio-node-board.md index fea7a75..f16c951 100644 --- a/docs/reference/studio-node-board.md +++ b/docs/reference/studio-node-board.md @@ -97,12 +97,15 @@ Picture actions in a card's menu (`studio/Board.tsx` `cardActions`): | Entry | Does | Code | | --- | --- | --- | | 用作起始图 | A new 画面 to the right, fed by this picture; it takes the settings, description and regions | `studio/canvases.ts` (`canvasFromPicture`) | +| 拆成区域 | A new 画面 to the right, shaped like the picture and not wired to it, that fills with the regions and description KohakuGBC finds. Offered while 看图拆区域 is on; while it cannot run, the entry names why and leads to 设置 › 功能 | `studio/pictureRegions.ts`, `studio/usePictureRegions.ts` | | 放大 | A 放大 node fed by this picture; offered while the engine has `imageInput` and the long edge is under 2048 | `studio/nodes.ts` (`upscaleFromPicture`) | | 固定到画板 | A folded run becomes a card under its 画面 and stays in the strip | `studio/runs.ts` (`pinRun`) | | 收回到「画面」 | A pinned result goes back into the strip, unless a canvas uses it | `unpinRun`, `canUnpin` | | 用这次的设置 | The 画面 takes a run's description, regions, size, model, quality, seed (locked), sampling and LoRA | `withRunSettings` | | 删除这次生成 | Removes a run that is only in the strip; its picture stays in 作品 | `dropRun`, `canDropRun` | +While 拆成区域 reads, the new 画面 shows the picture faintly with 正在看图… (or 正在下载看图模型 on the first run) and 停止 (`ParseOverlay` in `studio/CardStatus.tsx`). The regions arrive all at once, at most `GBC_MAX_REGIONS`, as 物体; a description typed meanwhile is kept. 停止, a failure, or deleting the 画面 ends the run, and a 画面 still untouched is taken away again. Setup and the main side are in [architecture.md](/docs/reference/architecture.md#gbc). + ## 7. Recipes A recipe lays out nodes already wired; nothing about them is special afterwards (`studio/recipes.ts`, started in `studio/StudioPage.tsx` `startRecipe`). They are offered in the empty board's menu and, while a new project has one blank 画面, in a row under it (`StartRow` in `studio/BoardControls.tsx`). That same blank 画面 shows three descriptions to try on its face (`FIRST_IDEAS` in `studio/Board.tsx`); a click writes one as its description.