feat(models): let models.json add models safely without a release

.github/workflows/catalog.yml

@@ -0,0 +1,35 @@
+name: Model catalog
+
+# Released apps read apps/desktop/resources/models.json from main, through raw.githubusercontent.com
+# (cached about five minutes) or jsDelivr, which can serve an old copy for hours. After a merge that changes
+# the file, ask jsDelivr to drop its copy so a new model reaches every app within minutes.
+# The file was checked on the pull request (test/shared/modelCatalog.test.ts); this only refreshes caches.
+on:
+  push:
+    branches: [main]
+    paths: [apps/desktop/resources/models.json]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  purge:
+    name: Refresh the jsDelivr copy
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Purge
+        run: |
+          url='https://purge.jsdelivr.net/gh/ShallowElec/MossStudio@main/apps/desktop/resources/models.json'
+          for attempt in 1 2 3 4 5; do
+            if body=$(curl --fail --silent --show-error --max-time 30 "$url"); then
+              echo "$body"
+              # A purge refused for rate limits still answers 200 and marks the path throttled.
+              if ! grep -q '"throttled":true' <<< "$body"; then exit 0; fi
+            fi
+            echo "attempt $attempt did not purge; retrying"
+            sleep $((attempt * 10))
+          done
+          echo "::error::jsDelivr did not purge models.json; apps that read it there see the change once its cache expires"
+          exit 1

apps/desktop/src/main/lib/content.ts

@@ -12,3 +12,12 @@ export function contentDir(): string | null {
   const dir = app.isPackaged ? process.resourcesPath : join(app.getAppPath(), 'resources')
   return existsSync(join(dir, 'models.json')) || existsSync(join(dir, 'rules.json')) ? dir : null
 }
+
+/**
+ * Where a models.json that wins over the catalog on `main` lives: the source tree, in development
+ * only, so an edit shows at once. A packaged app's copy is its fallback, never an override, or it
+ * would never see a model added after its release.
+ */
+export function catalogOverrideDir(): string | null {
+  return app.isPackaged ? null : contentDir()
+}

apps/desktop/src/main/services/models.ts

@@ -13,7 +13,9 @@ import { emit } from '../lib/events'
 import { paths } from '../lib/paths'
 import { assertFreeSpace, freeSpace, nearestExisting } from '../lib/disk'
 import { getLastNetwork } from './network'
-import { contentDir } from '../lib/content'
+import { app } from 'electron'
+import { catalogProblems, entriesForApp } from '@shared/modelCatalog'
+import { catalogOverrideDir } from '../lib/content'
 import { listJobs } from './supervisor'
 import packagedCatalog from '../../../resources/models.json'
 import {
@@ -254,6 +256,39 @@ async function fetchText(url: string, timeoutMs = 8000): Promise<string> {
   return response.text()
 }
 
+const CATALOG_PATH = '/apps/desktop/resources/models.json'
+
+/**
+ * The catalog on `main`, else the last copy of it this app could use (so a model added after the
+ * release stays listed offline), else null for the packaged one. A copy older than the packaged
+ * catalog loses to it: an update brings a newer list than a copy saved before it.
+ */
+async function downloadedCatalog(): Promise<{ raw: RawCatalog; source: 'remote' | 'cache' } | null> {
+  const saved = join(paths.cache(), 'models.json')
+  try {
+    const base = getLastNetwork()?.selected.manifest ?? 'https://raw.githubusercontent.com/ShallowElec/MossStudio/main'
+    const text = await fetchText(`${base.replace(/\/$/, '')}${CATALOG_PATH}`)
+    const raw = JSON.parse(text) as RawCatalog
+    if (catalogProblems(raw).length === 0) {
+      // Saving is a convenience for the next offline start; failing to save loses nothing now.
+      await mkdir(paths.cache(), { recursive: true })
+        .then(() => writeFile(saved, text))
+        .catch(() => undefined)
+      return { raw, source: 'remote' }
+    }
+  } catch {
+    /* Offline, or the mirror is unreachable: the saved copy below. */
+  }
+  try {
+    const raw = JSON.parse(await readFile(saved, 'utf8')) as RawCatalog
+    const older = (raw.updated_at ?? '') < (BUILTIN.updated_at ?? '')
+    if (!older && catalogProblems(raw).length === 0) return { raw, source: 'cache' }
+  } catch {
+    /* No saved copy yet. */
+  }
+  return null
+}
+
 function importedModelsIndex(settings: Settings): string {
   return join(paths.modelImports(settings.modelsRoot), 'index.json')
 }
@@ -360,21 +395,21 @@ export async function getCatalog(settings: Settings, force = false): Promise<Cat
   if (catalogCache && catalogRoot === key && !force) return catalogCache
   let raw = BUILTIN
   let source: Catalog['source'] = 'builtin'
-  const local = contentDir()
+  const local = catalogOverrideDir()
   try {
     if (local && existsSync(join(local, 'models.json'))) {
       raw = JSON.parse(await readFile(join(local, 'models.json'), 'utf8')) as RawCatalog
       source = 'local'
     } else {
-      const base = getLastNetwork()?.selected.manifest ?? 'https://raw.githubusercontent.com/ShallowElec/MossStudio/main'
-      raw = JSON.parse(await fetchText(`${base.replace(/\/$/, '')}/apps/desktop/resources/models.json`)) as RawCatalog
-      source = 'remote'
+      const downloaded = await downloadedCatalog()
+      if (downloaded) ({ raw, source } = downloaded)
     }
     if (!Array.isArray(raw.models)) throw new Error('无效目录')
   } catch {
     raw = BUILTIN
     source = 'builtin'
   }
+  raw = entriesForApp(raw, app.getVersion())
   // Old remote catalogs cannot reintroduce whole-repo downloads for known models.
   if (raw.schema_version < 2) {
     const known = new Set([

apps/desktop/src/shared/modelCatalog.ts

@@ -0,0 +1,174 @@
+/**
+ * The model catalog file (apps/desktop/resources/models.json) as it is written, and the rules a
+ * copy must meet before the app uses it. Released apps download this file from `main`, so a
+ * model added to it reaches them without a release (docs/how-to/add-a-model.md).
+ */
+import { z } from 'zod'
+import { compareSemver } from './update'
+
+/**
+ * The newest catalog layout this app reads. Raise it only when an older app would misread the
+ * file; such an app keeps the catalog it has instead. New optional fields do not raise it.
+ */
+export const CATALOG_SCHEMA_VERSION = 3
+
+const SEMVER = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/
+const semver = z.string().regex(SEMVER, 'not a version like 2.1.0 or 2.1.0-beta.2')
+/** A commit pinned by its full hash, or a branch the app follows. */
+const revision = z.string().regex(/^(?:[0-9a-f]{40}|main)$/, 'not a 40-character commit hash or "main"')
+const id = z.string().regex(/^[a-z0-9][a-z0-9._-]*$/, 'not a lowercase id')
+const text = z.string().trim().min(1)
+
+const model = z.object({
+  id,
+  name: text,
+  choice: text.optional(),
+  variant: z.array(text).optional(),
+  provider: z.literal('hf'),
+  repo: z.string().regex(/^[\w.-]+\/[\w.-]+$/, 'not an owner/name repository'),
+  revision,
+  role: z.enum(['dit', 'text_encoder', 'vae', 'support', 'full']),
+  precision: text,
+  format: z.enum(['diffusers', 'comfyui']).optional(),
+  family: text.optional(),
+  subfolder: z.string().optional(),
+  size_bytes: z.number().int().positive(),
+  files: z.array(text).min(1),
+  license: text,
+  tags: z.array(text).optional(),
+  notes: z.string().optional(),
+  preview: z.string().optional(),
+  min_app_version: semver.optional(),
+})
+
+const bundle = z.object({
+  id,
+  name: text,
+  variant: z.array(text).optional(),
+  description: z.string(),
+  components: z.object({ dit: id, textEncoder: id, vae: id }),
+  support: id,
+  recommended: z.boolean().optional(),
+  certified: z.boolean().optional(),
+  tier: z.enum(['high', 'turbo']).optional(),
+  requires: z.object({ computeCapability: z.object({ major: z.number().int(), minor: z.number().int() }), label: text }).optional(),
+  recommendedFor: z.literal('limited-vram').optional(),
+  defaults: z.object({ steps: z.number().int().positive().optional(), cfg: z.number().nonnegative().optional() }).optional(),
+  min_app_version: semver.optional(),
+})
+
+const adapter = z.object({
+  id,
+  name: text,
+  provider: z.literal('hf'),
+  repo: z.string().regex(/^[\w.-]+\/[\w.-]+$/, 'not an owner/name repository'),
+  revision,
+  kind: z.enum(['lora', 'lycoris', 'dmd']),
+  base: id.optional(),
+  license: text,
+  files: z.array(text).min(1),
+  size_bytes: z.number().int().positive(),
+  size_estimated: z.boolean().optional(),
+  requires_profile: text.optional(),
+  tags: z.array(text).optional(),
+  notes: z.string().optional(),
+  min_app_version: semver.optional(),
+})
+
+/** Unknown fields pass: a field added for newer apps must not make older ones reject the file. */
+export const catalogFileSchema = z.object({
+  schema_version: z.number().int().positive(),
+  updated_at: z.iso.datetime().optional(),
+  models: z.array(model),
+  bundles: z.array(bundle).optional(),
+  single_file_bundles: z.array(bundle).optional(),
+  adapters: z.array(adapter).optional(),
+})
+
+export type CatalogFile = z.infer<typeof catalogFileSchema>
+
+const ROLE_OF = { dit: 'dit', textEncoder: 'text_encoder', vae: 'vae' } as const
+
+/**
+ * Everything wrong with a catalog file, as one line each; empty when the app can use it.
+ * Beyond its shape: ids are unique, and every bundle names parts that exist in the role it uses them.
+ */
+export function catalogProblems(raw: unknown): string[] {
+  const parsed = catalogFileSchema.safeParse(raw)
+  if (!parsed.success) return parsed.error.issues.map((issue) => `${issue.path.join('.') || '(file)'}: ${issue.message}`)
+  const file = parsed.data
+  const problems: string[] = []
+  if (file.schema_version > CATALOG_SCHEMA_VERSION) {
+    problems.push(`schema_version ${file.schema_version} is newer than this app reads (${CATALOG_SCHEMA_VERSION})`)
+  }
+  const duplicates = (list: Array<{ id: string }>, where: string): void => {
+    const seen = new Set<string>()
+    for (const item of list) {
+      if (seen.has(item.id)) problems.push(`${where}: "${item.id}" appears twice`)
+      seen.add(item.id)
+    }
+  }
+  duplicates(file.models, 'models')
+  duplicates([...(file.bundles ?? []), ...(file.single_file_bundles ?? [])], 'bundles')
+  duplicates(file.adapters ?? [], 'adapters')
+  const models = new Map(file.models.map((item) => [item.id, item]))
+  for (const item of [...(file.bundles ?? []), ...(file.single_file_bundles ?? [])]) {
+    for (const [slot, role] of Object.entries(ROLE_OF) as Array<[keyof typeof ROLE_OF, string]>) {
+      const part = models.get(item.components[slot])
+      if (!part) problems.push(`bundle "${item.id}": ${slot} "${item.components[slot]}" is not in models`)
+      else if (part.role !== role) problems.push(`bundle "${item.id}": ${slot} "${part.id}" has role ${part.role}, not ${role}`)
+    }
+    const support = models.get(item.support)
+    if (!support) problems.push(`bundle "${item.id}": support "${item.support}" is not in models`)
+    else if (support.role !== 'support') problems.push(`bundle "${item.id}": support "${support.id}" has role ${support.role}, not support`)
+  }
+  return problems
+}
+
+/**
+ * Whether an app at `appVersion` may offer an entry that needs `minimum`. A prerelease of a
+ * version counts as that version (a nightly of 2.1.0 has the code 2.1.0 ships), unless the
+ * minimum names a prerelease itself.
+ */
+export function meetsMinimumVersion(appVersion: string, minimum: string | undefined): boolean {
+  if (!minimum) return true
+  const app = minimum.includes('-') ? appVersion : appVersion.replace(/[-+].*$/, '')
+  return compareSemver(app, minimum) >= 0
+}
+
+interface Gated {
+  id: string
+  min_app_version?: string
+}
+interface GatedBundle extends Gated {
+  components: { dit: string; textEncoder: string; vae: string }
+  support: string
+}
+
+/**
+ * The entries an app at `appVersion` can use: entries needing a newer app are left out, and so
+ * is every bundle that names a part left out.
+ */
+export function entriesForApp<
+  T extends {
+    models: Gated[]
+    bundles?: GatedBundle[]
+    single_file_bundles?: GatedBundle[]
+    adapters?: Gated[]
+  },
+>(catalog: T, appVersion: string): T {
+  const fits = (item: Gated): boolean => meetsMinimumVersion(appVersion, item.min_app_version)
+  const dropped = new Set(catalog.models.filter((item) => !fits(item)).map((item) => item.id))
+  if (!dropped.size && [...(catalog.bundles ?? []), ...(catalog.single_file_bundles ?? []), ...(catalog.adapters ?? [])].every(fits)) {
+    return catalog
+  }
+  const usable = (item: GatedBundle): boolean =>
+    fits(item) && ![item.components.dit, item.components.textEncoder, item.components.vae, item.support].some((part) => dropped.has(part))
+  return {
+    ...catalog,
+    models: catalog.models.filter(fits),
+    ...(catalog.bundles && { bundles: catalog.bundles.filter(usable) }),
+    ...(catalog.single_file_bundles && { single_file_bundles: catalog.single_file_bundles.filter(usable) }),
+    ...(catalog.adapters && { adapters: catalog.adapters.filter(fits) }),
+  }
+}

apps/desktop/test/main/lib/content.test.ts

@@ -0,0 +1,15 @@
+import { describe, expect, it, vi } from 'vitest'
+import { app } from 'electron'
+import { catalogOverrideDir, contentDir } from '@main/lib/content'
+
+describe('catalogOverrideDir', () => {
+  it('is the source tree in development, so an edit to models.json shows at once', () => {
+    expect(contentDir()).not.toBeNull()
+    expect(catalogOverrideDir()).toBe(contentDir())
+  })
+
+  it('is nothing in a packaged app: its models.json is a fallback, and the catalog on main is read', () => {
+    vi.spyOn(app, 'isPackaged', 'get').mockReturnValue(true)
+    expect(catalogOverrideDir()).toBeNull()
+  })
+})

apps/desktop/test/main/models/modelBundles.test.ts

@@ -7,7 +7,7 @@ import { useTempDataRoot } from '@test/helpers/tempDir'
 import { cacheDirFor, installOnDisk, rawModel, testSettings, writeCatalog } from './modelFixtures'
 
 const contentDir = vi.hoisted(() => vi.fn<() => string | null>())
-vi.mock('@main/lib/content', () => ({ contentDir }))
+vi.mock('@main/lib/content', () => ({ contentDir, catalogOverrideDir: () => contentDir() }))
 vi.mock('@main/services/supervisor', () => ({ listJobs: async () => [] }))
 vi.mock('@main/services/network', () => ({ getLastNetwork: () => null }))
 vi.mock('@main/services/hfNetwork', () => ({ fetchHf: (url: string) => fetch(url) }))