diff --git a/src/desktop/ipc-bridge.ts b/src/desktop/ipc-bridge.ts index cbeebaf..25bb8e8 100644 --- a/src/desktop/ipc-bridge.ts +++ b/src/desktop/ipc-bridge.ts @@ -32,6 +32,7 @@ import { log } from '../server/log'; */ const LOCAL_IPC_ARGS: Record = { 'files:preview': [a.string], + 'files:previewData': [a.string, a.optional(a.nullish(a.string)), a.optional(a.nullish(a.string))], 'files:download': [a.string], 'cfolders:reveal': [a.string], 'client:pair': [a.string, a.string], diff --git a/src/desktop/local/index.ts b/src/desktop/local/index.ts index f7f0244..728e33e 100644 --- a/src/desktop/local/index.ts +++ b/src/desktop/local/index.ts @@ -2,7 +2,7 @@ import { app, dialog, shell, type BrowserWindow } from 'electron'; import { join } from 'node:path'; import { handleLocal } from '../ipc-bridge'; import { ensureFilesRoot } from '../../server/files/store'; -import { imagePreviewDataUrl } from '../../server/pi/attachments'; +import { imagePreviewDataUrl, imagePreviewFromBytes } from '../../server/pi/attachments'; import { connectedFolderPath } from '../../server/workspace/connected-folders'; import { workspaceRoot } from '../../server/workspace/paths'; import { exportState } from '../../server/workspace/state-transfer'; @@ -262,10 +262,16 @@ export function registerLocalIpc(deps: LocalIpcDeps): void { revealable('Your Files folder'); await shell.openPath(await ensureFilesRoot()); }); - // Read-only, and reached only from the `att.path` branch of - // renderer/attachments.ts — i.e. for an image the user picked or dropped, which - // by construction is on the client's own disk. + // Read-only, and reached from renderer/attachments.ts: a path for images the + // user picked or dropped (on this disk), or pasted bytes for HEIC that + // Chromium cannot paint until the OS decoder turns them into JPEG. handleLocal('files:preview', (_e, path: string) => imagePreviewDataUrl(path)); + // Pasted HEIC has no on-disk path; the renderer sends the bytes so Chromium + // can show a JPEG thumbnail (it cannot paint image/heic). + handleLocal( + 'files:previewData', + (_e, dataBase64: string, mime?: string, name?: string) => imagePreviewFromBytes(dataBase64, mime, name) + ); /** * Fetch one file out of the server's Files folder and put it where downloads diff --git a/src/desktop/proxy.ts b/src/desktop/proxy.ts index e4d627b..638344e 100644 --- a/src/desktop/proxy.ts +++ b/src/desktop/proxy.ts @@ -19,6 +19,7 @@ import { type StartTurnInput, type TurnAttachment } from '../shared/types'; +import { convertHeicAttachments } from '../server/pi/heic'; import { uploadFile } from './file-transfer'; import { createOfflineCache } from './offline-cache'; import type { OAuthCourier } from './oauth-courier'; @@ -59,10 +60,12 @@ import { updateClientQuickChat, withClientSettings } from './settings'; // about the build installed HERE // (see desktop/updates.ts) // dialog:openFiles, dialog:openDirectory native pickers -// files:reveal, files:preview shell.showItemInFolder; preview -// reads an image path that, by -// construction, is on the client's -// own disk (the `att.path` branch +// files:reveal, files:preview, files:previewData +// shell.showItemInFolder; preview +// reads an image path (or pasted +// bytes) that, by construction, is +// on the client's own disk (the +// `att.path` / dataBase64 branches // of renderer/attachments.ts) // files:download GET /files/, saved into this // machine's Downloads folder and @@ -112,16 +115,18 @@ import { updateClientQuickChat, withClientSettings } from './settings'; // push stream is one this machine asked for — the // stream is a broadcast, and every other device // paired to the same server sees it too. -// backend:startTurn, both carry paths to files on THIS disk, which is -// files:add only a thing the server can read when it is on +// backend:startTurn, HEIC is decoded to JPEG here first (macOS sips / +// files:add a PATH tool) so a remote Linux server never has +// to. Remaining paths to files on THIS disk are +// only a thing the server can read when it is on // this disk too. When it isn't, the bytes are // streamed up first and the paths are replaced with -// handles to them — see attachmentsForServer(). The -// REMOTE case only: a local install keeps handing -// over paths, because copying every pasted -// screenshot through loopback to prove a point -// would be a cost with nothing on the other side -// of it. +// handles to them — see attachmentsForTurn(). The +// REMOTE case only for the upload half: a local +// install keeps handing over paths, because copying +// every pasted screenshot through loopback to prove +// a point would be a cost with nothing on the other +// side of it. // // SERVER-OWNED — everything else (~110 channels). The server's registry IS the // surface; this client asks for it at connect time (GET /channels) rather than @@ -346,34 +351,33 @@ export function createServerProxy(deps: ProxyDeps): ServerProxy { const signInStarted: WrappedChannel = { before: () => deps.oauthCourier.expectSignIn() }; /** - * Replace every on-disk path in a set of attachments with a handle to bytes the - * server now has. Pasted images (`dataBase64`) already travel in the envelope - * and are left exactly as they are — they are small, they are already on the - * wire, and uploading them separately would be strictly more work. + * Decode HEIC to JPEG on THIS machine (sips / PATH tool) so a Mac client + * talking to a Linux server still works — the server never has to ship HEVC. + * Then, when remote, replace remaining on-disk paths with upload handles. * - * A failure here is deliberately fatal to the call. The alternative is sending - * the message with the attachment quietly missing, which reads to the user as - * the assistant ignoring the thing they attached; throwing instead leaves the - * message in the composer, with the reason on screen, ready to send again. + * Pasted images (`dataBase64`) already travel in the envelope. After a HEIC + * conversion they are JPEG `dataBase64` and are left exactly as they are. + * + * A failure to *upload* is deliberately fatal to the call. The alternative is + * sending the message with the attachment quietly missing, which reads to the + * user as the assistant ignoring the thing they attached; throwing instead + * leaves the message in the composer, with the reason on screen, ready to send + * again. A HEIC that will not decode is left unchanged so the server can skip + * it with a named note rather than failing the whole send. */ - async function attachmentsForServer(atts: TurnAttachment[]): Promise { + async function attachmentsForTurn(atts: TurnAttachment[]): Promise { + const converted = await convertHeicAttachments(atts); + if (!deps.remote) return converted; return Promise.all( - atts.map(async (att) => { + converted.map(async (att) => { if (!att.path) return att; return { ...att, path: await uploadFile({ url: base, token: deps.token }, att.path) }; }) ); } - /** The remote half of `backend:startTurn` and `files:add`; absent when local. */ + /** Path rewrite for Files-panel drops; turn attachments go through attachmentsForTurn. */ const uploadPaths: Record = { - 'backend:startTurn': { - before: async ([input]) => { - const turn = input as StartTurnInput; - if (!turn?.attachments?.length) return; - return [{ ...turn, attachments: await attachmentsForServer(turn.attachments) }]; - } - }, 'files:add': { before: async ([paths, subdir]) => { const list = paths as string[]; @@ -384,12 +388,21 @@ export function createServerProxy(deps: ProxyDeps): ServerProxy { } }; + const startTurnWrap: WrappedChannel = { + before: async ([input]) => { + const turn = input as StartTurnInput; + if (!turn?.attachments?.length) return; + return [{ ...turn, attachments: await attachmentsForTurn(turn.attachments) }]; + } + }; + const wrapped: Readonly> = { 'chats:open': { before: ([threadId]) => deps.threadOpened(threadId as string) }, 'auth:providerLogin': signInStarted, 'mcp:login': signInStarted, + 'backend:startTurn': startTurnWrap, ...(deps.remote ? uploadPaths : {}), ...Object.fromEntries(SETTINGS_CHANNELS.map((c) => [c, mergeSettingsAnswer])), 'settings:updateQuickChat': { diff --git a/src/preload/index.ts b/src/preload/index.ts index 9576841..34466d5 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -118,6 +118,8 @@ const api: StemApi = { revealFiles: () => ipcRenderer.invoke('files:reveal'), downloadFile: (rel: string) => ipcRenderer.invoke('files:download', rel), previewImage: (path: string) => ipcRenderer.invoke('files:preview', path), + previewImageData: (dataBase64: string, mime?: string, name?: string) => + ipcRenderer.invoke('files:previewData', dataBase64, mime, name), listConnectedFolders: () => ipcRenderer.invoke('cfolders:list'), addConnectedFolders: (paths: string[]) => ipcRenderer.invoke('cfolders:add', paths), diff --git a/src/renderer/attachments.ts b/src/renderer/attachments.ts index fa61e95..5282d28 100644 --- a/src/renderer/attachments.ts +++ b/src/renderer/attachments.ts @@ -6,7 +6,23 @@ import type { ChatMessage, MessageAttachment, TurnAttachment } from '../shared/types'; -const IMAGE_EXT = /\.(png|jpe?g|gif|webp)$/i; +const IMAGE_EXT = /\.(png|jpe?g|gif|webp|heic|heif|hif)$/i; + +function isHeic(att: TurnAttachment): boolean { + const mime = att.mime?.toLowerCase() ?? ''; + if (mime === 'image/jpeg' || mime === 'image/png' || mime === 'image/gif' || mime === 'image/webp') { + return false; + } + if ( + mime === 'image/heic' || + mime === 'image/heif' || + mime === 'image/heic-sequence' || + mime === 'image/heif-sequence' + ) { + return true; + } + return /\.(heic|heif|hif)$/i.test(att.name || att.path || ''); +} function isImage(att: TurnAttachment): boolean { if (att.mime?.toLowerCase().startsWith('image/')) return true; @@ -17,10 +33,12 @@ function isImage(att: TurnAttachment): boolean { * Synchronous attachment shape for the optimistic bubble. On-disk images need an * IPC read before their thumbnail is available, so they begin as ordinary chips * and are upgraded by `toMessageAttachments()` without delaying the send itself. + * Pasted HEIC is the same: Chromium cannot paint `image/heic`, so it starts as a + * chip until the main process has decoded it to JPEG. */ export function optimisticMessageAttachments(atts: TurnAttachment[]): MessageAttachment[] { return atts.map((att) => { - if (isImage(att) && att.dataBase64) { + if (isImage(att) && att.dataBase64 && !isHeic(att)) { const mime = att.mime || 'image/png'; return { kind: 'image', name: att.name, mime, dataUrl: `data:${mime};base64,${att.dataBase64}` }; } @@ -32,13 +50,21 @@ export async function toMessageAttachments(atts: TurnAttachment[]): Promise => { if (isImage(att)) { - const mime = att.mime || 'image/png'; if (att.dataBase64) { - return { kind: 'image', name: att.name, mime, dataUrl: `data:${mime};base64,${att.dataBase64}` }; + if (isHeic(att)) { + const dataUrl = await window.stem.previewImageData(att.dataBase64, att.mime, att.name); + if (dataUrl) return { kind: 'image', name: att.name, mime: 'image/jpeg', dataUrl }; + } else { + const mime = att.mime || 'image/png'; + return { kind: 'image', name: att.name, mime, dataUrl: `data:${mime};base64,${att.dataBase64}` }; + } } if (att.path) { const dataUrl = await window.stem.previewImage(att.path); - if (dataUrl) return { kind: 'image', name: att.name, mime, dataUrl }; + if (dataUrl) { + const mime = isHeic(att) ? 'image/jpeg' : att.mime || 'image/png'; + return { kind: 'image', name: att.name, mime, dataUrl }; + } } } return { kind: 'file', name: att.name }; diff --git a/src/server/pi/attachments.ts b/src/server/pi/attachments.ts index 2880800..1707a6a 100644 --- a/src/server/pi/attachments.ts +++ b/src/server/pi/attachments.ts @@ -2,9 +2,10 @@ // // pi's `prompt` RPC accepts images natively (`images: [{type:'image', data, mimeType}]`). // It has no slot for arbitrary files, so text-like files are inlined into the message as -// fenced blocks, PDFs are inlined as their extracted text layer, and other binary files -// are rejected. This module is the single place that reads attachment bytes (from -// `dataBase64` or an on-disk `path`) and classifies them. +// fenced blocks, PDFs are inlined as their extracted text layer, HEIC/HEIF is decoded +// to JPEG via the OS (see heic.ts), and other binary files are rejected. This module +// is the single place that reads attachment bytes (from `dataBase64` or an on-disk +// `path`) and classifies them. // // `path` is the CLIENT's path, which is only a path we can read when the client is on // this machine. A client whose server is elsewhere streams the bytes to POST /upload @@ -15,6 +16,7 @@ import { readFile } from 'node:fs/promises'; import { extname } from 'node:path'; import { isUploadHandle, resolveUploadHandle } from '../files/staging'; import { extractPdfText } from '../folder-index/pdf'; +import { heicToJpeg, isHeicAttachment, isHeicNameOrMime } from './heic'; import type { TurnAttachment } from '../../shared/types'; /** pi `ImageContent` — the shape of each entry in the prompt's `images` array. */ @@ -29,7 +31,7 @@ export interface ResolvedAttachments { images: PiImageContent[]; /** Fenced file contents appended to the message text. */ textBlocks: string[]; - /** Basenames of attachments skipped: unsupported binaries, unreadable bytes, or PDFs with no text layer. */ + /** Basenames of attachments skipped: unsupported binaries, unreadable bytes, PDFs with no text layer, or HEIC this machine cannot decode. */ rejected: string[]; } @@ -71,6 +73,8 @@ function imageMimeFor(att: TurnAttachment, ext: string): string | null { const mime = att.mime?.toLowerCase(); if (mime?.startsWith('image/')) { // Normalise to a type pi accepts; drop unknown image subtypes to the ext map. + // HEIC is not in this list — Chromium and most model APIs cannot take it, so + // it is decoded to JPEG first (see maybeHeicJpeg). if (mime === 'image/png' || mime === 'image/jpeg' || mime === 'image/gif' || mime === 'image/webp') { return mime; } @@ -78,6 +82,17 @@ function imageMimeFor(att: TurnAttachment, ext: string): string | null { return IMAGE_EXT_MIME[ext] ?? null; } +/** JPEG base64 for a HEIC attachment, or null if this machine cannot decode it. */ +async function maybeHeicJpeg( + att: TurnAttachment, + bytes: Buffer +): Promise<{ data: string; mimeType: 'image/jpeg' } | null> { + if (!isHeicAttachment(att, bytes)) return null; + const jpeg = await heicToJpeg(bytes); + if (!jpeg) return null; + return { data: jpeg.toString('base64'), mimeType: 'image/jpeg' }; +} + function looksTextual(att: TurnAttachment, ext: string, bytes: Buffer): boolean { if (att.mime?.toLowerCase().startsWith('text/')) return true; if (TEXT_EXT.has(ext)) return true; @@ -110,6 +125,31 @@ function fenceText(name: string, lang: string, body: string, truncated: boolean) return `Attached file: ${name}\n\`\`\`${lang}\n${body}${note}\n\`\`\``; } +/** + * Turn image bytes into a Chromium-displayable `data:` URL. HEIC is decoded to + * JPEG first — Electron cannot paint `image/heic`. Null if it isn't a supported + * image or this machine cannot decode it. + */ +export async function imagePreviewFromBytes( + dataBase64: string, + mime?: string, + name?: string +): Promise { + try { + const bytes = Buffer.from(dataBase64, 'base64'); + const jpeg = await maybeHeicJpeg({ name: name || 'image', mime, dataBase64 }, bytes); + if (jpeg) return `data:${jpeg.mimeType};base64,${jpeg.data}`; + const ext = extname(name || '').toLowerCase(); + const resolved = imageMimeFor({ name: name || 'image', mime }, ext); + if (!resolved) return null; + return `data:${resolved};base64,${dataBase64}`; + } catch { + // quiet: this is the thumbnail in the live bubble, not the attachment — the + // send path reads the same bytes again, and that read is the one that reports. + return null; + } +} + /** * Read an on-disk image and return a `data:` URL for an inline thumbnail, or null if the * file isn't a supported image or can't be read. Used to preview dialog/drop-picked @@ -118,9 +158,16 @@ function fenceText(name: string, lang: string, body: string, truncated: boolean) export async function imagePreviewDataUrl(path: string): Promise { const ext = extname(path).toLowerCase(); const mime = IMAGE_EXT_MIME[ext]; - if (!mime) return null; + const heic = isHeicNameOrMime({ name: path, path }); + if (!mime && !heic) return null; try { const bytes = await readFile(path); + if (heic || isHeicAttachment({ name: path, path }, bytes)) { + const jpeg = await heicToJpeg(bytes); + if (!jpeg) return null; + return `data:image/jpeg;base64,${jpeg.toString('base64')}`; + } + if (!mime) return null; return `data:${mime};base64,${bytes.toString('base64')}`; } catch { // quiet: this is the thumbnail in the live bubble, not the attachment — the @@ -137,7 +184,8 @@ export async function resolveAttachments(atts: TurnAttachment[]): Promise): boolean { + const mime = att.mime?.toLowerCase(); + // Already converted (client-side sips left the .heic name but set jpeg mime). + if (mime === 'image/jpeg' || mime === 'image/png' || mime === 'image/gif' || mime === 'image/webp') { + return false; + } + if (mime && HEIC_MIME.has(mime)) return true; + const ext = extname(att.name || att.path || '').toLowerCase(); + return HEIC_EXT.has(ext); +} + +/** True when the file's `ftyp` box is a HEIF brand. */ +export function looksLikeHeicBytes(bytes: Buffer): boolean { + if (bytes.length < 12) return false; + if (bytes.toString('ascii', 4, 8) !== 'ftyp') return false; + return HEIC_BRANDS.has(bytes.toString('ascii', 8, 12)); +} + +export function isHeicAttachment( + att: Pick, + bytes?: Buffer | null +): boolean { + if (isHeicNameOrMime(att)) return true; + return bytes ? looksLikeHeicBytes(bytes) : false; +} + +function looksLikeJpeg(bytes: Buffer): boolean { + return bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff; +} + +type Decoder = (bytes: Buffer) => Promise; + +let decoderOverride: Decoder | null = null; + +/** Swap the decoder in unit tests. Pass `null` to restore the system decoder. */ +export function setHeicDecoderForTests(decoder: Decoder | null): void { + decoderOverride = decoder; +} + +/** + * JPEG bytes, or null if this machine has no decoder or the file will not decode. + * Never throws. + */ +export async function heicToJpeg(bytes: Buffer): Promise { + try { + const out = await (decoderOverride ?? convertHeicWithSystem)(bytes); + if (!out || !looksLikeJpeg(out)) return null; + return out; + } catch { + // quiet: the caller treats null like any other unreadable attachment. + return null; + } +} + +/** + * If this attachment is HEIC and this machine can decode it, return JPEG bytes + * as `dataBase64` (path dropped — the original file stays on disk). Otherwise + * return the attachment unchanged so a later stage can try, or reject it. + */ +export async function convertHeicAttachment(att: TurnAttachment): Promise { + if (!isHeicNameOrMime(att)) return att; + let bytes: Buffer | null = null; + if (att.dataBase64) { + bytes = Buffer.from(att.dataBase64, 'base64'); + } else if (att.path) { + try { + bytes = await readFile(att.path); + } catch { + // quiet: leave the attachment as-is so resolveAttachments can skip it by + // name. Failing the send here would drop the whole turn for a photo the + // later stage already knows how to refuse. + return att; + } + } + if (!bytes) return att; + const jpeg = await heicToJpeg(bytes); + if (!jpeg) return att; + return { name: att.name, mime: 'image/jpeg', dataBase64: jpeg.toString('base64') }; +} + +export async function convertHeicAttachments(atts: TurnAttachment[]): Promise { + return Promise.all(atts.map(convertHeicAttachment)); +} + +function runTool(command: string, args: readonly string[]): Promise { + return new Promise((resolve) => { + try { + execFile(command, args, { timeout: CONVERT_TIMEOUT_MS, windowsHide: true }, (error) => { + resolve(!error); + }); + } catch { + // quiet: a missing PATH tool (heif-convert/magick) is the Linux/Windows + // default, not a defect. The caller tries the next decoder, then null. + resolve(false); + } + }); +} + +async function convertHeicWithSystem(bytes: Buffer): Promise { + const dir = await mkdtemp(join(tmpdir(), 'stem-heic-')); + const input = join(dir, 'in.heic'); + const output = join(dir, 'out.jpg'); + try { + await writeFile(input, bytes); + if (process.platform === 'darwin') { + const ok = await runTool('/usr/bin/sips', ['-s', 'format', 'jpeg', input, '--out', output]); + if (ok) return await readJpeg(output); + } + // User-supplied tools only. Never ship these; never `apt install` them into + // the official image. `convert` is skipped: on Windows it is a filesystem + // utility, not ImageMagick. + for (const [command, args] of [ + ['heif-convert', [input, output]], + ['magick', [input, output]] + ] as const) { + const ok = await runTool(command, args); + if (ok) return await readJpeg(output); + } + return null; + } finally { + await rm(dir, { recursive: true, force: true }).catch(() => { + // quiet: temp cleanup is best-effort; the next boot's OS temp sweep gets it. + }); + } +} + +async function readJpeg(path: string): Promise { + try { + const bytes = await readFile(path); + return looksLikeJpeg(bytes) ? bytes : null; + } catch { + // quiet: sips/heif-convert can exit 0 and still leave no readable JPEG. + // null is the same answer as a decoder that is not installed. + return null; + } +} diff --git a/src/shared/types.ts b/src/shared/types.ts index b7dfdfd..1fc8e8b 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -3039,6 +3039,11 @@ export interface StemApi { downloadFile(rel: string): Promise; /** Read an on-disk image → `data:` URL for a bubble thumbnail (null if not an image). */ previewImage(path: string): Promise; + /** + * Decode pasted image bytes → `data:` URL. HEIC is converted to JPEG first + * because Chromium cannot display it. Null if this machine cannot decode it. + */ + previewImageData(dataBase64: string, mime?: string, name?: string): Promise; // Connected folders: external folders the assistant reads in place. Mutations // return the fresh list. diff --git a/tests/unit/attachments.test.ts b/tests/unit/attachments.test.ts index 1ff79a3..b1444b2 100644 --- a/tests/unit/attachments.test.ts +++ b/tests/unit/attachments.test.ts @@ -3,10 +3,26 @@ // tests — a PDF used to be silently dropped as "unsupported", and the regression // mode (extraction quietly failing and falling back to the skip note) produces // a turn that reads as though the feature never existed. -import { describe, expect, it } from 'vitest'; -import { resolveAttachments } from '../../src/server/pi/attachments'; +import { afterEach, describe, expect, it } from 'vitest'; +import { imagePreviewFromBytes, resolveAttachments } from '../../src/server/pi/attachments'; +import { setHeicDecoderForTests } from '../../src/server/pi/heic'; import { makePdf } from './make-pdf'; +const MINI_JPEG = Buffer.from([0xff, 0xd8, 0xff, 0xd9]); + +function fakeHeic(): Buffer { + const buf = Buffer.alloc(16); + buf.writeUInt32BE(16, 0); + buf.write('ftyp', 4); + buf.write('heic', 8); + buf.write('mif1', 12); + return buf; +} + +afterEach(() => { + setHeicDecoderForTests(null); +}); + describe('resolveAttachments and PDFs', () => { it('inlines a PDF text layer as a fenced block', async () => { const pdf = makePdf([ @@ -53,3 +69,46 @@ describe('resolveAttachments and PDFs', () => { expect(resolved.textBlocks).toHaveLength(0); }); }); + +describe('resolveAttachments and HEIC', () => { + it('converts a HEIC attachment to a JPEG image block', async () => { + setHeicDecoderForTests(async () => MINI_JPEG); + const resolved = await resolveAttachments([ + { name: 'IMG_1.HEIC', dataBase64: fakeHeic().toString('base64') } + ]); + expect(resolved.rejected).toHaveLength(0); + expect(resolved.images).toEqual([ + { type: 'image', data: MINI_JPEG.toString('base64'), mimeType: 'image/jpeg' } + ]); + }); + + it('rejects a HEIC this machine cannot decode, naming the file', async () => { + setHeicDecoderForTests(async () => null); + const resolved = await resolveAttachments([ + { name: 'broken.heic', dataBase64: Buffer.from('not really heic\0').toString('base64') } + ]); + expect(resolved.rejected).toEqual(['broken.heic (could not decode HEIC)']); + expect(resolved.images).toHaveLength(0); + }); + + it('does not re-decode a HEIC that is already JPEG (client conversion)', async () => { + let called = 0; + setHeicDecoderForTests(async () => { + called += 1; + return MINI_JPEG; + }); + const resolved = await resolveAttachments([ + { name: 'IMG_1.HEIC', mime: 'image/jpeg', dataBase64: MINI_JPEG.toString('base64') } + ]); + expect(called).toBe(0); + expect(resolved.images).toEqual([ + { type: 'image', data: MINI_JPEG.toString('base64'), mimeType: 'image/jpeg' } + ]); + }); + + it('builds a JPEG data URL for a pasted HEIC thumbnail', async () => { + setHeicDecoderForTests(async () => MINI_JPEG); + const url = await imagePreviewFromBytes(fakeHeic().toString('base64'), 'image/heic', 'paste.heic'); + expect(url).toBe(`data:image/jpeg;base64,${MINI_JPEG.toString('base64')}`); + }); +}); diff --git a/tests/unit/heic.test.ts b/tests/unit/heic.test.ts new file mode 100644 index 0000000..c9f6cd2 --- /dev/null +++ b/tests/unit/heic.test.ts @@ -0,0 +1,125 @@ +// HEIC detection and OS-decoder conversion. The decoder is mocked except for a +// macOS smoke that uses real `sips` — that's the path Stem ships, and a mock +// cannot catch a sips flag regression. +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + convertHeicAttachment, + heicToJpeg, + isHeicNameOrMime, + looksLikeHeicBytes, + setHeicDecoderForTests +} from '../../src/server/pi/heic'; + +const MINI_JPEG = Buffer.from([0xff, 0xd8, 0xff, 0xd9]); + +/** ISO BMFF `ftypheic` header — enough for brand detection, not a real photo. */ +function fakeHeic(): Buffer { + const buf = Buffer.alloc(16); + buf.writeUInt32BE(16, 0); + buf.write('ftyp', 4); + buf.write('heic', 8); + buf.write('mif1', 12); + return buf; +} + +afterEach(() => { + setHeicDecoderForTests(null); +}); + +describe('HEIC detection', () => { + it('recognises names, MIME types, and ftyp brands', () => { + expect(isHeicNameOrMime({ name: 'IMG_1.HEIC' })).toBe(true); + expect(isHeicNameOrMime({ name: 'shot.heif' })).toBe(true); + expect(isHeicNameOrMime({ name: 'x', mime: 'image/heic' })).toBe(true); + expect(isHeicNameOrMime({ name: 'photo.png' })).toBe(false); + // Client-side conversion keeps the .heic name but sets jpeg mime. + expect(isHeicNameOrMime({ name: 'IMG_1.HEIC', mime: 'image/jpeg' })).toBe(false); + expect(looksLikeHeicBytes(fakeHeic())).toBe(true); + expect(looksLikeHeicBytes(MINI_JPEG)).toBe(false); + expect(looksLikeHeicBytes(Buffer.from('hello'))).toBe(false); + }); +}); + +describe('convertHeicAttachment', () => { + it('rewrites a HEIC attachment to JPEG dataBase64', async () => { + setHeicDecoderForTests(async () => MINI_JPEG); + const next = await convertHeicAttachment({ + name: 'IMG_1.HEIC', + path: '/tmp/IMG_1.HEIC', + dataBase64: fakeHeic().toString('base64') + }); + expect(next).toEqual({ + name: 'IMG_1.HEIC', + mime: 'image/jpeg', + dataBase64: MINI_JPEG.toString('base64') + }); + expect(next.path).toBeUndefined(); + }); + + it('leaves a non-HEIC attachment alone', async () => { + const att = { name: 'note.txt', path: '/tmp/note.txt' }; + expect(await convertHeicAttachment(att)).toEqual(att); + }); + + it('leaves a HEIC unchanged when the decoder returns null', async () => { + setHeicDecoderForTests(async () => null); + const att = { name: 'broken.heic', dataBase64: 'not-a-photo' }; + expect(await convertHeicAttachment(att)).toEqual(att); + }); + + it('reads HEIC from disk when there is no dataBase64', async () => { + setHeicDecoderForTests(async () => MINI_JPEG); + const dir = await mkdtemp(join(tmpdir(), 'stem-heic-att-')); + const path = join(dir, 'shot.heic'); + try { + await writeFile(path, fakeHeic()); + const next = await convertHeicAttachment({ name: 'shot.heic', path }); + expect(next).toEqual({ + name: 'shot.heic', + mime: 'image/jpeg', + dataBase64: MINI_JPEG.toString('base64') + }); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); + +describe('heicToJpeg with the system decoder', () => { + it.skipIf(process.platform !== 'darwin')('round-trips a sips-made HEIC to JPEG', async () => { + const dir = await mkdtemp(join(tmpdir(), 'stem-heic-smoke-')); + const pngPath = join(dir, 'in.png'); + const heicPath = join(dir, 'in.heic'); + // 1×1 PNG — sips can re-encode this as HEIC on any recent macOS. + const png = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==', + 'base64' + ); + try { + await writeFile(pngPath, png); + const made = await new Promise((resolve) => { + execFile('/usr/bin/sips', ['-s', 'format', 'heic', pngPath, '--out', heicPath], (error) => { + resolve(!error); + }); + }); + // Encode can be blocked in a sandbox even though decode works in the app. + if (!made) return; + const jpeg = await heicToJpeg(await readFile(heicPath)); + expect(jpeg).not.toBeNull(); + expect(jpeg![0]).toBe(0xff); + expect(jpeg![1]).toBe(0xd8); + expect(jpeg![2]).toBe(0xff); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('returns null for garbage without throwing', async () => { + setHeicDecoderForTests(null); + await expect(heicToJpeg(Buffer.from('not really heic\0'))).resolves.toBeNull(); + }); +}); diff --git a/tests/unit/renderer-regressions.test.ts b/tests/unit/renderer-regressions.test.ts index ea31c87..a6ad340 100644 --- a/tests/unit/renderer-regressions.test.ts +++ b/tests/unit/renderer-regressions.test.ts @@ -1,8 +1,8 @@ import { createElement } from 'react'; import { renderToStaticMarkup } from 'react-dom/server'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import type { ChatMessage, QuickChatHandoff, TurnAttachment } from '../../src/shared/types'; -import { optimisticMessageAttachments, resendAttachments } from '../../src/renderer/attachments'; +import { optimisticMessageAttachments, resendAttachments, toMessageAttachments } from '../../src/renderer/attachments'; import { EMPTY_STATE, mergeDraftIntoReal, @@ -157,6 +157,30 @@ describe('renderer async race regressions', () => { { kind: 'image', name: 'paste.png', mime: 'image/png', dataUrl: 'data:image/png;base64,YWJj' } ]); }); + + it('keeps pasted HEIC as a chip until the main process decodes it to JPEG', () => { + // Chromium cannot paint data:image/heic, so a JPEG data URL here would be a lie + // and a HEIC data URL would show a broken thumbnail. + expect( + optimisticMessageAttachments([{ name: 'IMG_1.HEIC', mime: 'image/heic', dataBase64: 'YWJj' }]) + ).toEqual([{ kind: 'file', name: 'IMG_1.HEIC' }]); + }); + + it('upgrades pasted HEIC through previewImageData', async () => { + const previewImageData = vi.fn(async () => 'data:image/jpeg;base64,eA=='); + vi.stubGlobal('window', { stem: { previewImageData, previewImage: vi.fn() } }); + try { + const next = await toMessageAttachments([ + { name: 'IMG_1.HEIC', mime: 'image/heic', dataBase64: 'YWJj' } + ]); + expect(previewImageData).toHaveBeenCalledWith('YWJj', 'image/heic', 'IMG_1.HEIC'); + expect(next).toEqual([ + { kind: 'image', name: 'IMG_1.HEIC', mime: 'image/jpeg', dataUrl: 'data:image/jpeg;base64,eA==' } + ]); + } finally { + vi.unstubAllGlobals(); + } + }); }); describe('main-to-renderer lifecycle regressions', () => { diff --git a/tests/unit/transport-conformance.test.ts b/tests/unit/transport-conformance.test.ts index f02e382..d819f5b 100644 --- a/tests/unit/transport-conformance.test.ts +++ b/tests/unit/transport-conformance.test.ts @@ -39,6 +39,7 @@ const CLIENT_OWNED = [ 'dialog:openDirectory', 'files:reveal', 'files:preview', + 'files:previewData', 'cfolders:reveal', 'cfolders:revealWorkspace', 'quickchat:newThread',